From e6d1206c4501fc3032abca9774cdfa52662cfc70 Mon Sep 17 00:00:00 2001 From: Paul Bakaus Date: Mon, 13 Jul 2026 10:47:44 -0700 Subject: [PATCH] Update Live Lab production evidence AI-assisted: Codex --- site/data/live-codex-worker-result.ts | 37 ++++++++++++++++++++++----- site/data/live-harnesses.ts | 30 +++++++++++----------- site/pages/live-lab/index.astro | 32 ++++++++++++++++++++--- 3 files changed, 73 insertions(+), 26 deletions(-) diff --git a/site/data/live-codex-worker-result.ts b/site/data/live-codex-worker-result.ts index 17fb76241..819ab733c 100644 --- a/site/data/live-codex-worker-result.ts +++ b/site/data/live-codex-worker-result.ts @@ -49,14 +49,37 @@ export const liveCodexWorkerResult = { warmRepeatCacheRate: 0.823, }, productionBrowser: { + runs: 5, + firstVariantMs: 19283, + firstVariantP95Ms: 20412, + allVariantsMs: 52795, + allVariantsP95Ms: 55115, + deliveryGapMs: 33308, + serverPickupMs: 72.81, + sourcePreflightMs: 44, + validationMs: 5, + acceptRuns: 1, + acceptToResetMs: 196.5, + acceptToNextGoDispatchMs: 790.94, + acceptToNextFirstVariantMs: 23820, + blockedTailBaselineMs: 68141, + }, + renderedQuality: { + runs: 3, + variants: 9, + passed: 1, + commandFidelity: 5.67, + brandFidelity: 5, + renderQuality: 5.44, + taskCompletion: 5.11, + }, + atomicExperiment: { runs: 1, - firstVariantMs: 9868.3, - allVariantsMs: 31151.17, - acceptToResetMs: 298.71, - acceptToNextGoDispatchMs: 902.72, - acceptToNextFirstVariantMs: 16668, - sourcePreflightMs: 52, - validationMs: 10, + firstVariantMs: 40225, + allVariantsMs: 40227, + variants: 3, + passed: 1, + status: 'Opt-in experiment', }, annotatedBrowser: { runs: 2, diff --git a/site/data/live-harnesses.ts b/site/data/live-harnesses.ts index 422213e74..054918282 100644 --- a/site/data/live-harnesses.ts +++ b/site/data/live-harnesses.ts @@ -49,15 +49,15 @@ export const liveExperiments = [ { rank: 2, title: 'Accept any arrived variant — shipped', - evidence: 'A real Vite/React + Sol run accepted variant 1, interrupted variants 2–3, returned the page in 299 ms, and dispatched the next Go 903 ms after Accept.', - move: 'Durably fence generation on Accept/Discard, prioritize control events, publish through an epoch + source-hash transaction, and keep a separate poll supervisor active.', + evidence: 'A real Vite/React + Sol run returned the page in 197 ms and dispatched the next Go in 791 ms. Rotating the canceled app-server thread cut Accept → next first variant from 68.1 s to 23.8 s.', + move: 'Durably fence generation on Accept/Discard, rotate the canceled generation thread, prioritize control events, and keep a separate poll supervisor active.', expected: 'Removes the full-set wait and keeps the page interactive while canceled work unwinds.', confidence: 'Production browser', }, { rank: 3, title: 'Reveal variants progressively — shipped', - evidence: 'The production run showed variant 1 at 9.87 s and the full set at 31.2 s. Progressive publication removed 21.3 s from the first decision.', + evidence: 'Five production Sol/medium runs showed variant 1 at 19.3 s median / 20.4 s p95 and the full set at 52.8 s median / 55.1 s p95.', move: 'Codex publishes a monotonically growing prefix as each variant validates. The browser reveals arrived variants and pending dots immediately; Accept and Discard fence unfinished work.', expected: 'Lets review begin as soon as one credible option exists.', confidence: 'Production browser', @@ -89,26 +89,26 @@ export const liveExperiments = [ { rank: 7, title: 'Acknowledge first, clean up off-path — shipped', - evidence: 'The production page returned to Pick in 299 ms and dispatched new work while carbonize cleanup remained on its separate foreground control lane.', - move: 'Keep the durable accept acknowledgement on the foreground path, then hand carbonize cleanup to a source-locked worker while the poll supervisor leases new work.', + evidence: 'The production page returned to Pick in 197 ms. Cleanup is durably enqueued before acknowledgement, and replayed Accept events are idempotent after a worker or network failure.', + move: 'Persist the Accept receipt, enqueue carbonize cleanup before replying, then let the poll supervisor lease new work while cleanup stays source-locked.', expected: 'Preserves immediate interaction without leaving temporary Live source behind.', confidence: 'Production browser + build control', }, { rank: 8, - title: 'Trim the producer contract — selected', - evidence: 'Progressive compact passed 5/5 for Claude Sonnet 4.6 and GPT-5.5. A repeated warm Sol turn cached 38,656 of 46,973 input tokens (82.3%) and completed in 21.8 s versus 26.0 s cold.', - move: 'Keep the stable generation contract cacheable and send only the picked-element context plus the current action and identity lock.', - expected: 'Use provider caching instead of risky context deltas; always send current project and brand evidence.', - confidence: 'Candidate matrix + token telemetry', + title: 'Protect rendered quality before trimming context — blocking', + evidence: 'Only 1 of 9 rendered /bolder variants passed the multimodal gate. A compact-prompt trial made the progressive tail slower and passed 0/3, so it was reverted.', + move: 'Keep the full skill, sub-command, and project context. Improve diversity and fidelity against screenshots before pursuing prompt-size or lower-reasoning shortcuts.', + expected: 'Prevents a latency win that produces off-brand or structurally weak frontend work.', + confidence: 'Three production runs + rendered judge', }, { rank: 9, - title: 'Parallelize ideas only where reliable — provider-specific', - evidence: 'Parallel compact passed 5/5 for GPT-5.5 and Gemini 3.1 Flash-Lite, but only 3/5 for Claude Sonnet 4.6.', - move: 'Use parallel compact for Gemini and as an OpenAI full-set option. Keep progressive compact for Anthropic until its validation reliability improves.', - expected: 'Lowers full-set latency without shared-file races or pretending one call shape fits every provider.', - confidence: 'Sixty-run paid matrix', + title: 'Keep atomic delivery as an opt-in experiment', + evidence: 'One production Sol/medium run completed all three variants in 40.2 s versus 56.1 s progressive median, but delayed first review by about 18 s and only 1/3 rendered variants passed.', + move: 'Expose atomic in the benchmark and configuration surface, but do not make it the Codex default until repeated rendered runs clear the same quality gate.', + expected: 'Preserves a promising full-set path without regressing the default first-review experience.', + confidence: 'One production run · needs replication', }, ]; diff --git a/site/pages/live-lab/index.astro b/site/pages/live-lab/index.astro index 368aada9b..4a70facf9 100644 --- a/site/pages/live-lab/index.astro +++ b/site/pages/live-lab/index.astro @@ -406,9 +406,33 @@ const providerColumns = [
Dedicated Codex worker

- A real Pick → /bolder → Go run produced variant 1 in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} and all three in {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}. Accept returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}; the next Go reached the independent worker {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)} after Accept. The annotated browser path preserved its screenshot, comment, and stroke, then produced variant 1 in {displayMs(liveCodexWorkerResult.annotatedBrowser.firstVariantMs)} median. In matched full tasks, warm app-server reached usable output in {displayMs(liveCodexWorkerResult.architectureComparison.warmAppServerFirstUsableMs)} versus {displayMs(liveCodexWorkerResult.architectureComparison.directFirstUsableMs)} direct; both passed 4/4. Plain browser figures are one run; annotated figures are two. + Five real Pick → /bolder → Go runs produced variant 1 in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} median and all three in {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}. Accept returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}; rotating the canceled producer cut Accept → next first variant from {displayMs(liveCodexWorkerResult.productionBrowser.blockedTailBaselineMs)} to {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextFirstVariantMs)}. In matched full tasks, warm app-server reached usable output in {displayMs(liveCodexWorkerResult.architectureComparison.warmAppServerFirstUsableMs)} versus {displayMs(liveCodexWorkerResult.architectureComparison.directFirstUsableMs)} direct; both passed 4/4.

+ +
+ + + + + + + + + + + + + + + + + + + +
Production Codex delivery candidates and quality evidence
CandidateFirst reviewFull setRendered gateDecision
Progressive{liveCodexWorkerResult.productionBrowser.runs} Sol/medium runs{displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)}p95 {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantP95Ms)}{displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}p95 {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsP95Ms)}{liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants}production /bolder variantsDefault · quality work next
Atomic{liveCodexWorkerResult.atomicExperiment.runs} Sol/medium run{displayMs(liveCodexWorkerResult.atomicExperiment.firstVariantMs)}waits for trio{displayMs(liveCodexWorkerResult.atomicExperiment.allVariantsMs)}single turn{liveCodexWorkerResult.atomicExperiment.passed}/{liveCodexWorkerResult.atomicExperiment.variants}production /bolder variantsOpt-in · replicate
+
+

Timing and rendered-quality samples are reported separately. The current blocker is output quality, not worker pickup: server pickup is {displayMs(liveCodexWorkerResult.productionBrowser.serverPickupMs)} median.

@@ -441,7 +465,7 @@ const providerColumns = [

bun run bench:live boots a real framework fixture and Chromium, drives Pick → Go → Cycle, and records browser preparation, server pickup, scaffold, generation, write, and render boundaries. - Add --agent=codex --action=bolder to run the production app-server worker, or --acceptFirst to measure Accept → next Go. + Add --agent=codex --action=bolder to run the production app-server worker, --accept-first to measure Accept → next Go, --render-quality for screenshot evidence, or --delivery=atomic for the opt-in atomic candidate. node scripts/benchmark-live-init.mjs --iterations 10 measures configured cold and warm helper initialization separately. The deterministic agent makes Impeccable overhead visible; model-backed runs remain opt-in because they send fixture context to an external provider.

@@ -476,9 +500,9 @@ const providerColumns = [
Accept → next Go
{displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)}
Live returns
{displayMs(liveCodexWorkerResult.prewarmReturnMs)}
Worker ready
{displayMs(liveCodexWorkerResult.coldWorkerReadyMs)}
-
Quality gate
{liveCodexWorkerResult.quality.passed}/{liveCodexWorkerResult.quality.runs}
+
Rendered gate
{liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants}
-

Default in Codex; other harnesses keep the foreground path.

+

Progressive remains the Codex default. Atomic is measured but opt-in; rendered quality is the next gate.