From 50fc88ec8516caf0b84a6e06490257332cb86df5 Mon Sep 17 00:00:00 2001 From: Paul Bakaus Date: Mon, 13 Jul 2026 13:09:29 -0700 Subject: [PATCH] Update Live Lab performance evidence AI-assisted implementation under maintainer direction. --- site/data/live-codex-worker-result.ts | 66 +++++++++++++++------------ site/data/live-harnesses.ts | 10 ++-- site/pages/live-lab/index.astro | 17 +++---- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/site/data/live-codex-worker-result.ts b/site/data/live-codex-worker-result.ts index 2e6c6eac6..df3e66334 100644 --- a/site/data/live-codex-worker-result.ts +++ b/site/data/live-codex-worker-result.ts @@ -1,5 +1,5 @@ export const liveCodexWorkerResult = { - generatedAt: '2026-07-13T19:06:57.179Z', + generatedAt: '2026-07-13T20:01:08.415Z', codexVersion: '0.144.0-alpha.4', transport: 'stdio JSONL', model: 'GPT-5.6-Sol', @@ -51,22 +51,16 @@ export const liveCodexWorkerResult = { productionBrowser: { runs: 1, selectedTarget: 'article.offer-card', - firstVariantMs: 34177.39, - firstVariantP95Ms: 34177.39, - secondVariantMs: 54156.22, - allVariantsMs: 122089.41, - allVariantsP95Ms: 122089.41, - firstToSecondMs: 19978.83, - secondToAllMs: 67933.19, - serverPickupMs: 54.02, - sourcePreflightMs: 43, - validationMs: 8.67, - acceptRuns: 1, - acceptedVariant: 2, - acceptToResetMs: 231.1, - acceptToNextGoDispatchMs: 833.68, - acceptToNextFirstVariantMs: 52612.73, - blockedTailBaselineMs: 68141, + firstVariantMs: 29682.58, + firstVariantP95Ms: 29682.58, + secondVariantMs: 54597.81, + allVariantsMs: 102182.9, + allVariantsP95Ms: 102182.9, + firstToSecondMs: 24915.23, + secondToAllMs: 47585.09, + serverPickupMs: 76.82, + sourcePreflightMs: 45, + validationMs: 9.33, }, fullSourceControl: { runs: 1, @@ -77,11 +71,26 @@ export const liveCodexWorkerResult = { firstToSecondMs: 43874.54, }, sourceDeltaExperiment: { + runs: 5, + fullSourceFirstVariantMs: 34177.39, + deltaFirstVariantMs: 28074.92, + firstVariantImprovement: 0.1786, baselineFirstToSecondMs: 43874.54, - deltaFirstToSecondMs: 19978.83, - improvement: 0.5446, - variant2Passed: true, - status: 'Enabled for Codex source mode', + deltaFirstToSecondMs: 24624.41, + secondVariantImprovement: 0.4388, + finalDeltaAllVariantsMs: 102182.9, + status: 'Enabled for all Codex source-mode phases', + }, + earlyAcceptVerified: { + runs: 1, + acceptedVariant: 2, + sourceCommitted: true, + followupCleanupPassed: true, + acceptToResetMs: 238.18, + acceptToNextGoDispatchMs: 831.36, + acceptToNextFirstVariantMs: 16938.05, + controlNextFirstVariantMs: 42969.34, + nextFirstImprovement: 0.6058, }, standbyExperiment: { controlFirstVariantMs: 33283.06, @@ -112,12 +121,13 @@ export const liveCodexWorkerResult = { renderedQuality: { runs: 1, variants: 3, - passed: 1, - commandFidelity: 6.33, - brandFidelity: 7.33, - renderQuality: 6.33, - taskCompletion: 5.67, - variant2Passed: true, + passed: 2, + commandFidelity: 6.67, + brandFidelity: 7.67, + renderQuality: 6.67, + taskCompletion: 6.33, + variant1Passed: true, + variant3Passed: true, }, calibratedHeadingQuality: { runs: 3, @@ -146,5 +156,5 @@ export const liveCodexWorkerResult = { }, result: '4/4 full-task quality gate', passed: true, - architecture: 'A dedicated persistent Live app-server connection and worker thread with native Impeccable skill, sub-command, project context, and annotation-image inputs. Variants 1 and 2 publish as independent reviewable checkpoints; source-mode variant 2 uses a fenced delta while variant 3 and parameters finish last. Accept interrupts unfinished variants; cleanup stays on the foreground control lane while the next generation starts independently.', + architecture: 'A dedicated persistent Live app-server connection and worker thread with native Impeccable skill, sub-command, project context, and annotation-image inputs. Every source-mode variant publishes through an independently validated fenced delta; the final delta adds deferred parameter manifests without rewriting prior output. Accept interrupts unfinished variants; cleanup stays on the foreground control lane while the next generation starts independently.', } as const; diff --git a/site/data/live-harnesses.ts b/site/data/live-harnesses.ts index e7c29ccf2..10f1fac1e 100644 --- a/site/data/live-harnesses.ts +++ b/site/data/live-harnesses.ts @@ -49,17 +49,17 @@ export const liveExperiments = [ { rank: 2, title: 'Accept any arrived variant — shipped', - evidence: 'A real Vite/React + Sol run returned the page in 197 ms and dispatched the next Go in 791 ms. Rotating the canceled app-server thread cut Accept → next first variant from 68.1 s to 23.8 s.', + evidence: 'A verified Vite/React + Sol run committed variant 2 to source, returned the page in 238 ms, and dispatched the next Go in 831 ms. The next first variant arrived in 16.9 s versus the 43.0 s control, a 60.6% reduction.', move: 'Durably fence generation on Accept/Discard, rotate the canceled generation thread, prioritize control events, and keep a separate poll supervisor active.', expected: 'Removes the full-set wait and keeps the page interactive while canceled work unwinds.', confidence: 'Production browser', }, { rank: 3, - title: 'Send variant 2 as a fenced delta — shipped', - evidence: 'On the proven article.offer-card target, variant 2 arrived 20.0 s after variant 1 instead of 43.9 s with full-source output, a 54.5% reduction. The rendered judge passed variant 2 at 7/8/7/7.', - move: 'Keep the full skill, plan, project context, and source evidence, but return only variant 2 markup and scoped CSS. The supervisor validates its variant identity and merges it transactionally without exposing prior output to rewrites.', - expected: 'Makes the second credible option reviewable much sooner without weakening design context.', + title: 'Send every source variant as a fenced delta — shipped', + evidence: 'Across five Sol/medium samples, first review was 28.1 s median versus the 34.2 s full-source control. Variant 2 followed 24.6 s later versus 43.9 s, and the guarded all-delta run completed in 102.2 s with 2/3 rendered variants passing and 7.7/10 average brand fidelity.', + move: 'Keep the full skill, plan, project context, and source evidence, but return only each new variant’s markup, scoped CSS, and the final deferred parameter manifest. The supervisor validates and merges each delta without exposing prior output to rewrites.', + expected: 'Shrinks every model response while preserving full design context and immutable review checkpoints.', confidence: 'Production browser + rendered gate', }, { diff --git a/site/pages/live-lab/index.astro b/site/pages/live-lab/index.astro index f9a2e86d4..48fd83542 100644 --- a/site/pages/live-lab/index.astro +++ b/site/pages/live-lab/index.astro @@ -406,7 +406,7 @@ const providerColumns = [
Dedicated Codex worker

- With the target proven as {liveCodexWorkerResult.productionBrowser.selectedTarget}, variant 1 arrived in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} and the fenced variant-2 delta followed {displayMs(liveCodexWorkerResult.productionBrowser.firstToSecondMs)} later, {(liveCodexWorkerResult.sourceDeltaExperiment.improvement * 100).toFixed(1)}% faster than full-source output. Variant 2 passed the independent rendered gate. Accepting it while variant 3 was pending returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)} and dispatched the next Go in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)}. + With the target proven as {liveCodexWorkerResult.productionBrowser.selectedTarget}, fenced deltas delivered first review in {displayMs(liveCodexWorkerResult.sourceDeltaExperiment.deltaFirstVariantMs)} median and cut the variant-2 gap by {(liveCodexWorkerResult.sourceDeltaExperiment.secondVariantImprovement * 100).toFixed(1)}%. Accepting variant 2 while variant 3 was pending committed it to source, returned the page in {displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToResetMs)}, and produced the next first variant in {displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToNextFirstVariantMs)}.

@@ -416,12 +416,12 @@ const providerColumns = [ CandidateVariant 1Variant 2Full setRendered gateDecision - Root · delta v2{liveCodexWorkerResult.productionBrowser.runs} Sol/medium run + Root · all deltas{liveCodexWorkerResult.productionBrowser.runs} guarded Sol/medium run {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)}p95 {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantP95Ms)} {displayMs(liveCodexWorkerResult.productionBrowser.secondVariantMs)}independently acceptable {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}p95 {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsP95Ms)} - v2 passed{liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants} full-set variants - Default · 54.5% faster v2 + {liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants} passed{liveCodexWorkerResult.renderedQuality.brandFidelity.toFixed(1)}/10 brand fidelity + Default · immutable deltas Root · full source{liveCodexWorkerResult.fullSourceControl.runs} Sol/medium control @@ -450,7 +450,7 @@ const providerColumns = [ -

Root-task samples prove target identity in the posted Generate payload. Speculative standby warmup was rejected: it slowed the next result to {displayMs(liveCodexWorkerResult.standbyExperiment.prewarmedNextFirstVariantMs)} versus {displayMs(liveCodexWorkerResult.standbyExperiment.controlNextFirstVariantMs)}. Next gates: replicate root-level rendered quality and reduce the cold post-Accept generation wait; server pickup is already {displayMs(liveCodexWorkerResult.productionBrowser.serverPickupMs)}.

+

Root-task samples prove target identity in the posted Generate payload. Speculative standby warmup was rejected: it slowed the next result to {displayMs(liveCodexWorkerResult.standbyExperiment.prewarmedNextFirstVariantMs)} versus {displayMs(liveCodexWorkerResult.standbyExperiment.controlNextFirstVariantMs)}. The next gate is consistent 3/3 rendered quality; server pickup is already {displayMs(liveCodexWorkerResult.productionBrowser.serverPickupMs)} and post-Accept next review is down to {displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToNextFirstVariantMs)}.

@@ -516,13 +516,14 @@ const providerColumns = [
Second review
{displayMs(liveCodexWorkerResult.productionBrowser.secondVariantMs)}
v1 → v2
{displayMs(liveCodexWorkerResult.productionBrowser.firstToSecondMs)}
All variants
{displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}
-
Accept → page ready
{displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}
-
Accept → next Go
{displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)}
+
Accept → page ready
{displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToResetMs)}
+
Accept → next Go
{displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToNextGoDispatchMs)}
+
Accept → next review
{displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToNextFirstVariantMs)}
Live returns
{displayMs(liveCodexWorkerResult.prewarmReturnMs)}
Worker ready
{displayMs(liveCodexWorkerResult.coldWorkerReadyMs)}
Rendered gate
{liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants}
-

Variant 2 is independently reviewable, acceptable, and 54.5% faster with fenced source deltas. Cold post-Accept generation and replicated full-set quality remain the next gates.

+

Every source variant now arrives through an immutable fenced delta. Early Accept is source-verified and the next review is 60.6% faster; consistent 3/3 rendered quality remains the next gate.