diff --git a/scripts/lib/live-benchmark.mjs b/scripts/lib/live-benchmark.mjs index 0810ea838..a6302c7e8 100644 --- a/scripts/lib/live-benchmark.mjs +++ b/scripts/lib/live-benchmark.mjs @@ -182,6 +182,7 @@ export function summarizeSetup(events) { const stages = [ ['dependencies', 'setup.install.start', 'setup.install.end'], ['liveServer', 'setup.live_server.start', 'setup.live_server.end'], + ['codexWorker', 'setup.worker.start', 'setup.worker.end'], ['injection', 'setup.inject.start', 'setup.inject.end'], ['devServer', 'setup.dev_server.start', 'setup.dev_server.end'], ['pageLoad', 'setup.page_load.start', 'setup.page_load.end'], diff --git a/site/data/live-codex-worker-result.ts b/site/data/live-codex-worker-result.ts index de02b37b6..ffe619237 100644 --- a/site/data/live-codex-worker-result.ts +++ b/site/data/live-codex-worker-result.ts @@ -1,12 +1,12 @@ export const liveCodexWorkerResult = { - generatedAt: '2026-07-13T03:03:13.899Z', + generatedAt: '2026-07-13T03:57:44.263Z', codexVersion: '0.144.0-alpha.4', transport: 'stdio JSONL', model: 'GPT-5.6-Sol', effort: 'medium', profile: 'quality', defaultInCodex: true, - coldWorkerReadyMs: 5600, + coldWorkerReadyMs: 656.82, timings: { coldHandshakeMs: 36.83, coldThreadStartMs: 400.19, @@ -31,7 +31,30 @@ export const liveCodexWorkerResult = { sparkControlRuns: 2, sparkMedianGenerationMs: 16470.5, }, + architectureComparison: { + directFirstUsableMs: 64649.73, + coldAppServerFirstUsableMs: 35591.71, + warmAppServerFirstUsableMs: 34931.08, + directQualityPassed: 4, + directQualityRuns: 4, + warmQualityPassed: 4, + warmQualityRuns: 4, + annotatedQualityPassed: 3, + annotatedQualityRuns: 3, + coldInputTokens: 41066, + coldOutputTokens: 2148, + }, + productionBrowser: { + runs: 1, + firstVariantMs: 9868.3, + allVariantsMs: 31151.17, + acceptToResetMs: 298.71, + acceptToNextGoDispatchMs: 902.72, + acceptToNextFirstVariantMs: 16668, + sourcePreflightMs: 52, + validationMs: 10, + }, result: '4/4 full-task quality gate', passed: true, - architecture: 'A dedicated persistent Live app-server connection and worker thread with native Impeccable skill, sub-command, project context, and annotation-image inputs. Do not attach a second app-server to the desktop task.', + architecture: 'A dedicated persistent Live app-server connection and worker thread with native Impeccable skill, sub-command, project context, and annotation-image inputs. Accept interrupts unfinished variants; cleanup stays on the foreground control lane while the next generation can start independently.', } as const; diff --git a/site/pages/live-lab/index.astro b/site/pages/live-lab/index.astro index 8069fb033..fbcf69e36 100644 --- a/site/pages/live-lab/index.astro +++ b/site/pages/live-lab/index.astro @@ -406,7 +406,7 @@ const providerColumns = [
Dedicated Codex worker

- Codex now defaults to {liveCodexWorkerResult.model} at {liveCodexWorkerResult.effort} effort in a separate Live-owned thread. It passed {liveCodexWorkerResult.quality.passed}/{liveCodexWorkerResult.quality.runs} full /bolder and /polish tasks at a {displayMs(liveCodexWorkerResult.quality.medianGenerationMs)} median; the Spark control passed {liveCodexWorkerResult.quality.sparkControlPassed}/{liveCodexWorkerResult.quality.sparkControlRuns}. Protocol wake is {displayMs(liveCodexWorkerResult.timings.wakeToTurnStartedMs)}; the measured cold worker reached ready in {displayMs(liveCodexWorkerResult.coldWorkerReadyMs)} during Live initialization. + A real Pick → /bolder → Go run produced variant 1 in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} and all three in {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}. Accept returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}; the next Go reached the independent worker {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)} after Accept. In the matched full-task benchmark, warm app-server reached usable output in {displayMs(liveCodexWorkerResult.architectureComparison.warmAppServerFirstUsableMs)} versus {displayMs(liveCodexWorkerResult.architectureComparison.directFirstUsableMs)} direct; both passed 4/4. Production browser timing is one run.

@@ -441,6 +441,7 @@ const providerColumns = [

bun run bench:live boots a real framework fixture and Chromium, drives Pick → Go → Cycle, and records browser preparation, server pickup, scaffold, generation, write, and render boundaries. + Add --agent=codex --action=bolder to run the production app-server worker, or --acceptFirst to measure Accept → next Go. node scripts/benchmark-live-init.mjs --iterations 10 measures configured cold and warm helper initialization separately. The deterministic agent makes Impeccable overhead visible; model-backed runs remain opt-in because they send fixture context to an external provider.

@@ -469,8 +470,11 @@ const providerColumns = [