diff --git a/scripts/benchmark-live-codex-architecture.mjs b/scripts/benchmark-live-codex-architecture.mjs index 4b0bc2b76..695ceaa88 100644 --- a/scripts/benchmark-live-codex-architecture.mjs +++ b/scripts/benchmark-live-codex-architecture.mjs @@ -34,7 +34,7 @@ const timeoutMs = positiveInteger(args.timeout, 300_000); const outputPath = args.output ? path.resolve(ROOT, String(args.output)) : null; const profileIds = csv(args.profiles || 'direct-sol,cold-app-server,warm-app-server'); const taskIds = csv(args.tasks || 'editorial-bolder,operations-polish,operations-annotated'); -const judgeEnabled = args.judge !== false; +const judgeEnabled = args.judge !== false && args.judge !== 'false'; const loadedEnv = loadBenchmarkEnv({ repoRoot: ROOT, explicitPath: args.envFile && path.resolve(args.envFile) }); const skillPath = path.join(ROOT, '.agents', 'skills', 'impeccable', 'SKILL.md'); const referenceDir = path.join(ROOT, 'skill', 'reference'); diff --git a/site/data/live-codex-worker-result.ts b/site/data/live-codex-worker-result.ts index 389ac877d..17fb76241 100644 --- a/site/data/live-codex-worker-result.ts +++ b/site/data/live-codex-worker-result.ts @@ -44,6 +44,9 @@ export const liveCodexWorkerResult = { annotatedQualityRuns: 3, coldInputTokens: 41066, coldOutputTokens: 2148, + warmRepeatInputTokens: 46973, + warmRepeatCachedInputTokens: 38656, + warmRepeatCacheRate: 0.823, }, productionBrowser: { runs: 1, @@ -55,6 +58,14 @@ export const liveCodexWorkerResult = { sourcePreflightMs: 52, validationMs: 10, }, + annotatedBrowser: { + runs: 2, + firstVariantMs: 14316.21, + allVariantsMs: 45424.24, + comments: 1, + strokes: 1, + screenshotAttached: true, + }, result: '4/4 full-task quality gate', passed: true, architecture: 'A dedicated persistent Live app-server connection and worker thread with native Impeccable skill, sub-command, project context, and annotation-image inputs. Accept interrupts unfinished variants; cleanup stays on the foreground control lane while the next generation can start independently.', diff --git a/site/data/live-harnesses.ts b/site/data/live-harnesses.ts index a05f4ebef..422213e74 100644 --- a/site/data/live-harnesses.ts +++ b/site/data/live-harnesses.ts @@ -97,10 +97,10 @@ export const liveExperiments = [ { rank: 8, title: 'Trim the producer contract — selected', - evidence: 'Progressive compact passed 5/5 for Claude Sonnet 4.6 and GPT-5.5. It was OpenAI’s fastest and cheapest progressive path; Claude’s other candidates all missed reliability.', + evidence: 'Progressive compact passed 5/5 for Claude Sonnet 4.6 and GPT-5.5. A repeated warm Sol turn cached 38,656 of 46,973 input tokens (82.3%) and completed in 21.8 s versus 26.0 s cold.', move: 'Keep the stable generation contract cacheable and send only the picked-element context plus the current action and identity lock.', - expected: 'Reduces prefill and output work without weakening the design brief.', - confidence: 'Five runs per candidate', + expected: 'Use provider caching instead of risky context deltas; always send current project and brand evidence.', + confidence: 'Candidate matrix + token telemetry', }, { rank: 9, diff --git a/site/pages/live-lab/index.astro b/site/pages/live-lab/index.astro index 1fd7bfd8d..368aada9b 100644 --- a/site/pages/live-lab/index.astro +++ b/site/pages/live-lab/index.astro @@ -406,7 +406,7 @@ const providerColumns = [
Dedicated Codex worker

- A real Pick → /bolder → Go run produced variant 1 in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} and all three in {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}. Accept returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}; the next Go reached the independent worker {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)} after Accept. In the matched full-task benchmark, warm app-server reached usable output in {displayMs(liveCodexWorkerResult.architectureComparison.warmAppServerFirstUsableMs)} versus {displayMs(liveCodexWorkerResult.architectureComparison.directFirstUsableMs)} direct; both passed 4/4. Production browser timing is one run. + A real Pick → /bolder → Go run produced variant 1 in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} and all three in {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}. Accept returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}; the next Go reached the independent worker {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)} after Accept. The annotated browser path preserved its screenshot, comment, and stroke, then produced variant 1 in {displayMs(liveCodexWorkerResult.annotatedBrowser.firstVariantMs)} median. In matched full tasks, warm app-server reached usable output in {displayMs(liveCodexWorkerResult.architectureComparison.warmAppServerFirstUsableMs)} versus {displayMs(liveCodexWorkerResult.architectureComparison.directFirstUsableMs)} direct; both passed 4/4. Plain browser figures are one run; annotated figures are two.