diff --git a/site/data/live-codex-worker-result.ts b/site/data/live-codex-worker-result.ts index 17fb76241..819ab733c 100644 --- a/site/data/live-codex-worker-result.ts +++ b/site/data/live-codex-worker-result.ts @@ -49,14 +49,37 @@ export const liveCodexWorkerResult = { warmRepeatCacheRate: 0.823, }, productionBrowser: { + runs: 5, + firstVariantMs: 19283, + firstVariantP95Ms: 20412, + allVariantsMs: 52795, + allVariantsP95Ms: 55115, + deliveryGapMs: 33308, + serverPickupMs: 72.81, + sourcePreflightMs: 44, + validationMs: 5, + acceptRuns: 1, + acceptToResetMs: 196.5, + acceptToNextGoDispatchMs: 790.94, + acceptToNextFirstVariantMs: 23820, + blockedTailBaselineMs: 68141, + }, + renderedQuality: { + runs: 3, + variants: 9, + passed: 1, + commandFidelity: 5.67, + brandFidelity: 5, + renderQuality: 5.44, + taskCompletion: 5.11, + }, + atomicExperiment: { runs: 1, - firstVariantMs: 9868.3, - allVariantsMs: 31151.17, - acceptToResetMs: 298.71, - acceptToNextGoDispatchMs: 902.72, - acceptToNextFirstVariantMs: 16668, - sourcePreflightMs: 52, - validationMs: 10, + firstVariantMs: 40225, + allVariantsMs: 40227, + variants: 3, + passed: 1, + status: 'Opt-in experiment', }, annotatedBrowser: { runs: 2, diff --git a/site/data/live-harnesses.ts b/site/data/live-harnesses.ts index 422213e74..054918282 100644 --- a/site/data/live-harnesses.ts +++ b/site/data/live-harnesses.ts @@ -49,15 +49,15 @@ export const liveExperiments = [ { rank: 2, title: 'Accept any arrived variant — shipped', - evidence: 'A real Vite/React + Sol run accepted variant 1, interrupted variants 2–3, returned the page in 299 ms, and dispatched the next Go 903 ms after Accept.', - move: 'Durably fence generation on Accept/Discard, prioritize control events, publish through an epoch + source-hash transaction, and keep a separate poll supervisor active.', + evidence: 'A real Vite/React + Sol run returned the page in 197 ms and dispatched the next Go in 791 ms. Rotating the canceled app-server thread cut Accept → next first variant from 68.1 s to 23.8 s.', + move: 'Durably fence generation on Accept/Discard, rotate the canceled generation thread, prioritize control events, and keep a separate poll supervisor active.', expected: 'Removes the full-set wait and keeps the page interactive while canceled work unwinds.', confidence: 'Production browser', }, { rank: 3, title: 'Reveal variants progressively — shipped', - evidence: 'The production run showed variant 1 at 9.87 s and the full set at 31.2 s. Progressive publication removed 21.3 s from the first decision.', + evidence: 'Five production Sol/medium runs showed variant 1 at 19.3 s median / 20.4 s p95 and the full set at 52.8 s median / 55.1 s p95.', move: 'Codex publishes a monotonically growing prefix as each variant validates. The browser reveals arrived variants and pending dots immediately; Accept and Discard fence unfinished work.', expected: 'Lets review begin as soon as one credible option exists.', confidence: 'Production browser', @@ -89,26 +89,26 @@ export const liveExperiments = [ { rank: 7, title: 'Acknowledge first, clean up off-path — shipped', - evidence: 'The production page returned to Pick in 299 ms and dispatched new work while carbonize cleanup remained on its separate foreground control lane.', - move: 'Keep the durable accept acknowledgement on the foreground path, then hand carbonize cleanup to a source-locked worker while the poll supervisor leases new work.', + evidence: 'The production page returned to Pick in 197 ms. Cleanup is durably enqueued before acknowledgement, and replayed Accept events are idempotent after a worker or network failure.', + move: 'Persist the Accept receipt, enqueue carbonize cleanup before replying, then let the poll supervisor lease new work while cleanup stays source-locked.', expected: 'Preserves immediate interaction without leaving temporary Live source behind.', confidence: 'Production browser + build control', }, { rank: 8, - title: 'Trim the producer contract — selected', - evidence: 'Progressive compact passed 5/5 for Claude Sonnet 4.6 and GPT-5.5. A repeated warm Sol turn cached 38,656 of 46,973 input tokens (82.3%) and completed in 21.8 s versus 26.0 s cold.', - move: 'Keep the stable generation contract cacheable and send only the picked-element context plus the current action and identity lock.', - expected: 'Use provider caching instead of risky context deltas; always send current project and brand evidence.', - confidence: 'Candidate matrix + token telemetry', + title: 'Protect rendered quality before trimming context — blocking', + evidence: 'Only 1 of 9 rendered /bolder variants passed the multimodal gate. A compact-prompt trial made the progressive tail slower and passed 0/3, so it was reverted.', + move: 'Keep the full skill, sub-command, and project context. Improve diversity and fidelity against screenshots before pursuing prompt-size or lower-reasoning shortcuts.', + expected: 'Prevents a latency win that produces off-brand or structurally weak frontend work.', + confidence: 'Three production runs + rendered judge', }, { rank: 9, - title: 'Parallelize ideas only where reliable — provider-specific', - evidence: 'Parallel compact passed 5/5 for GPT-5.5 and Gemini 3.1 Flash-Lite, but only 3/5 for Claude Sonnet 4.6.', - move: 'Use parallel compact for Gemini and as an OpenAI full-set option. Keep progressive compact for Anthropic until its validation reliability improves.', - expected: 'Lowers full-set latency without shared-file races or pretending one call shape fits every provider.', - confidence: 'Sixty-run paid matrix', + title: 'Keep atomic delivery as an opt-in experiment', + evidence: 'One production Sol/medium run completed all three variants in 40.2 s versus 56.1 s progressive median, but delayed first review by about 18 s and only 1/3 rendered variants passed.', + move: 'Expose atomic in the benchmark and configuration surface, but do not make it the Codex default until repeated rendered runs clear the same quality gate.', + expected: 'Preserves a promising full-set path without regressing the default first-review experience.', + confidence: 'One production run · needs replication', }, ]; diff --git a/site/pages/live-lab/index.astro b/site/pages/live-lab/index.astro index 368aada9b..4a70facf9 100644 --- a/site/pages/live-lab/index.astro +++ b/site/pages/live-lab/index.astro @@ -406,9 +406,33 @@ const providerColumns = [
- A real Pick → /bolder → Go run produced variant 1 in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} and all three in {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}. Accept returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}; the next Go reached the independent worker {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)} after Accept. The annotated browser path preserved its screenshot, comment, and stroke, then produced variant 1 in {displayMs(liveCodexWorkerResult.annotatedBrowser.firstVariantMs)} median. In matched full tasks, warm app-server reached usable output in {displayMs(liveCodexWorkerResult.architectureComparison.warmAppServerFirstUsableMs)} versus {displayMs(liveCodexWorkerResult.architectureComparison.directFirstUsableMs)} direct; both passed 4/4. Plain browser figures are one run; annotated figures are two.
+ Five real Pick → /bolder → Go runs produced variant 1 in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} median and all three in {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}. Accept returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}; rotating the canceled producer cut Accept → next first variant from {displayMs(liveCodexWorkerResult.productionBrowser.blockedTailBaselineMs)} to {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextFirstVariantMs)}. In matched full tasks, warm app-server reached usable output in {displayMs(liveCodexWorkerResult.architectureComparison.warmAppServerFirstUsableMs)} versus {displayMs(liveCodexWorkerResult.architectureComparison.directFirstUsableMs)} direct; both passed 4/4.
| Candidate | First review | Full set | Rendered gate | Decision |
|---|---|---|---|---|
| Progressive{liveCodexWorkerResult.productionBrowser.runs} Sol/medium runs | +{displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)}p95 {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantP95Ms)} | +{displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}p95 {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsP95Ms)} | +{liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants}production /bolder variants | +Default · quality work next | +
| Atomic{liveCodexWorkerResult.atomicExperiment.runs} Sol/medium run | +{displayMs(liveCodexWorkerResult.atomicExperiment.firstVariantMs)}waits for trio | +{displayMs(liveCodexWorkerResult.atomicExperiment.allVariantsMs)}single turn | +{liveCodexWorkerResult.atomicExperiment.passed}/{liveCodexWorkerResult.atomicExperiment.variants}production /bolder variants | +Opt-in · replicate | +
Timing and rendered-quality samples are reported separately. The current blocker is output quality, not worker pickup: server pickup is {displayMs(liveCodexWorkerResult.productionBrowser.serverPickupMs)} median.
bun run bench:live boots a real framework fixture and Chromium, drives Pick → Go → Cycle,
and records browser preparation, server pickup, scaffold, generation, write, and render boundaries.
- Add --agent=codex --action=bolder to run the production app-server worker, or --acceptFirst to measure Accept → next Go.
+ Add --agent=codex --action=bolder to run the production app-server worker, --accept-first to measure Accept → next Go, --render-quality for screenshot evidence, or --delivery=atomic for the opt-in atomic candidate.
node scripts/benchmark-live-init.mjs --iterations 10 measures configured cold and warm helper initialization separately.
The deterministic agent makes Impeccable overhead visible; model-backed runs remain opt-in because they send fixture context to an external provider.
Default in Codex; other harnesses keep the foreground path.
+Progressive remains the Codex default. Atomic is measured but opt-in; rendered quality is the next gate.