Update Live Lab production evidence

AI-assisted: Codex

site/data/live-codex-worker-result.ts

@@ -49,14 +49,37 @@ export const liveCodexWorkerResult = {
     warmRepeatCacheRate: 0.823,
   },
   productionBrowser: {
+    runs: 5,
+    firstVariantMs: 19283,
+    firstVariantP95Ms: 20412,
+    allVariantsMs: 52795,
+    allVariantsP95Ms: 55115,
+    deliveryGapMs: 33308,
+    serverPickupMs: 72.81,
+    sourcePreflightMs: 44,
+    validationMs: 5,
+    acceptRuns: 1,
+    acceptToResetMs: 196.5,
+    acceptToNextGoDispatchMs: 790.94,
+    acceptToNextFirstVariantMs: 23820,
+    blockedTailBaselineMs: 68141,
+  },
+  renderedQuality: {
+    runs: 3,
+    variants: 9,
+    passed: 1,
+    commandFidelity: 5.67,
+    brandFidelity: 5,
+    renderQuality: 5.44,
+    taskCompletion: 5.11,
+  },
+  atomicExperiment: {
     runs: 1,
-    firstVariantMs: 9868.3,
-    allVariantsMs: 31151.17,
-    acceptToResetMs: 298.71,
-    acceptToNextGoDispatchMs: 902.72,
-    acceptToNextFirstVariantMs: 16668,
-    sourcePreflightMs: 52,
-    validationMs: 10,
+    firstVariantMs: 40225,
+    allVariantsMs: 40227,
+    variants: 3,
+    passed: 1,
+    status: 'Opt-in experiment',
   },
   annotatedBrowser: {
     runs: 2,

site/data/live-harnesses.ts

@@ -49,15 +49,15 @@ export const liveExperiments = [
   {
     rank: 2,
     title: 'Accept any arrived variant — shipped',
-    evidence: 'A real Vite/React + Sol run accepted variant 1, interrupted variants 2–3, returned the page in 299 ms, and dispatched the next Go 903 ms after Accept.',
-    move: 'Durably fence generation on Accept/Discard, prioritize control events, publish through an epoch + source-hash transaction, and keep a separate poll supervisor active.',
+    evidence: 'A real Vite/React + Sol run returned the page in 197 ms and dispatched the next Go in 791 ms. Rotating the canceled app-server thread cut Accept → next first variant from 68.1 s to 23.8 s.',
+    move: 'Durably fence generation on Accept/Discard, rotate the canceled generation thread, prioritize control events, and keep a separate poll supervisor active.',
     expected: 'Removes the full-set wait and keeps the page interactive while canceled work unwinds.',
     confidence: 'Production browser',
   },
   {
     rank: 3,
     title: 'Reveal variants progressively — shipped',
-    evidence: 'The production run showed variant 1 at 9.87 s and the full set at 31.2 s. Progressive publication removed 21.3 s from the first decision.',
+    evidence: 'Five production Sol/medium runs showed variant 1 at 19.3 s median / 20.4 s p95 and the full set at 52.8 s median / 55.1 s p95.',
     move: 'Codex publishes a monotonically growing prefix as each variant validates. The browser reveals arrived variants and pending dots immediately; Accept and Discard fence unfinished work.',
     expected: 'Lets review begin as soon as one credible option exists.',
     confidence: 'Production browser',
@@ -89,26 +89,26 @@ export const liveExperiments = [
   {
     rank: 7,
     title: 'Acknowledge first, clean up off-path — shipped',
-    evidence: 'The production page returned to Pick in 299 ms and dispatched new work while carbonize cleanup remained on its separate foreground control lane.',
-    move: 'Keep the durable accept acknowledgement on the foreground path, then hand carbonize cleanup to a source-locked worker while the poll supervisor leases new work.',
+    evidence: 'The production page returned to Pick in 197 ms. Cleanup is durably enqueued before acknowledgement, and replayed Accept events are idempotent after a worker or network failure.',
+    move: 'Persist the Accept receipt, enqueue carbonize cleanup before replying, then let the poll supervisor lease new work while cleanup stays source-locked.',
     expected: 'Preserves immediate interaction without leaving temporary Live source behind.',
     confidence: 'Production browser + build control',
   },
   {
     rank: 8,
-    title: 'Trim the producer contract — selected',
-    evidence: 'Progressive compact passed 5/5 for Claude Sonnet 4.6 and GPT-5.5. A repeated warm Sol turn cached 38,656 of 46,973 input tokens (82.3%) and completed in 21.8 s versus 26.0 s cold.',
-    move: 'Keep the stable generation contract cacheable and send only the picked-element context plus the current action and identity lock.',
-    expected: 'Use provider caching instead of risky context deltas; always send current project and brand evidence.',
-    confidence: 'Candidate matrix + token telemetry',
+    title: 'Protect rendered quality before trimming context — blocking',
+    evidence: 'Only 1 of 9 rendered /bolder variants passed the multimodal gate. A compact-prompt trial made the progressive tail slower and passed 0/3, so it was reverted.',
+    move: 'Keep the full skill, sub-command, and project context. Improve diversity and fidelity against screenshots before pursuing prompt-size or lower-reasoning shortcuts.',
+    expected: 'Prevents a latency win that produces off-brand or structurally weak frontend work.',
+    confidence: 'Three production runs + rendered judge',
   },
   {
     rank: 9,
-    title: 'Parallelize ideas only where reliable — provider-specific',
-    evidence: 'Parallel compact passed 5/5 for GPT-5.5 and Gemini 3.1 Flash-Lite, but only 3/5 for Claude Sonnet 4.6.',
-    move: 'Use parallel compact for Gemini and as an OpenAI full-set option. Keep progressive compact for Anthropic until its validation reliability improves.',
-    expected: 'Lowers full-set latency without shared-file races or pretending one call shape fits every provider.',
-    confidence: 'Sixty-run paid matrix',
+    title: 'Keep atomic delivery as an opt-in experiment',
+    evidence: 'One production Sol/medium run completed all three variants in 40.2 s versus 56.1 s progressive median, but delayed first review by about 18 s and only 1/3 rendered variants passed.',
+    move: 'Expose atomic in the benchmark and configuration surface, but do not make it the Codex default until repeated rendered runs clear the same quality gate.',
+    expected: 'Preserves a promising full-set path without regressing the default first-review experience.',
+    confidence: 'One production run · needs replication',
   },
 ];
 

site/pages/live-lab/index.astro

@@ -406,9 +406,33 @@ const providerColumns = [
       <div class="live-performance-finding live-finding-card--quiet" role="note">
         <strong>Dedicated Codex worker</strong>
         <p>
-          A real Pick → <code>/bolder</code> → Go run produced variant 1 in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} and all three in {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}. Accept returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}; the next Go reached the independent worker {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)} after Accept. The annotated browser path preserved its screenshot, comment, and stroke, then produced variant 1 in {displayMs(liveCodexWorkerResult.annotatedBrowser.firstVariantMs)} median. In matched full tasks, warm app-server reached usable output in {displayMs(liveCodexWorkerResult.architectureComparison.warmAppServerFirstUsableMs)} versus {displayMs(liveCodexWorkerResult.architectureComparison.directFirstUsableMs)} direct; both passed 4/4. Plain browser figures are one run; annotated figures are two.
+          Five real Pick → <code>/bolder</code> → Go runs produced variant 1 in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} median and all three in {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}. Accept returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}; rotating the canceled producer cut Accept → next first variant from {displayMs(liveCodexWorkerResult.productionBrowser.blockedTailBaselineMs)} to {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextFirstVariantMs)}. In matched full tasks, warm app-server reached usable output in {displayMs(liveCodexWorkerResult.architectureComparison.warmAppServerFirstUsableMs)} versus {displayMs(liveCodexWorkerResult.architectureComparison.directFirstUsableMs)} direct; both passed 4/4.
         </p>
       </div>
+      <p class="table-scroll-hint" aria-hidden="true">Scroll horizontally to compare →</p>
+      <div class="harness-table-wrap" tabindex="0" aria-label="Scrollable production delivery evidence">
+        <table class="harness-table">
+          <caption class="sr-only">Production Codex delivery candidates and quality evidence</caption>
+          <thead><tr><th>Candidate</th><th>First review</th><th>Full set</th><th>Rendered gate</th><th>Decision</th></tr></thead>
+          <tbody>
+            <tr>
+              <td><strong>Progressive</strong><span>{liveCodexWorkerResult.productionBrowser.runs} Sol/medium runs</span></td>
+              <td><strong>{displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)}</strong><span>p95 {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantP95Ms)}</span></td>
+              <td><strong>{displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}</strong><span>p95 {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsP95Ms)}</span></td>
+              <td><strong>{liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants}</strong><span>production /bolder variants</span></td>
+              <td><span class="ks-tag" data-tone="ready">Default · quality work next</span></td>
+            </tr>
+            <tr>
+              <td><strong>Atomic</strong><span>{liveCodexWorkerResult.atomicExperiment.runs} Sol/medium run</span></td>
+              <td><strong>{displayMs(liveCodexWorkerResult.atomicExperiment.firstVariantMs)}</strong><span>waits for trio</span></td>
+              <td><strong>{displayMs(liveCodexWorkerResult.atomicExperiment.allVariantsMs)}</strong><span>single turn</span></td>
+              <td><strong>{liveCodexWorkerResult.atomicExperiment.passed}/{liveCodexWorkerResult.atomicExperiment.variants}</strong><span>production /bolder variants</span></td>
+              <td><span class="ks-tag is-neutral" data-tone="measure">Opt-in · replicate</span></td>
+            </tr>
+          </tbody>
+        </table>
+      </div>
+      <p class="live-performance-caption">Timing and rendered-quality samples are reported separately. The current blocker is output quality, not worker pickup: server pickup is {displayMs(liveCodexWorkerResult.productionBrowser.serverPickupMs)} median.</p>
     </section>
 
     <section class="live-performance-section live-experiments-section ks-section" aria-labelledby="experiments-title" data-lab-panel="harness">
@@ -441,7 +465,7 @@ const providerColumns = [
       <p>
         <code>bun run bench:live</code> boots a real framework fixture and Chromium, drives Pick → Go → Cycle,
         and records browser preparation, server pickup, scaffold, generation, write, and render boundaries.
-        Add <code>--agent=codex --action=bolder</code> to run the production app-server worker, or <code>--acceptFirst</code> to measure Accept → next Go.
+        Add <code>--agent=codex --action=bolder</code> to run the production app-server worker, <code>--accept-first</code> to measure Accept → next Go, <code>--render-quality</code> for screenshot evidence, or <code>--delivery=atomic</code> for the opt-in atomic candidate.
         <code>node scripts/benchmark-live-init.mjs --iterations 10</code> measures configured cold and warm helper initialization separately.
         The deterministic agent makes Impeccable overhead visible; model-backed runs remain opt-in because they send fixture context to an external provider.
       </p>
@@ -476,9 +500,9 @@ const providerColumns = [
                 <div><dt>Accept → next Go</dt><dd>{displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)}</dd></div>
                 <div><dt>Live returns</dt><dd>{displayMs(liveCodexWorkerResult.prewarmReturnMs)}</dd></div>
                 <div><dt>Worker ready</dt><dd>{displayMs(liveCodexWorkerResult.coldWorkerReadyMs)}</dd></div>
-                <div><dt>Quality gate</dt><dd>{liveCodexWorkerResult.quality.passed}/{liveCodexWorkerResult.quality.runs}</dd></div>
+                <div><dt>Rendered gate</dt><dd>{liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants}</dd></div>
               </dl>
-              <p>Default in Codex; other harnesses keep the foreground path.</p>
+              <p>Progressive remains the Codex default. Atomic is measured but opt-in; rendered quality is the next gate.</p>
             </section>
           </aside>
         </div>