Update Live Lab performance evidence

AI-assisted implementation under maintainer direction.
2026-09-20 18:16:30 +03:00 · 2026-07-13 13:09:29 -07:00
parent 0ae45b9f62
commit 50fc88ec85
3 changed files with 52 additions and 41 deletions
@@ -1,5 +1,5 @@
 export const liveCodexWorkerResult = {
-  generatedAt: '2026-07-13T19:06:57.179Z',
+  generatedAt: '2026-07-13T20:01:08.415Z',
  codexVersion: '0.144.0-alpha.4',
  transport: 'stdio JSONL',
  model: 'GPT-5.6-Sol',
@@ -51,22 +51,16 @@ export const liveCodexWorkerResult = {
  productionBrowser: {
    runs: 1,
    selectedTarget: 'article.offer-card',
-    firstVariantMs: 34177.39,
-    firstVariantP95Ms: 34177.39,
-    secondVariantMs: 54156.22,
-    allVariantsMs: 122089.41,
-    allVariantsP95Ms: 122089.41,
-    firstToSecondMs: 19978.83,
-    secondToAllMs: 67933.19,
-    serverPickupMs: 54.02,
-    sourcePreflightMs: 43,
-    validationMs: 8.67,
-    acceptRuns: 1,
-    acceptedVariant: 2,
-    acceptToResetMs: 231.1,
-    acceptToNextGoDispatchMs: 833.68,
-    acceptToNextFirstVariantMs: 52612.73,
-    blockedTailBaselineMs: 68141,
+    firstVariantMs: 29682.58,
+    firstVariantP95Ms: 29682.58,
+    secondVariantMs: 54597.81,
+    allVariantsMs: 102182.9,
+    allVariantsP95Ms: 102182.9,
+    firstToSecondMs: 24915.23,
+    secondToAllMs: 47585.09,
+    serverPickupMs: 76.82,
+    sourcePreflightMs: 45,
+    validationMs: 9.33,
  },
  fullSourceControl: {
    runs: 1,
@@ -77,11 +71,26 @@ export const liveCodexWorkerResult = {
    firstToSecondMs: 43874.54,
  },
  sourceDeltaExperiment: {
+    runs: 5,
+    fullSourceFirstVariantMs: 34177.39,
+    deltaFirstVariantMs: 28074.92,
+    firstVariantImprovement: 0.1786,
    baselineFirstToSecondMs: 43874.54,
-    deltaFirstToSecondMs: 19978.83,
-    improvement: 0.5446,
-    variant2Passed: true,
-    status: 'Enabled for Codex source mode',
+    deltaFirstToSecondMs: 24624.41,
+    secondVariantImprovement: 0.4388,
+    finalDeltaAllVariantsMs: 102182.9,
+    status: 'Enabled for all Codex source-mode phases',
+  },
+  earlyAcceptVerified: {
+    runs: 1,
+    acceptedVariant: 2,
+    sourceCommitted: true,
+    followupCleanupPassed: true,
+    acceptToResetMs: 238.18,
+    acceptToNextGoDispatchMs: 831.36,
+    acceptToNextFirstVariantMs: 16938.05,
+    controlNextFirstVariantMs: 42969.34,
+    nextFirstImprovement: 0.6058,
  },
  standbyExperiment: {
    controlFirstVariantMs: 33283.06,
@@ -112,12 +121,13 @@ export const liveCodexWorkerResult = {
  renderedQuality: {
    runs: 1,
    variants: 3,
-    passed: 1,
-    commandFidelity: 6.33,
-    brandFidelity: 7.33,
-    renderQuality: 6.33,
-    taskCompletion: 5.67,
-    variant2Passed: true,
+    passed: 2,
+    commandFidelity: 6.67,
+    brandFidelity: 7.67,
+    renderQuality: 6.67,
+    taskCompletion: 6.33,
+    variant1Passed: true,
+    variant3Passed: true,
  },
  calibratedHeadingQuality: {
    runs: 3,
@@ -146,5 +156,5 @@ export const liveCodexWorkerResult = {
  },
  result: '4/4 full-task quality gate',
  passed: true,
-  architecture: 'A dedicated persistent Live app-server connection and worker thread with native Impeccable skill, sub-command, project context, and annotation-image inputs. Variants 1 and 2 publish as independent reviewable checkpoints; source-mode variant 2 uses a fenced delta while variant 3 and parameters finish last. Accept interrupts unfinished variants; cleanup stays on the foreground control lane while the next generation starts independently.',
+  architecture: 'A dedicated persistent Live app-server connection and worker thread with native Impeccable skill, sub-command, project context, and annotation-image inputs. Every source-mode variant publishes through an independently validated fenced delta; the final delta adds deferred parameter manifests without rewriting prior output. Accept interrupts unfinished variants; cleanup stays on the foreground control lane while the next generation starts independently.',
 } as const;
@@ -49,17 +49,17 @@ export const liveExperiments = [
  {
    rank: 2,
    title: 'Accept any arrived variant — shipped',
-    evidence: 'A real Vite/React + Sol run returned the page in 197 ms and dispatched the next Go in 791 ms. Rotating the canceled app-server thread cut Accept → next first variant from 68.1 s to 23.8 s.',
+    evidence: 'A verified Vite/React + Sol run committed variant 2 to source, returned the page in 238 ms, and dispatched the next Go in 831 ms. The next first variant arrived in 16.9 s versus the 43.0 s control, a 60.6% reduction.',
    move: 'Durably fence generation on Accept/Discard, rotate the canceled generation thread, prioritize control events, and keep a separate poll supervisor active.',
    expected: 'Removes the full-set wait and keeps the page interactive while canceled work unwinds.',
    confidence: 'Production browser',
  },
  {
    rank: 3,
-    title: 'Send variant 2 as a fenced delta — shipped',
-    evidence: 'On the proven article.offer-card target, variant 2 arrived 20.0 s after variant 1 instead of 43.9 s with full-source output, a 54.5% reduction. The rendered judge passed variant 2 at 7/8/7/7.',
-    move: 'Keep the full skill, plan, project context, and source evidence, but return only variant 2 markup and scoped CSS. The supervisor validates its variant identity and merges it transactionally without exposing prior output to rewrites.',
-    expected: 'Makes the second credible option reviewable much sooner without weakening design context.',
+    title: 'Send every source variant as a fenced delta — shipped',
+    evidence: 'Across five Sol/medium samples, first review was 28.1 s median versus the 34.2 s full-source control. Variant 2 followed 24.6 s later versus 43.9 s, and the guarded all-delta run completed in 102.2 s with 2/3 rendered variants passing and 7.7/10 average brand fidelity.',
+    move: 'Keep the full skill, plan, project context, and source evidence, but return only each new variant’s markup, scoped CSS, and the final deferred parameter manifest. The supervisor validates and merges each delta without exposing prior output to rewrites.',
+    expected: 'Shrinks every model response while preserving full design context and immutable review checkpoints.',
    confidence: 'Production browser + rendered gate',
  },
  {
@@ -406,7 +406,7 @@ const providerColumns = [
      <div class="live-performance-finding live-finding-card--quiet" role="note">
        <strong>Dedicated Codex worker</strong>
        <p>
-          With the target proven as <code>{liveCodexWorkerResult.productionBrowser.selectedTarget}</code>, variant 1 arrived in {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)} and the fenced variant-2 delta followed {displayMs(liveCodexWorkerResult.productionBrowser.firstToSecondMs)} later, {(liveCodexWorkerResult.sourceDeltaExperiment.improvement * 100).toFixed(1)}% faster than full-source output. Variant 2 passed the independent rendered gate. Accepting it while variant 3 was pending returned the page in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)} and dispatched the next Go in {displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)}.
+          With the target proven as <code>{liveCodexWorkerResult.productionBrowser.selectedTarget}</code>, fenced deltas delivered first review in {displayMs(liveCodexWorkerResult.sourceDeltaExperiment.deltaFirstVariantMs)} median and cut the variant-2 gap by {(liveCodexWorkerResult.sourceDeltaExperiment.secondVariantImprovement * 100).toFixed(1)}%. Accepting variant 2 while variant 3 was pending committed it to source, returned the page in {displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToResetMs)}, and produced the next first variant in {displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToNextFirstVariantMs)}.
        </p>
      </div>
      <p class="table-scroll-hint" aria-hidden="true">Scroll horizontally to compare →</p>
@@ -416,12 +416,12 @@ const providerColumns = [
          <thead><tr><th>Candidate</th><th>Variant 1</th><th>Variant 2</th><th>Full set</th><th>Rendered gate</th><th>Decision</th></tr></thead>
          <tbody>
            <tr>
-              <td><strong>Root · delta v2</strong><span>{liveCodexWorkerResult.productionBrowser.runs} Sol/medium run</span></td>
+              <td><strong>Root · all deltas</strong><span>{liveCodexWorkerResult.productionBrowser.runs} guarded Sol/medium run</span></td>
              <td><strong>{displayMs(liveCodexWorkerResult.productionBrowser.firstVariantMs)}</strong><span>p95 {displayMs(liveCodexWorkerResult.productionBrowser.firstVariantP95Ms)}</span></td>
              <td><strong>{displayMs(liveCodexWorkerResult.productionBrowser.secondVariantMs)}</strong><span>independently acceptable</span></td>
              <td><strong>{displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}</strong><span>p95 {displayMs(liveCodexWorkerResult.productionBrowser.allVariantsP95Ms)}</span></td>
-              <td><strong>v2 passed</strong><span>{liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants} full-set variants</span></td>
-              <td><span class="ks-tag" data-tone="ready">Default · 54.5% faster v2</span></td>
+              <td><strong>{liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants} passed</strong><span>{liveCodexWorkerResult.renderedQuality.brandFidelity.toFixed(1)}/10 brand fidelity</span></td>
+              <td><span class="ks-tag" data-tone="ready">Default · immutable deltas</span></td>
            </tr>
            <tr>
              <td><strong>Root · full source</strong><span>{liveCodexWorkerResult.fullSourceControl.runs} Sol/medium control</span></td>
@@ -450,7 +450,7 @@ const providerColumns = [
          </tbody>
        </table>
      </div>
-      <p class="live-performance-caption">Root-task samples prove target identity in the posted Generate payload. Speculative standby warmup was rejected: it slowed the next result to {displayMs(liveCodexWorkerResult.standbyExperiment.prewarmedNextFirstVariantMs)} versus {displayMs(liveCodexWorkerResult.standbyExperiment.controlNextFirstVariantMs)}. Next gates: replicate root-level rendered quality and reduce the cold post-Accept generation wait; server pickup is already {displayMs(liveCodexWorkerResult.productionBrowser.serverPickupMs)}.</p>
+      <p class="live-performance-caption">Root-task samples prove target identity in the posted Generate payload. Speculative standby warmup was rejected: it slowed the next result to {displayMs(liveCodexWorkerResult.standbyExperiment.prewarmedNextFirstVariantMs)} versus {displayMs(liveCodexWorkerResult.standbyExperiment.controlNextFirstVariantMs)}. The next gate is consistent 3/3 rendered quality; server pickup is already {displayMs(liveCodexWorkerResult.productionBrowser.serverPickupMs)} and post-Accept next review is down to {displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToNextFirstVariantMs)}.</p>
    </section>

    <section class="live-performance-section live-experiments-section ks-section" aria-labelledby="experiments-title" data-lab-panel="harness">
@@ -516,13 +516,14 @@ const providerColumns = [
                <div><dt>Second review</dt><dd>{displayMs(liveCodexWorkerResult.productionBrowser.secondVariantMs)}</dd></div>
                <div><dt>v1 → v2</dt><dd>{displayMs(liveCodexWorkerResult.productionBrowser.firstToSecondMs)}</dd></div>
                <div><dt>All variants</dt><dd>{displayMs(liveCodexWorkerResult.productionBrowser.allVariantsMs)}</dd></div>
-                <div><dt>Accept → page ready</dt><dd>{displayMs(liveCodexWorkerResult.productionBrowser.acceptToResetMs)}</dd></div>
-                <div><dt>Accept → next Go</dt><dd>{displayMs(liveCodexWorkerResult.productionBrowser.acceptToNextGoDispatchMs)}</dd></div>
+                <div><dt>Accept → page ready</dt><dd>{displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToResetMs)}</dd></div>
+                <div><dt>Accept → next Go</dt><dd>{displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToNextGoDispatchMs)}</dd></div>
+                <div><dt>Accept → next review</dt><dd>{displayMs(liveCodexWorkerResult.earlyAcceptVerified.acceptToNextFirstVariantMs)}</dd></div>
                <div><dt>Live returns</dt><dd>{displayMs(liveCodexWorkerResult.prewarmReturnMs)}</dd></div>
                <div><dt>Worker ready</dt><dd>{displayMs(liveCodexWorkerResult.coldWorkerReadyMs)}</dd></div>
                <div><dt>Rendered gate</dt><dd>{liveCodexWorkerResult.renderedQuality.passed}/{liveCodexWorkerResult.renderedQuality.variants}</dd></div>
              </dl>
-              <p>Variant 2 is independently reviewable, acceptable, and 54.5% faster with fenced source deltas. Cold post-Accept generation and replicated full-set quality remain the next gates.</p>
+              <p>Every source variant now arrives through an immutable fenced delta. Early Accept is source-verified and the next review is 60.6% faster; consistent 3/3 rendered quality remains the next gate.</p>
            </section>
          </aside>
        </div>