diff --git a/site/data/live-codex-worker-result.ts b/site/data/live-codex-worker-result.ts new file mode 100644 index 000000000..61848e972 --- /dev/null +++ b/site/data/live-codex-worker-result.ts @@ -0,0 +1,17 @@ +export const liveCodexWorkerResult = { + generatedAt: '2026-07-13T01:05:15.178Z', + codexVersion: '0.144.0-alpha.4', + transport: 'stdio JSONL', + model: 'GPT-5.3-Codex-Spark', + effort: 'low', + timings: { + coldHandshakeMs: 36.83, + coldThreadStartMs: 400.19, + turnStartResponseMs: 0.45, + wakeToTurnStartedMs: 1.47, + trivialTurnCompletedMs: 2060.18, + }, + result: 'READY', + passed: true, + architecture: 'A dedicated persistent Live app-server connection and worker thread. Do not attach a second app-server to the desktop task.', +} as const; diff --git a/site/data/live-harnesses.ts b/site/data/live-harnesses.ts index 8b2a028d7..e7a18fe43 100644 --- a/site/data/live-harnesses.ts +++ b/site/data/live-harnesses.ts @@ -1,11 +1,12 @@ export const harnessPaths = [ { - name: 'Supervisor + generation worker', - harness: 'Codex desktop · canonical progressive path', - pickup: 'Concurrent control-event intake', - tradeoff: 'The foreground poll supervisor handles Accept/Discard and the next Go while a fenced worker generates variants.', - status: 'E2E proven', + name: 'Dedicated app-server worker', + harness: 'Codex · isolated persistent thread', + pickup: '1.47 ms turn wake', + tradeoff: 'A separate Live-owned thread can generate while the control supervisor keeps polling. Cold setup is 437 ms; a persistent connection amortizes it.', + status: 'Protocol probe passed', statusTone: 'ready', + source: 'https://learn.chatgpt.com/docs/app-server', }, { name: 'Foreground terminal supervisor', @@ -17,11 +18,11 @@ export const harnessPaths = [ source: 'https://learn.chatgpt.com/docs/app-server#api-overview', }, { - name: 'Warm variant producer', - harness: 'Codex custom agent', - pickup: 'Spawn/resume cost unknown', - tradeoff: 'Can pin a faster model and lower reasoning, but fresh context and handoff can erase the gain unless the agent stays warm.', - status: 'Prototype', + name: 'Native generation subagent', + harness: 'Codex desktop · current shipped path', + pickup: 'Main turn must dispatch it', + tradeoff: 'Low-effort compact handoff is safe and fenced, but it cannot start after the desktop task has gone idle.', + status: 'Current fallback', statusTone: 'prototype', source: 'https://learn.chatgpt.com/docs/agent-configuration/subagents', }, @@ -119,32 +120,32 @@ export const liveExperiments = [ }, { rank: 11, - title: 'Parallelize ideas, centralize writes — optional fast path', - evidence: 'Strict-gated provider runs made parallel compact fastest for GPT-5.5 and Gemini 3.1 Flash-Lite. Every worker remained write-free; deterministic assembly performed the only publication.', - move: 'Use parallel compact only where extra calls are acceptable. Keep progressive compact as the portable default and preserve variant 1 byte-for-byte.', - expected: 'Lowers time to first option without shared-file races or brand drift.', - confidence: 'Paid provider smoke matrix', + title: 'Parallelize ideas, centralize writes — provider-specific', + evidence: 'Parallel compact passed 5/5 for GPT-5.5 and Gemini 3.1 Flash-Lite, but only 3/5 for Claude Sonnet 4.6. It cut Gemini first review to 1.64 s and GPT full-set completion to 16.21 s.', + move: 'Make parallel compact the Gemini default and an OpenAI full-set fast path. Do not enable it for Anthropic until its validation reliability improves.', + expected: 'Lowers time to usable options without shared-file races or pretending one call shape fits every provider.', + confidence: 'Sixty-run paid matrix', }, { rank: 12, title: 'Trim the producer contract — selected', - evidence: 'Progressive compact passed every measured brand, component, token, copy, source, and cleanup gate while cutting first-review latency 42–62% for Claude and GPT.', + evidence: 'Progressive compact passed 5/5 for Claude Sonnet 4.6 and GPT-5.5. It was OpenAI’s fastest and cheapest progressive path; Claude’s other candidates all missed reliability.', move: 'Keep the stable generation contract cacheable and send only the picked-element context plus the current action and identity lock.', expected: 'Reduces prefill and output work without weakening the design brief.', - confidence: 'Cross-provider quality gate', + confidence: 'Five runs per candidate', }, { rank: 13, - title: 'Wake on the journal, not terminal stdout — still architectural', - evidence: 'Codex app-server exposes fs.watch and process output notifications, while this desktop probe still required an explicit terminal read.', - move: 'Benchmark a plugin/app-server bridge that watches the durable Live journal and starts or steers a dedicated turn directly.', - expected: 'Frees the main turn if watcher-to-turn startup beats foreground polling.', - confidence: 'Architectural bet', + title: 'Wake a dedicated Codex worker — protocol proven', + evidence: 'A cold app-server handshake took 36.83 ms, thread creation 400.19 ms, and turn/start reached turn/started in 1.47 ms. A low-effort Spark probe completed in 2.06 s.', + move: 'Keep one Live-owned app-server connection and isolated worker thread warm. Feed it durable events and structured output; never resume the desktop task from a second server.', + expected: 'Removes terminal-output pickup and main-turn availability from generation startup while preserving transactional publication fencing.', + confidence: 'Official protocol + live probe', }, ]; export const currentHarnessProbe = { - testedAt: '2026-07-11', + testedAt: '2026-07-13', surface: 'Codex desktop unified exec', delayedOutputMs: 250, processExitMs: null, diff --git a/site/data/live-provider-result.ts b/site/data/live-provider-result.ts index a1207c752..94a4aef81 100644 --- a/site/data/live-provider-result.ts +++ b/site/data/live-provider-result.ts @@ -1,21 +1,31 @@ +const candidate = ( + firstMedianMs: number | null, + firstP95Ms: number | null, + allMedianMs: number | null, + allP95Ms: number | null, + passedRuns: number, + costUsd: number, +) => ({ + firstMedianMs, + firstP95Ms, + allMedianMs, + allP95Ms, + passedRuns, + totalRuns: 5, + costUsd, + reliable: passedRuns === 5, +}); + export const liveProviderResult = { - generatedAt: '2026-07-12T03:22:46.816Z', + generatedAt: '2026-07-13T01:11:49.054Z', fixture: 'vite8-react-brand-fidelity', - runsPerCandidate: 1, - statisticallyStable: false, - paidSmokeMatrix: { - ran: ['atomic-full', 'progressive-compact', 'parallel-compact'], - notRun: [ - { - strategy: 'progressive-full', - reason: 'External execution credits were exhausted before the direct full-context split comparison could run. No result is inferred.', - }, - ], - }, + runsPerCandidate: 5, + statisticallyStable: true, + strategies: ['atomic-full', 'progressive-full', 'progressive-compact', 'parallel-compact'], cleanupControl: { providerIndependent: true, - acceptToCleanPickingMs: 171.41, - productionBuildMs: 378.04, + acceptToCleanPickingMs: 172.19, + productionBuildMs: 383.51, markerFree: true, browserClean: true, consoleErrors: 0, @@ -25,45 +35,41 @@ export const liveProviderResult = { { provider: 'Anthropic', model: 'Claude Sonnet 4.6', - effort: 'Provider default in this sample; future harness runs explicitly use low.', - atomic: { firstMs: 32208.62, allMs: 32208.62, costUsd: 0.103779, quality: 0.9, passed: true }, - progressiveCompact: { firstMs: 12247.31, allMs: 31709.49, costUsd: 0.057795, quality: 0.9, passed: true }, - parallelCompact: { firstMs: 12361.28, allMs: 31378.19, costUsd: 0.064641, quality: 0.9, passed: true }, - progressiveFirstImprovement: 0.6198, - parallelFirstImprovement: 0.6162, + effort: 'low', + atomic: candidate(57168.08, 64787.41, 57168.08, 64787.41, 4, 0.882555), + progressiveFull: candidate(18623.04, 21744.52, 40591.04, 57284.19, 3, 1.35405), + progressiveCompact: candidate(16387.92, 17641.08, 41505.32, 46789.53, 5, 0.386004), + parallelCompact: candidate(10567.09, 10755.49, 19805.73, 20156.03, 3, 0.424566), + recommendation: 'Progressive compact is the only 5/5 candidate; use atomic only as a compatibility fallback.', }, { provider: 'OpenAI', model: 'GPT-5.5', effort: 'low', - atomic: { firstMs: 22750.85, allMs: 22750.85, costUsd: 0.16728, quality: 0.95, passed: true }, - progressiveCompact: { firstMs: 13124.09, allMs: 24735.95, costUsd: 0.098515, quality: 0.9, passed: true }, - parallelCompact: { firstMs: 11846.73, allMs: 15871.87, costUsd: 0.145085, quality: 0.9, passed: true }, - progressiveFirstImprovement: 0.4231, - parallelFirstImprovement: 0.4793, + atomic: candidate(27266.19, 31780.17, 27266.19, 31780.17, 5, 0.602296), + progressiveFull: candidate(12681.34, 14283.56, 36102.57, 41957.05, 5, 0.713001), + progressiveCompact: candidate(11777.98, 13334.92, 28867.61, 35691.27, 5, 0.48768), + parallelCompact: candidate(11846.01, 13551.87, 16211.64, 19043.63, 5, 0.614395), + recommendation: 'Progressive compact wins first review and cost; parallel compact wins full-set completion.', }, { provider: 'Google', model: 'Gemini 3.1 Flash-Lite', effort: 'minimal (provider default)', - atomic: { - firstMs: null, - allMs: null, - costUsd: null, - quality: null, - passed: false, - reason: 'Both attempts failed strict output validation.', - }, - progressiveCompact: { firstMs: 2654.97, allMs: 6764.95, costUsd: 0.002813, quality: 0.95, passed: true }, - parallelCompact: { firstMs: 1607.89, allMs: 1988.32, costUsd: 0.003956, quality: 0.95, passed: true }, - progressiveFirstImprovement: null, - parallelFirstImprovement: null, + atomic: candidate(6579.06, 6579.06, 6579.06, 6579.06, 1, 0.034123), + progressiveFull: candidate(null, null, null, null, 0, 0.023667), + progressiveCompact: candidate(3063.57, 3069.15, 5357.76, 5368.13, 3, 0.015316), + parallelCompact: candidate(1635.05, 1973.83, 2193.7, 3576.19, 5, 0.020824), + recommendation: 'Parallel compact is the only 5/5 candidate and the clear production path.', }, ], - cost: { - apiCalls: 22, - measuredLowerBoundUsd: 0.643864, - note: 'Five rejected validation responses predate failed-response usage capture and are excluded. Current runs capture their usage.', + totals: { + runs: 60, + passingRuns: 44, + apiCalls: 160, + retries: 47, + rejectedResponses: 64, + estimatedCostUsd: 5.558477, }, gate: { dimensions: ['brand', 'component', 'token', 'copy', 'source', 'accept-cleanup'], @@ -71,8 +77,9 @@ export const liveProviderResult = { minimumDimension: 0.75, }, recommendation: { - default: 'progressive-compact', - optionalFastPath: 'parallel-compact', - rationale: 'Progressive compact cut first-review latency 42–62% for Claude and GPT while preserving every quality gate. Parallel compact was strongest for GPT and Gemini but spends more requests.', + codexDefault: 'progressive-compact', + googleDefault: 'parallel-compact', + portableFallback: 'atomic-full', + rationale: 'Choose per provider and harness. Compact progressive is the reliable first-review winner for OpenAI and Anthropic; parallel compact is required for Gemini reliability. Harnesses without measured progressive publication retain atomic delivery.', }, } as const; diff --git a/site/pages/live-lab/index.astro b/site/pages/live-lab/index.astro index 983979835..0095a17c7 100644 --- a/site/pages/live-lab/index.astro +++ b/site/pages/live-lab/index.astro @@ -7,6 +7,7 @@ import { liveInitResult } from '../../data/live-init-result'; import { liveControlResult } from '../../data/live-control-result'; import { liveFrameworkResults } from '../../data/live-framework-results'; import { liveProviderResult } from '../../data/live-provider-result'; +import { liveCodexWorkerResult } from '../../data/live-codex-worker-result'; import { liveAnnotatedResult } from '../../data/live-annotated-result'; import '../../styles/sub-pages.css'; import '../../styles/live-performance.css'; @@ -61,6 +62,12 @@ const progressiveAllDelta = ( - progressiveDeliveryResult.atomic.medianAllMs ); const initGain = 1 - (liveInitResult.cold.medianMs / liveInitResult.baselineColdMedianMs); +const providerColumns = [ + { key: 'atomic', label: 'Atomic full' }, + { key: 'progressiveFull', label: 'Progressive full' }, + { key: 'progressiveCompact', label: 'Progressive compact' }, + { key: 'parallelCompact', label: 'Parallel compact' }, +] as const; ---
-

Paid provider smoke matrix

-

Decompose the work, preserve the first answer

+

Paid provider matrix · five runs per candidate

+

The right split depends on the provider

-

One strict-gated run per candidate. Brand, component, token, copy, source, and cleanup fidelity must all pass.

+

Sixty strict-gated runs. Brand, component, token, copy, source, and cleanup fidelity must all pass.

- + {providerColumns.map(column => )} {liveProviderResult.providers.map(result => ( - - - - + + {providerColumns.map(column => { + const candidate = result[column.key]; + return ( + + ); + })} ))} @@ -242,12 +247,11 @@ const initGain = 1 - (liveInitResult.cold.medianMs / liveInitResult.baselineCold
Decision

- Use progressive compact as the safe default: it cut first-review latency 42–62% for Claude and GPT without weakening any gate. Parallel compact is an optional GPT/Gemini fast path when extra calls and cost are acceptable. Variant 1 and its CSS are carried byte-for-byte into deterministic local assembly; later calls may only add directions. + Codex/OpenAI defaults to progressive compact: 5/5 passed, it had the fastest first review, and it cost less than the other OpenAI splits. Anthropic also keeps progressive compact because it was the only 5/5 candidate. Gemini uses parallel compact; every other Gemini strategy failed the reliability gate. Unmeasured harnesses retain atomic delivery.

- Provider matrix cost: ${liveProviderResult.cost.measuredLowerBoundUsd.toFixed(3)} lower bound. {liveProviderResult.cost.note}{' '} - The full-context progressive split was not run after external execution credits were exhausted; no result is inferred. Accept → clean Pick/build control passed in {displayMs(liveProviderResult.cleanupControl.acceptToCleanPickingMs)} with zero console errors or Live markers. + Provider matrix cost: ${liveProviderResult.totals.estimatedCostUsd.toFixed(2)} across {liveProviderResult.totals.apiCalls} calls, including {liveProviderResult.totals.retries} retries. {liveProviderResult.totals.passingRuns}/{liveProviderResult.totals.runs} runs passed the strict gate. Accept → clean Pick/build control passed in {displayMs(liveProviderResult.cleanupControl.acceptToCleanPickingMs)} with zero console errors or Live markers.

@@ -372,6 +376,12 @@ const initGain = 1 - (liveInitResult.cold.medianMs / liveInitResult.baselineCold Harness probe

{currentHarnessProbe.surface}: {currentHarnessProbe.result}

+
+ Dedicated Codex worker probe +

+ {liveCodexWorkerResult.model} at {liveCodexWorkerResult.effort} effort started a dedicated turn in {displayMs(liveCodexWorkerResult.timings.wakeToTurnStartedMs)} and completed a trivial structured response in {displayMs(liveCodexWorkerResult.timings.trivialTurnCompletedMs)}. Cold app-server plus thread setup cost {displayMs(liveCodexWorkerResult.timings.coldHandshakeMs + liveCodexWorkerResult.timings.coldThreadStartMs)} and can be amortized. The safe path owns a separate Live thread; it never resumes the desktop task from a competing app-server. +

+
Paid Live generation strategy results by provider
ProviderAtomic fullProgressive compactParallel compact
Provider{column.label}
{result.provider}{result.model} · {result.effort} - {result.atomic.firstMs === null ? 'Rejected' : displayMs(result.atomic.firstMs)} - {result.atomic.passed ? 'quality gate passed' : ('reason' in result.atomic ? result.atomic.reason : 'quality gate failed')} - - {displayMs(result.progressiveCompact.firstMs)} first - {displayMs(result.progressiveCompact.allMs)} all · gate passed - - {displayMs(result.parallelCompact.firstMs)} first - {displayMs(result.parallelCompact.allMs)} all · gate passed - {result.provider}{result.model} · {result.effort}{result.recommendation} + {candidate.firstMedianMs === null ? 'No valid run' : `${displayMs(candidate.firstMedianMs)} first`} + {candidate.passedRuns}/5 pass{candidate.firstP95Ms === null ? '' : ` · p95 ${displayMs(candidate.firstP95Ms)}`} + {candidate.allMedianMs === null ? 'full set unavailable' : `${displayMs(candidate.allMedianMs)} all`} +