Improve Codex Live architecture benchmark

Compare production-equivalent direct and app-server paths, include annotated UI work, expose first-usable latency, and keep semantic quality gates honest.\n\nAI-assisted: OpenAI Codex.

scripts/benchmark-live-codex-architecture.mjs

@@ -1,7 +1,6 @@
 #!/usr/bin/env node
 
 import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
-import os from 'node:os';
 import path from 'node:path';
 import { performance } from 'node:perf_hooks';
 import { fileURLToPath } from 'node:url';
@@ -34,7 +33,7 @@ const iterations = positiveInteger(args.iterations, 2);
 const timeoutMs = positiveInteger(args.timeout, 300_000);
 const outputPath = args.output ? path.resolve(ROOT, String(args.output)) : null;
 const profileIds = csv(args.profiles || 'direct-sol,cold-app-server,warm-app-server');
-const taskIds = csv(args.tasks || 'editorial-bolder,operations-polish');
+const taskIds = csv(args.tasks || 'editorial-bolder,operations-polish,operations-annotated');
 const judgeEnabled = args.judge !== false;
 const loadedEnv = loadBenchmarkEnv({ repoRoot: ROOT, explicitPath: args.envFile && path.resolve(args.envFile) });
 const skillPath = path.join(ROOT, '.agents', 'skills', 'impeccable', 'SKILL.md');
@@ -79,9 +78,12 @@ if (args.dryRun) {
   process.exit(0);
 }
 
-const scratch = await mkdtemp(path.join(os.tmpdir(), 'impeccable-codex-architecture-'));
+const scratchRoot = path.join(ROOT, '.impeccable', 'live');
+await mkdir(scratchRoot, { recursive: true });
+const scratch = await mkdtemp(path.join(scratchRoot, 'architecture-'));
 const schemaPath = path.join(scratch, 'output-schema.json');
 await writeFile(schemaPath, JSON.stringify(CODEX_QUALITY_OUTPUT_SCHEMA));
+await prepareAnnotationScreenshots();
 const runs = [];
 try {
   for (const profile of profileIds) {
@@ -126,20 +128,25 @@ async function runDirect({ profile, task, iteration }) {
   const prompt = [
     '$impeccable',
     'Use the attached Impeccable skill. This automated benchmark already resolved Setup context below; do not rerun setup or edit files.',
+    '<production_worker_contract>',
+    buildCodexWorkerInstructions(liveSpec),
+    '</production_worker_contract>',
     buildCodexQualityPrompt(task, { actionReference, fullContext: true }),
   ].join('\n\n');
   try {
+    const directArgs = [
+      'exec', '--ephemeral', '--ignore-user-config', '--dangerously-bypass-hook-trust',
+      '-C', ROOT, '-s', 'read-only', '-m', model,
+      '-c', 'model_reasoning_effort="medium"',
+      '--output-schema', schemaPath,
+      '--output-last-message', outputFile,
+    ];
+    if (task.screenshotPath) directArgs.push('-i', task.screenshotPath);
+    directArgs.push('--json', prompt);
     const result = await runCodexExecBenchmark({
       cwd: ROOT,
       timeoutMs,
-      args: [
-        'exec', '--ephemeral', '--ignore-user-config', '--dangerously-bypass-hook-trust',
-        '-C', ROOT, '-s', 'read-only', '-m', model,
-        '-c', 'model_reasoning_effort="medium"',
-        '--output-schema', schemaPath,
-        '--output-last-message', outputFile,
-        '--json', prompt,
-      ],
+      args: directArgs,
     });
     const output = JSON.parse(await readFile(outputFile, 'utf-8'));
     return finishRun({
@@ -151,6 +158,7 @@ async function runDirect({ profile, task, iteration }) {
       generationMs: result.firstAgentMessageMs == null || result.turnStartedMs == null
         ? result.durationMs
         : result.firstAgentMessageMs - result.turnStartedMs,
+      firstUsableMs: result.firstAgentMessageMs ?? result.durationMs,
       totalMs: result.durationMs,
       usage: result.usage,
       transport: {
@@ -181,6 +189,7 @@ async function runColdAppServer({ profile, task, iteration }) {
       output: turn.output,
       startupMs,
       generationMs: turn.durationMs,
+      firstUsableMs: startupMs + turn.durationMs,
       totalMs: performance.now() - startedAt,
       usage: normalizeAppServerUsage(turn.turn),
     });
@@ -214,6 +223,7 @@ async function runWarmProfile(profile) {
             output: turn.output,
             startupMs: iteration === 1 && task === tasks[0] ? coldStartupMs : 0,
             generationMs: turn.durationMs,
+            firstUsableMs: turn.durationMs + (iteration === 1 && task === tasks[0] ? coldStartupMs : 0),
             totalMs: performance.now() - turnStartedAt + (iteration === 1 && task === tasks[0] ? coldStartupMs : 0),
             usage: normalizeAppServerUsage(turn.turn),
             transport: { persistentThread: true, coldStartupMs: round(coldStartupMs) },
@@ -244,6 +254,9 @@ function threadParams(profile) {
 async function runAppServerTurn(client, thread, task) {
   const actionReference = await readFile(path.join(referenceDir, `${task.action}.md`), 'utf-8');
   const prompt = buildCodexQualityPrompt(task, { actionReference, fullContext: true });
+  let output = null;
+  let firstAgentMessageMs = null;
+  const startedAt = performance.now();
   const result = await client.startTurn({
     threadId: thread.id,
     input: buildCodexWorkerTurnInputs({ prompt, skillPath, screenshotPath: task.screenshotPath, cwd: ROOT }),
@@ -254,11 +267,63 @@ async function runAppServerTurn(client, thread, task) {
     approvalPolicy: 'never',
     sandboxPolicy: { type: 'readOnly' },
     outputSchema: CODEX_QUALITY_OUTPUT_SCHEMA,
+    onAgentMessage: (message) => {
+      if (output) return;
+      output = JSON.parse(message);
+      firstAgentMessageMs = performance.now() - startedAt;
+    },
   });
-  return { output: JSON.parse(result.message), durationMs: result.durationMs, turn: result };
+  return {
+    output: output || JSON.parse(result.message),
+    durationMs: firstAgentMessageMs ?? result.firstAgentMessageMs ?? result.durationMs,
+    completionMs: result.durationMs,
+    turn: result,
+  };
 }
 
-async function finishRun({ profile, task, iteration, output, startupMs, generationMs, totalMs, usage, transport = null }) {
+async function prepareAnnotationScreenshots() {
+  const annotated = tasks.filter((task) => task.annotation);
+  if (annotated.length === 0) return;
+  const { chromium } = await import('playwright');
+  const browser = await chromium.launch({ headless: true });
+  try {
+    for (const task of annotated) {
+      const screenshotPath = path.join(scratch, `${task.id}.png`);
+      const page = await browser.newPage({ viewport: { width: 1080, height: 720 }, deviceScaleFactor: 1 });
+      await page.setContent(`<!doctype html><html><head><style>
+        ${task.files['src/styles.css']}
+        body { padding: 1px; }
+        .workspace { position: relative; }
+        .metric--warning { outline: 3px solid #d94343; outline-offset: 5px; }
+        .benchmark-annotation {
+          position: absolute; z-index: 10; top: 12.5rem; left: 48%; width: 18rem;
+          padding: 0.7rem 0.85rem; border: 2px solid #d94343; border-radius: 0.25rem;
+          background: #fff8ef; color: #6b1919; font: 700 0.8rem/1.35 Inter, sans-serif;
+          transform: rotate(-1.5deg);
+        }
+        .benchmark-annotation::after {
+          position: absolute; top: 100%; left: 2rem; width: 7rem; height: 3rem;
+          border-left: 3px solid #d94343; border-bottom: 3px solid #d94343;
+          content: ""; transform: skewX(-28deg);
+        }
+      </style></head><body>
+        <main class="workspace">
+          <header class="workspace__header"><div><p class="eyebrow">Monday, 14 July</p><h1>Fulfillment overview</h1><p class="summary">Monitor the work that can put today’s dispatch at risk.</p></div><button class="button button--primary">Create dispatch</button></header>
+          <section class="metrics"><article class="metric metric--positive"><p class="metric__label">Ready</p><strong class="metric__value">184</strong><p class="metric__detail">31 due before noon</p></article><article class="metric metric--warning"><p class="metric__label">At risk</p><strong class="metric__value">12</strong><p class="metric__detail">4 need assignment</p></article><article class="metric metric--critical"><p class="metric__label">Blocked</p><strong class="metric__value">3</strong><p class="metric__detail">Oldest waiting 42 min</p></article></section>
+          <section class="queue"><div class="queue__heading"><div><p class="eyebrow">Priority queue</p><h2>Needs attention</h2></div><button class="button button--quiet">View all 19</button></div><table><thead><tr><th>Dispatch</th><th>Destination</th><th>Owner</th><th>Status</th><th>Due</th></tr></thead><tbody><tr><td>DP-2048</td><td>Portland</td><td>Unassigned</td><td><span class="status status--critical">Blocked</span></td><td>09:30</td></tr><tr><td>DP-2051</td><td>Oakland</td><td>M. Chen</td><td><span class="status status--warning">At risk</span></td><td>10:15</td></tr></tbody></table></section>
+          <div class="benchmark-annotation">${escapeHtml(task.annotation.comment)}</div>
+        </main>
+      </body></html>`, { waitUntil: 'load' });
+      await page.screenshot({ path: screenshotPath, fullPage: true });
+      await page.close();
+      task.screenshotPath = screenshotPath;
+    }
+  } finally {
+    await browser.close();
+  }
+}
+
+async function finishRun({ profile, task, iteration, output, startupMs, generationMs, firstUsableMs, totalMs, usage, transport = null }) {
   const deterministic = scoreCodexQualityOutput(task, output);
   const judge = judgeEnabled ? await judgeOutput(task, output) : null;
   return {
@@ -269,6 +334,7 @@ async function finishRun({ profile, task, iteration, output, startupMs, generati
     effort: 'medium',
     startupMs: round(startupMs),
     generationMs: round(generationMs),
+    firstUsableMs: round(firstUsableMs),
     totalMs: round(totalMs),
     usage,
     transport,
@@ -302,7 +368,7 @@ async function judgeOutput(task, output) {
 }
 
 function normalizeAppServerUsage(turn) {
-  const usage = turn?.completed?.params?.turn?.usage || turn?.turn?.usage || null;
+  const usage = turn?.tokenUsage?.last || turn?.completed?.params?.turn?.usage || turn?.turn?.usage || null;
   if (!usage) return null;
   return {
     input_tokens: usage.inputTokens ?? usage.input_tokens ?? null,
@@ -347,3 +413,9 @@ function positiveInteger(value, fallback) {
 function round(value) {
   return Number.isFinite(value) ? Math.round(value * 100) / 100 : null;
 }
+
+function escapeHtml(value) {
+  return String(value).replace(/[&<>"']/g, (character) => ({
+    '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;',
+  })[character]);
+}

scripts/lib/codex-exec-benchmark.mjs

@@ -89,6 +89,8 @@ export function summarizeArchitectureRuns(runs) {
   return {
     runs: runs.length,
     passed: completed.filter((run) => run.passed).length,
+    medianFirstUsableMs: percentile(completed.map((run) => run.firstUsableMs).filter(Number.isFinite), 0.5),
+    p95FirstUsableMs: percentile(completed.map((run) => run.firstUsableMs).filter(Number.isFinite), 0.95),
     medianTotalMs: percentile(completed.map((run) => run.totalMs), 0.5),
     p95TotalMs: percentile(completed.map((run) => run.totalMs), 0.95),
     medianStartupMs: percentile(completed.map((run) => run.startupMs).filter(Number.isFinite), 0.5),

scripts/lib/live-codex-quality-benchmark.mjs

@@ -137,7 +137,7 @@ th { background: var(--surface-subtle); color: var(--ink-muted); font-size: 0.75
 
 export function createCodexQualityTasks({ repoRoot }) {
   const fixtureDir = path.join(repoRoot, 'tests', 'framework-fixtures', 'vite8-react-brand-fidelity', 'files', 'src');
-  return [
+  const tasks = [
     {
       id: 'editorial-bolder',
       action: 'bolder',
@@ -168,6 +168,22 @@ export function createCodexQualityTasks({ repoRoot }) {
       judgeFocus: 'Is this a materially more polished, efficient operations surface, with excellent scan hierarchy and interaction detail, without changing its product model or turning it into a decorative dashboard?',
     },
   ];
+  const operations = tasks[1];
+  tasks.push({
+    ...operations,
+    id: 'operations-annotated',
+    brief: 'Polish this fulfillment dashboard while following the attached annotation. Make the At risk state easier to identify in a fast scan without making the whole dashboard louder. Preserve the existing information architecture, components, terminology, and token palette.',
+    annotation: {
+      comment: 'Make this risk state easier to scan without making the whole dashboard louder.',
+      target: 'The At risk metric card and its relationship to the priority queue.',
+      strokes: 1,
+    },
+    allowedForbiddenMatches: [
+      /box-shadow\s*:\s*inset\s+(?:0|0?\.\d+(?:rem|px))\s+0(?:\s+0)?\s+var\(--warning\)\s*;?/gi,
+    ],
+    judgeFocus: 'Does the implementation respond precisely to the visual annotation by strengthening risk-state scanning, while remaining a calm operations UI and preserving the established component and status system?',
+  });
+  return tasks;
 }
 
 export function buildCodexQualityPrompt(task, { actionReference = '', fullContext = false } = {}) {
@@ -182,6 +198,7 @@ export function buildCodexQualityPrompt(task, { actionReference = '', fullContex
     '<product_context>', task.product, '</product_context>',
     '<design_context>', task.design, '</design_context>',
     '<action_reference>', actionReference, '</action_reference>',
+    '<annotation_context>', JSON.stringify(task.annotation || {}, null, 2), '</annotation_context>',
     '<source_files>', JSON.stringify(task.files, null, 2), '</source_files>',
   ].filter((line) => line !== '').join('\n');
 }
@@ -193,17 +210,25 @@ export function scoreCodexQualityOutput(task, output) {
   const source = byPath['src/App.jsx'] || '';
   const css = byPath['src/styles.css'] || '';
   const normalizedSource = source.replace(/[\s"']/g, '');
+  const visibleSourceText = source.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
   const checks = {
     exactFiles: files.length === 2 && Boolean(source) && Boolean(css),
     implementationChanged: source !== task.files['src/App.jsx'] || css !== task.files['src/styles.css'],
-    copyPreserved: task.requiredCopy.every((value) => combined.includes(value)),
+    copyPreserved: task.requiredCopy.every((value) => (
+      combined.includes(value) || visibleSourceText.includes(value.replace(/\s+/g, ' ').trim())
+    )),
     contractsPreserved: task.requiredSource.every((value) => normalizedSource.includes(value.replace(/[\s"']/g, ''))),
     tokensPreserved: task.requiredTokens.every((value) => css.includes(value)),
     noForbiddenDrift: task.forbidden.every((pattern) => {
+      const outputWithoutAllowedMatches = stripAllowedForbiddenMatches(combined, task.allowedForbiddenMatches);
+      const inputWithoutAllowedMatches = stripAllowedForbiddenMatches(
+        Object.values(task.files).join('\n'),
+        task.allowedForbiddenMatches,
+      );
       pattern.lastIndex = 0;
-      const outputMatches = combined.match(pattern) || [];
+      const outputMatches = outputWithoutAllowedMatches.match(pattern) || [];
       pattern.lastIndex = 0;
-      const inputMatches = Object.values(task.files).join('\n').match(pattern) || [];
+      const inputMatches = inputWithoutAllowedMatches.match(pattern) || [];
       return outputMatches.length <= inputMatches.length;
     }),
   };
@@ -213,6 +238,13 @@ export function scoreCodexQualityOutput(task, output) {
   };
 }
 
+function stripAllowedForbiddenMatches(value, patterns = []) {
+  return patterns.reduce((result, pattern) => {
+    pattern.lastIndex = 0;
+    return result.replace(pattern, '');
+  }, value);
+}
+
 export function buildJudgePrompt(task, output) {
   return [
     'You are an exacting independent frontend design reviewer. Score the revised code, not the prose around it.',
@@ -221,6 +253,7 @@ export function buildJudgePrompt(task, output) {
     '',
     `TASK: /${task.action} — ${task.brief}`,
     `REVIEW FOCUS: ${task.judgeFocus}`,
+    '<annotation_context>', JSON.stringify(task.annotation || {}, null, 2), '</annotation_context>',
     '<product_context>', task.product, '</product_context>',
     '<design_context>', task.design, '</design_context>',
     '<before>', JSON.stringify(task.files, null, 2), '</before>',

tests/live-codex-exec-benchmark.test.mjs

@@ -32,12 +32,13 @@ describe('direct Codex architecture benchmark', () => {
 
   it('summarizes startup, generation, total, quality, and token medians', () => {
     const summary = summarizeArchitectureRuns([
-      { passed: true, startupMs: 10, generationMs: 100, totalMs: 110, usage: { input_tokens: 1000, cached_input_tokens: 500, output_tokens: 100 } },
-      { passed: false, startupMs: 20, generationMs: 200, totalMs: 220, usage: { input_tokens: 2000, cached_input_tokens: 1000, output_tokens: 200 } },
+      { passed: true, startupMs: 10, generationMs: 100, firstUsableMs: 110, totalMs: 110, usage: { input_tokens: 1000, cached_input_tokens: 500, output_tokens: 100 } },
+      { passed: false, startupMs: 20, generationMs: 200, firstUsableMs: 220, totalMs: 220, usage: { input_tokens: 2000, cached_input_tokens: 1000, output_tokens: 200 } },
     ]);
     assert.equal(summary.runs, 2);
     assert.equal(summary.passed, 1);
     assert.equal(summary.medianTotalMs, 165);
+    assert.equal(summary.medianFirstUsableMs, 165);
     assert.equal(summary.medianInputTokens, 1500);
   });
 });

tests/live-codex-quality-benchmark.test.mjs

@@ -18,6 +18,7 @@ describe('Codex Live quality benchmark', () => {
     assert.deepEqual(tasks.map((task) => `${task.action}:${task.id}`), [
       'bolder:editorial-bolder',
       'polish:operations-polish',
+      'polish:operations-annotated',
     ]);
     const prompt = buildCodexQualityPrompt(tasks[0], { actionReference: 'BOLDER', fullContext: true });
     assert.match(prompt, /Impeccable skill is attached/);
@@ -25,6 +26,7 @@ describe('Codex Live quality benchmark', () => {
     assert.match(prompt, /<design_context>/);
     assert.match(prompt, /BOLDER/);
     assert.match(prompt, /src\/App\.jsx/);
+    assert.equal(tasks[2].annotation.strokes, 1);
   });
 
   it('rejects no-op, contract-breaking, and design-system-drifting output', () => {
@@ -40,6 +42,14 @@ describe('Codex Live quality benchmark', () => {
     };
     assert.equal(scoreCodexQualityOutput(task, cssOnly).passed, true, 'CSS-only design work is a material implementation change');
 
+    const splitVisibleCopy = {
+      files: [
+        { path: 'src/App.jsx', content: task.files['src/App.jsx'].replace('Field Notes', '<span>Field</span> <span>Notes</span>') },
+        { path: 'src/styles.css', content: `${task.files['src/styles.css']}\n.offer-card { min-height: 30rem; }` },
+      ],
+    };
+    assert.equal(scoreCodexQualityOutput(task, splitVisibleCopy).checks.copyPreserved, true);
+
     const drift = {
       files: [
         { path: 'src/App.jsx', content: task.files['src/App.jsx'].replace('Field Notes', 'Neon Notes') },
@@ -51,6 +61,20 @@ describe('Codex Live quality benchmark', () => {
     assert.equal(score.checks.noForbiddenDrift, false);
   });
 
+  it('allows an annotation-scoped semantic risk rail but still rejects decorative shadows', () => {
+    const task = tasks[2];
+    const withRiskRail = {
+      files: [
+        { path: 'src/App.jsx', content: task.files['src/App.jsx'] },
+        { path: 'src/styles.css', content: `${task.files['src/styles.css']}\n.metric--warning { box-shadow: inset 0.1875rem 0 0 var(--warning); }` },
+      ],
+    };
+    assert.equal(scoreCodexQualityOutput(task, withRiskRail).checks.noForbiddenDrift, true);
+
+    withRiskRail.files[1].content += '\n.queue { box-shadow: 0 1rem 3rem rgb(0 0 0 / 0.2); }';
+    assert.equal(scoreCodexQualityOutput(task, withRiskRail).checks.noForbiddenDrift, false);
+  });
+
   it('parses strict judge results and summarizes latency and quality', () => {
     const judge = parseJudgeResult('{"commandFidelity":8,"brandAndSystemFidelity":9,"frontendQuality":7,"taskCompletion":8,"criticalFailure":false,"summary":"Good."}');
     assert.equal(judge.passed, true);