Benchmark the production Codex Live worker

Let the browser E2E harness launch an independent production worker, exercise real sub-command selection, and carry realistic product/design context.\n\nAI-assisted: OpenAI Codex.

scripts/benchmark-live.mjs

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 
+import { spawn } from 'node:child_process';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { dirname, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -12,6 +13,7 @@ import {
   clickGo,
   drawAnnotationPinAndStroke,
   pickElement,
+  selectAction,
   waitForCycling,
   waitForHandshake,
 } from '../tests/live-e2e/ui.mjs';
@@ -27,7 +29,7 @@ const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
 const args = parseArgs(process.argv.slice(2));
 const fixtureName = String(args.fixture || 'vite8-react-plain');
 const iterations = positiveInt(args.iterations, 5);
-const agentMode = args.agent === 'llm' ? 'llm' : 'fake';
+const agentMode = args.agent === 'codex' ? 'codex' : args.agent === 'llm' ? 'llm' : 'fake';
 const scenario = args.scenario === 'annotated' ? 'annotated' : 'plain';
 const delivery = args.delivery === 'progressive' ? 'progressive' : 'atomic';
 const simulatedTailMs = positiveInt(args.simulatedTailMs, 0);
@@ -51,6 +53,7 @@ try {
     fixture,
     browser,
     agent: agentInfo.agent,
+    startWorker: agentInfo.startWorker,
     wrapTarget: wrapTargetFromPickedElement,
     trace: recorder.trace,
     progressive: delivery === 'progressive',
@@ -81,6 +84,7 @@ try {
   const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title';
   for (let iteration = 1; iteration <= iterations; iteration += 1) {
     await pickElement(session.page, pickSelector, { resetPickMode: iteration > 1 });
+    if (args.action) await selectAction(session.page, String(args.action));
     if (scenario === 'annotated') {
       await drawAnnotationPinAndStroke(session.page, { comment: 'Benchmark annotation' });
     }
@@ -94,7 +98,7 @@ try {
     await clickGo(session.page);
     recorder.mark('ui.generating_visible', { iteration, scenario });
     await firstVariant;
-    await waitForCycling(session.page, 3, { timeout: agentMode === 'llm' ? 150_000 : 30_000 });
+    await waitForCycling(session.page, 3, { timeout: agentMode === 'fake' ? 30_000 : 240_000 });
     recorder.mark('browser.all_variants', { iteration, scenario });
     const browserTiming = await readBrowserTimingProbe(session.page);
 
@@ -116,7 +120,7 @@ try {
     fixture: fixtureName,
     agent: agentMode,
     provider: agentInfo.provider,
-    model: agentInfo.model,
+    model: session.worker?.state?.model || agentInfo.model,
     scenario,
     runs,
     events: recorder.events,
@@ -150,6 +154,15 @@ try {
 
 async function resolveAgent(mode, options) {
   if (mode === 'fake') return { agent: createFakeAgent(), provider: 'deterministic', model: null, promptMode: null };
+  if (mode === 'codex') {
+    return {
+      agent: null,
+      provider: 'openai-codex-app-server',
+      model: options.model || null,
+      promptMode: 'production-live-contract',
+      startWorker: (context) => startCodexProductionWorker(context, options),
+    };
+  }
   const config = resolveLlmAgentConfig({
     provider: options.provider,
     model: options.model,
@@ -165,6 +178,62 @@ async function resolveAgent(mode, options) {
   return { agent, provider: config.provider, model: config.model, promptMode: 'synthetic-element-contract' };
 }
 
+async function startCodexProductionWorker({ tmp, scriptsDir, log }, options) {
+  const script = join(scriptsDir, 'live-codex-worker.mjs');
+  const statePath = join(tmp, '.impeccable', 'live', 'codex-worker.json');
+  const child = spawn(process.execPath, [script, '--foreground'], {
+    cwd: tmp,
+    env: {
+      ...process.env,
+      IMPECCABLE_LIVE_CODEX_WORKER: '1',
+      IMPECCABLE_LIVE_CODEX_PROFILE: String(options.profile || 'quality'),
+      IMPECCABLE_LIVE_CODEX_EFFORT: String(options.effort || 'medium'),
+      ...(options.model ? { IMPECCABLE_LIVE_CODEX_MODEL: String(options.model) } : {}),
+    },
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+  const output = [];
+  const capture = (chunk) => {
+    const text = chunk.toString();
+    output.push(text);
+    if (output.length > 200) output.shift();
+    log(`[codex-worker] ${text.trimEnd()}`);
+  };
+  child.stdout.on('data', capture);
+  child.stderr.on('data', capture);
+  const done = new Promise((resolve) => child.once('exit', (code, signal) => resolve({ code, signal })));
+  const state = await waitForWorkerState(statePath, child, output, positiveInt(options.workerTimeoutMs, 20_000));
+  return {
+    child,
+    state,
+    done,
+    async stop() {
+      if (child.exitCode != null || child.signalCode != null) return;
+      child.kill('SIGTERM');
+      await Promise.race([done, new Promise((resolve) => setTimeout(resolve, 5_000))]);
+      if (child.exitCode == null && child.signalCode == null) child.kill('SIGKILL');
+    },
+  };
+}
+
+async function waitForWorkerState(statePath, child, output, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (child.exitCode != null || child.signalCode != null) {
+      throw new Error(`Codex production worker exited before ready.\n${output.join('')}`);
+    }
+    try {
+      const state = JSON.parse(await readFile(statePath, 'utf-8'));
+      if (state.status === 'error') throw new Error(`Codex production worker failed: ${state.error}\n${output.join('')}`);
+      if (['ready', 'working'].includes(state.status)) return state;
+    } catch (error) {
+      if (error.code !== 'ENOENT' && error.name !== 'SyntaxError') throw error;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+  throw new Error(`Codex production worker was not ready after ${timeoutMs}ms.\n${output.join('')}`);
+}
+
 function createSplitProgressiveAgent(agent) {
   const firstBySession = new Map();
   return {

tests/framework-fixtures/vite8-react-brand-fidelity/files/DESIGN.md

@@ -0,0 +1,7 @@
+# Design system
+
+- Warm paper, dark ink, moss, and brass only.
+- Georgia display type with a restrained sans body.
+- Editorial, practical, quiet, and tactile.
+- Reuse the existing CSS custom properties. Do not add colors, fonts, gradients, shadows, glow, glass, or decorative effects.
+- Square, rule-led compositions are preferred to card stacks and rounded containers.

tests/framework-fixtures/vite8-react-brand-fidelity/files/PRODUCT.md

@@ -0,0 +1,7 @@
+# Northstar Field Journal
+
+An independent quarterly field guide for design-conscious weekend walkers. Readers value practical detail, editorial restraint, and objects worth keeping. The offer card should make issue eight feel collectible without becoming luxurious or loud.
+
+## Platform
+
+web

tests/framework-fixtures/vite8-react-brand-fidelity/fixture.json

@@ -5,7 +5,7 @@
     "insertBefore": "</body>",
     "commentSyntax": "html"
   },
-  "sourceFiles": ["index.html", "src/App.jsx", "src/main.jsx", "src/styles.css", "vite.config.js"],
+  "sourceFiles": ["PRODUCT.md", "DESIGN.md", "index.html", "src/App.jsx", "src/main.jsx", "src/styles.css", "vite.config.js"],
   "generatedFiles": [],
   "wrapCases": [
     {

tests/live-e2e/session.mjs

@@ -219,6 +219,10 @@ export async function stopDevServer(child) {
  * @param {import('playwright').Browser} opts.browser   shared browser instance
  * @param {object} opts.agent             VariantAgent (defaults to fake)
  * @param {object|function=} opts.wrapTarget live-wrap target or event mapper
+ * @param {(context: object) => Promise<object|void>} [opts.startWorker]
+ *        Optional production worker factory. Return {stop, done}; when used,
+ *        omit `agent` so the deterministic in-process loop is not started.
+ * @param {(context: object) => Promise<void>|void} [opts.prepareTmp]
  * @param {(msg: string) => void} [opts.log]
  */
 export async function bootFixtureSession({
@@ -227,6 +231,8 @@ export async function bootFixtureSession({
   browser,
   agent,
   wrapTarget,
+  startWorker,
+  prepareTmp,
   log = () => {},
   trace = () => {},
   progressive = false,
@@ -242,12 +248,15 @@ export async function bootFixtureSession({
   let dev;
   let agentAbort;
   let agentDone;
+  let externalWorker;
   let ctx;
 
   const teardown = async () => {
     try { if (ctx) await ctx.close(); } catch {}
     try { if (agentAbort) agentAbort.abort(); } catch {}
     try { if (agentDone) await agentDone.catch(() => {}); } catch {}
+    try { if (externalWorker?.stop) await externalWorker.stop(); } catch {}
+    try { if (externalWorker?.done) await externalWorker.done.catch(() => {}); } catch {}
     try { if (dev?.child) await stopDevServer(dev.child); } catch {}
     try { if (live) stopLiveServer(tmp); } catch {}
     try { rmSync(tmp, { recursive: true, force: true }); } catch {}
@@ -261,6 +270,7 @@ export async function bootFixtureSession({
 
   try {
     const startedAt = Date.now();
+    if (prepareTmp) await prepareTmp({ tmp, fixture, scriptsDir: SCRIPTS_DIR, trace, log });
     trace('setup.install.start', { fixture: name });
     log(`installing deps`);
     runInstall(tmp, runtime.install);
@@ -274,6 +284,12 @@ export async function bootFixtureSession({
     trace('setup.live_server.end', { fixture: name, port: live.port });
     log(`live-server ready in ${formatDuration(Date.now() - liveStartedAt)}`);
 
+    if (startWorker) {
+      trace('setup.worker.start', { fixture: name });
+      externalWorker = await startWorker({ tmp, fixture, scriptsDir: SCRIPTS_DIR, live, trace, log });
+      trace('setup.worker.end', { fixture: name });
+    }
+
     const injectStartedAt = Date.now();
     trace('setup.inject.start', { fixture: name });
     log(`live-inject --port ${live.port}`);
@@ -291,28 +307,30 @@ export async function bootFixtureSession({
     log(`dev server ready on ${devPort} in ${formatDuration(Date.now() - devStartedAt)}`);
 
     // Agent loop runs concurrently — abort on teardown.
-    agentAbort = new AbortController();
-    const loopOptions = {
-      tmp,
-      scriptsDir: SCRIPTS_DIR,
-      port: live.port,
-      token: live.token,
-      agent,
-      wrapTarget,
-      signal: agentAbort.signal,
-      trace,
-      progressive,
-      progressiveDelayMs,
-      progressiveInitialCount,
-      atomicDelayMs,
-      steerSourceFile: runtime.steer?.sourceFile,
-      steerTarget: runtime.steer?.target,
-    };
-    const loops = [runAgentLoop({ ...loopOptions, log: (m) => log('[worker] ' + m) })];
-    if (progressive) {
-      loops.push(runAgentLoop({ ...loopOptions, log: (m) => log('[supervisor] ' + m) }));
+    if (agent) {
+      agentAbort = new AbortController();
+      const loopOptions = {
+        tmp,
+        scriptsDir: SCRIPTS_DIR,
+        port: live.port,
+        token: live.token,
+        agent,
+        wrapTarget,
+        signal: agentAbort.signal,
+        trace,
+        progressive,
+        progressiveDelayMs,
+        progressiveInitialCount,
+        atomicDelayMs,
+        steerSourceFile: runtime.steer?.sourceFile,
+        steerTarget: runtime.steer?.target,
+      };
+      const loops = [runAgentLoop({ ...loopOptions, log: (m) => log('[worker] ' + m) })];
+      if (progressive) {
+        loops.push(runAgentLoop({ ...loopOptions, log: (m) => log('[supervisor] ' + m) }));
+      }
+      agentDone = Promise.all(loops);
     }
-    agentDone = Promise.all(loops);
 
     const scheme = runtime.scheme || 'http';
     ctx = await browser.newContext({
@@ -342,6 +360,7 @@ export async function bootFixtureSession({
       ctx,
       dev,
       live,
+      worker: externalWorker,
       consoleErrors,
       stopLiveServer: stopLiveForDeferredWork,
       teardown,

tests/live-e2e/ui.mjs

@@ -544,6 +544,36 @@ export async function setCount(page, count) {
   throw new Error(`could not cycle count to ${count}`);
 }
 
+/** Select a named Impeccable sub-command from the configure-row picker. */
+export async function selectAction(page, action) {
+  const pickerSelector = '#impeccable-live-picker';
+  const opened = await page.evaluate(({ barSel, pickerSel }) => {
+    const query = window.__impeccableLiveQuery || ((selector) => document.querySelector(selector));
+    const bar = query(barSel);
+    const picker = query(pickerSel);
+    const actionControl = [...(bar?.querySelectorAll('button') || [])]
+      .find((button) => (button.textContent || '').includes('\u25BE'));
+    if (!actionControl || !picker) return false;
+    actionControl.click();
+    return true;
+  }, { barSel: BAR_ID, pickerSel: pickerSelector });
+  if (!opened) throw new Error('could not open Live action picker');
+
+  await page.waitForFunction((selector) => {
+    const picker = window.__impeccableLiveQuery(selector);
+    return picker && picker.style.display !== 'none';
+  }, pickerSelector, { timeout: 5_000 });
+
+  const selected = await page.evaluate(({ pickerSel, value }) => {
+    const picker = window.__impeccableLiveQuery(pickerSel);
+    const chip = picker?.querySelector(`button[data-action="${CSS.escape(value)}"]`);
+    if (!chip) return false;
+    chip.click();
+    return true;
+  }, { pickerSel: pickerSelector, value: action });
+  if (!selected) throw new Error(`Live action ${JSON.stringify(action)} is unavailable`);
+}
+
 /**
  * Click Go. Browser POSTs the generate event; the agent picks it up. Headed
  * browser runs can occasionally accept the click without leaving configure