From aba1067cdfe4dcd489600e0f15b5adfcb99adb15 Mon Sep 17 00:00:00 2001 From: Paul Bakaus Date: Sun, 12 Jul 2026 20:50:28 -0700 Subject: [PATCH] Benchmark the production Codex Live worker Let the browser E2E harness launch an independent production worker, exercise real sub-command selection, and carry realistic product/design context.\n\nAI-assisted: OpenAI Codex. --- scripts/benchmark-live.mjs | 75 ++++++++++++++++++- .../files/DESIGN.md | 7 ++ .../files/PRODUCT.md | 7 ++ .../vite8-react-brand-fidelity/fixture.json | 2 +- tests/live-e2e/session.mjs | 61 +++++++++------ tests/live-e2e/ui.mjs | 30 ++++++++ 6 files changed, 157 insertions(+), 25 deletions(-) create mode 100644 tests/framework-fixtures/vite8-react-brand-fidelity/files/DESIGN.md create mode 100644 tests/framework-fixtures/vite8-react-brand-fidelity/files/PRODUCT.md diff --git a/scripts/benchmark-live.mjs b/scripts/benchmark-live.mjs index 0256f3098..e2a1164b4 100644 --- a/scripts/benchmark-live.mjs +++ b/scripts/benchmark-live.mjs @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { spawn } from 'node:child_process'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -12,6 +13,7 @@ import { clickGo, drawAnnotationPinAndStroke, pickElement, + selectAction, waitForCycling, waitForHandshake, } from '../tests/live-e2e/ui.mjs'; @@ -27,7 +29,7 @@ const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const args = parseArgs(process.argv.slice(2)); const fixtureName = String(args.fixture || 'vite8-react-plain'); const iterations = positiveInt(args.iterations, 5); -const agentMode = args.agent === 'llm' ? 'llm' : 'fake'; +const agentMode = args.agent === 'codex' ? 'codex' : args.agent === 'llm' ? 'llm' : 'fake'; const scenario = args.scenario === 'annotated' ? 'annotated' : 'plain'; const delivery = args.delivery === 'progressive' ? 'progressive' : 'atomic'; const simulatedTailMs = positiveInt(args.simulatedTailMs, 0); @@ -51,6 +53,7 @@ try { fixture, browser, agent: agentInfo.agent, + startWorker: agentInfo.startWorker, wrapTarget: wrapTargetFromPickedElement, trace: recorder.trace, progressive: delivery === 'progressive', @@ -81,6 +84,7 @@ try { const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title'; for (let iteration = 1; iteration <= iterations; iteration += 1) { await pickElement(session.page, pickSelector, { resetPickMode: iteration > 1 }); + if (args.action) await selectAction(session.page, String(args.action)); if (scenario === 'annotated') { await drawAnnotationPinAndStroke(session.page, { comment: 'Benchmark annotation' }); } @@ -94,7 +98,7 @@ try { await clickGo(session.page); recorder.mark('ui.generating_visible', { iteration, scenario }); await firstVariant; - await waitForCycling(session.page, 3, { timeout: agentMode === 'llm' ? 150_000 : 30_000 }); + await waitForCycling(session.page, 3, { timeout: agentMode === 'fake' ? 30_000 : 240_000 }); recorder.mark('browser.all_variants', { iteration, scenario }); const browserTiming = await readBrowserTimingProbe(session.page); @@ -116,7 +120,7 @@ try { fixture: fixtureName, agent: agentMode, provider: agentInfo.provider, - model: agentInfo.model, + model: session.worker?.state?.model || agentInfo.model, scenario, runs, events: recorder.events, @@ -150,6 +154,15 @@ try { async function resolveAgent(mode, options) { if (mode === 'fake') return { agent: createFakeAgent(), provider: 'deterministic', model: null, promptMode: null }; + if (mode === 'codex') { + return { + agent: null, + provider: 'openai-codex-app-server', + model: options.model || null, + promptMode: 'production-live-contract', + startWorker: (context) => startCodexProductionWorker(context, options), + }; + } const config = resolveLlmAgentConfig({ provider: options.provider, model: options.model, @@ -165,6 +178,62 @@ async function resolveAgent(mode, options) { return { agent, provider: config.provider, model: config.model, promptMode: 'synthetic-element-contract' }; } +async function startCodexProductionWorker({ tmp, scriptsDir, log }, options) { + const script = join(scriptsDir, 'live-codex-worker.mjs'); + const statePath = join(tmp, '.impeccable', 'live', 'codex-worker.json'); + const child = spawn(process.execPath, [script, '--foreground'], { + cwd: tmp, + env: { + ...process.env, + IMPECCABLE_LIVE_CODEX_WORKER: '1', + IMPECCABLE_LIVE_CODEX_PROFILE: String(options.profile || 'quality'), + IMPECCABLE_LIVE_CODEX_EFFORT: String(options.effort || 'medium'), + ...(options.model ? { IMPECCABLE_LIVE_CODEX_MODEL: String(options.model) } : {}), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const output = []; + const capture = (chunk) => { + const text = chunk.toString(); + output.push(text); + if (output.length > 200) output.shift(); + log(`[codex-worker] ${text.trimEnd()}`); + }; + child.stdout.on('data', capture); + child.stderr.on('data', capture); + const done = new Promise((resolve) => child.once('exit', (code, signal) => resolve({ code, signal }))); + const state = await waitForWorkerState(statePath, child, output, positiveInt(options.workerTimeoutMs, 20_000)); + return { + child, + state, + done, + async stop() { + if (child.exitCode != null || child.signalCode != null) return; + child.kill('SIGTERM'); + await Promise.race([done, new Promise((resolve) => setTimeout(resolve, 5_000))]); + if (child.exitCode == null && child.signalCode == null) child.kill('SIGKILL'); + }, + }; +} + +async function waitForWorkerState(statePath, child, output, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (child.exitCode != null || child.signalCode != null) { + throw new Error(`Codex production worker exited before ready.\n${output.join('')}`); + } + try { + const state = JSON.parse(await readFile(statePath, 'utf-8')); + if (state.status === 'error') throw new Error(`Codex production worker failed: ${state.error}\n${output.join('')}`); + if (['ready', 'working'].includes(state.status)) return state; + } catch (error) { + if (error.code !== 'ENOENT' && error.name !== 'SyntaxError') throw error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`Codex production worker was not ready after ${timeoutMs}ms.\n${output.join('')}`); +} + function createSplitProgressiveAgent(agent) { const firstBySession = new Map(); return { diff --git a/tests/framework-fixtures/vite8-react-brand-fidelity/files/DESIGN.md b/tests/framework-fixtures/vite8-react-brand-fidelity/files/DESIGN.md new file mode 100644 index 000000000..235deb133 --- /dev/null +++ b/tests/framework-fixtures/vite8-react-brand-fidelity/files/DESIGN.md @@ -0,0 +1,7 @@ +# Design system + +- Warm paper, dark ink, moss, and brass only. +- Georgia display type with a restrained sans body. +- Editorial, practical, quiet, and tactile. +- Reuse the existing CSS custom properties. Do not add colors, fonts, gradients, shadows, glow, glass, or decorative effects. +- Square, rule-led compositions are preferred to card stacks and rounded containers. diff --git a/tests/framework-fixtures/vite8-react-brand-fidelity/files/PRODUCT.md b/tests/framework-fixtures/vite8-react-brand-fidelity/files/PRODUCT.md new file mode 100644 index 000000000..7c9bd704b --- /dev/null +++ b/tests/framework-fixtures/vite8-react-brand-fidelity/files/PRODUCT.md @@ -0,0 +1,7 @@ +# Northstar Field Journal + +An independent quarterly field guide for design-conscious weekend walkers. Readers value practical detail, editorial restraint, and objects worth keeping. The offer card should make issue eight feel collectible without becoming luxurious or loud. + +## Platform + +web diff --git a/tests/framework-fixtures/vite8-react-brand-fidelity/fixture.json b/tests/framework-fixtures/vite8-react-brand-fidelity/fixture.json index 6cd9f1fa9..7b2ed9ce4 100644 --- a/tests/framework-fixtures/vite8-react-brand-fidelity/fixture.json +++ b/tests/framework-fixtures/vite8-react-brand-fidelity/fixture.json @@ -5,7 +5,7 @@ "insertBefore": "", "commentSyntax": "html" }, - "sourceFiles": ["index.html", "src/App.jsx", "src/main.jsx", "src/styles.css", "vite.config.js"], + "sourceFiles": ["PRODUCT.md", "DESIGN.md", "index.html", "src/App.jsx", "src/main.jsx", "src/styles.css", "vite.config.js"], "generatedFiles": [], "wrapCases": [ { diff --git a/tests/live-e2e/session.mjs b/tests/live-e2e/session.mjs index bb5b34192..90419bd14 100644 --- a/tests/live-e2e/session.mjs +++ b/tests/live-e2e/session.mjs @@ -219,6 +219,10 @@ export async function stopDevServer(child) { * @param {import('playwright').Browser} opts.browser shared browser instance * @param {object} opts.agent VariantAgent (defaults to fake) * @param {object|function=} opts.wrapTarget live-wrap target or event mapper + * @param {(context: object) => Promise} [opts.startWorker] + * Optional production worker factory. Return {stop, done}; when used, + * omit `agent` so the deterministic in-process loop is not started. + * @param {(context: object) => Promise|void} [opts.prepareTmp] * @param {(msg: string) => void} [opts.log] */ export async function bootFixtureSession({ @@ -227,6 +231,8 @@ export async function bootFixtureSession({ browser, agent, wrapTarget, + startWorker, + prepareTmp, log = () => {}, trace = () => {}, progressive = false, @@ -242,12 +248,15 @@ export async function bootFixtureSession({ let dev; let agentAbort; let agentDone; + let externalWorker; let ctx; const teardown = async () => { try { if (ctx) await ctx.close(); } catch {} try { if (agentAbort) agentAbort.abort(); } catch {} try { if (agentDone) await agentDone.catch(() => {}); } catch {} + try { if (externalWorker?.stop) await externalWorker.stop(); } catch {} + try { if (externalWorker?.done) await externalWorker.done.catch(() => {}); } catch {} try { if (dev?.child) await stopDevServer(dev.child); } catch {} try { if (live) stopLiveServer(tmp); } catch {} try { rmSync(tmp, { recursive: true, force: true }); } catch {} @@ -261,6 +270,7 @@ export async function bootFixtureSession({ try { const startedAt = Date.now(); + if (prepareTmp) await prepareTmp({ tmp, fixture, scriptsDir: SCRIPTS_DIR, trace, log }); trace('setup.install.start', { fixture: name }); log(`installing deps`); runInstall(tmp, runtime.install); @@ -274,6 +284,12 @@ export async function bootFixtureSession({ trace('setup.live_server.end', { fixture: name, port: live.port }); log(`live-server ready in ${formatDuration(Date.now() - liveStartedAt)}`); + if (startWorker) { + trace('setup.worker.start', { fixture: name }); + externalWorker = await startWorker({ tmp, fixture, scriptsDir: SCRIPTS_DIR, live, trace, log }); + trace('setup.worker.end', { fixture: name }); + } + const injectStartedAt = Date.now(); trace('setup.inject.start', { fixture: name }); log(`live-inject --port ${live.port}`); @@ -291,28 +307,30 @@ export async function bootFixtureSession({ log(`dev server ready on ${devPort} in ${formatDuration(Date.now() - devStartedAt)}`); // Agent loop runs concurrently — abort on teardown. - agentAbort = new AbortController(); - const loopOptions = { - tmp, - scriptsDir: SCRIPTS_DIR, - port: live.port, - token: live.token, - agent, - wrapTarget, - signal: agentAbort.signal, - trace, - progressive, - progressiveDelayMs, - progressiveInitialCount, - atomicDelayMs, - steerSourceFile: runtime.steer?.sourceFile, - steerTarget: runtime.steer?.target, - }; - const loops = [runAgentLoop({ ...loopOptions, log: (m) => log('[worker] ' + m) })]; - if (progressive) { - loops.push(runAgentLoop({ ...loopOptions, log: (m) => log('[supervisor] ' + m) })); + if (agent) { + agentAbort = new AbortController(); + const loopOptions = { + tmp, + scriptsDir: SCRIPTS_DIR, + port: live.port, + token: live.token, + agent, + wrapTarget, + signal: agentAbort.signal, + trace, + progressive, + progressiveDelayMs, + progressiveInitialCount, + atomicDelayMs, + steerSourceFile: runtime.steer?.sourceFile, + steerTarget: runtime.steer?.target, + }; + const loops = [runAgentLoop({ ...loopOptions, log: (m) => log('[worker] ' + m) })]; + if (progressive) { + loops.push(runAgentLoop({ ...loopOptions, log: (m) => log('[supervisor] ' + m) })); + } + agentDone = Promise.all(loops); } - agentDone = Promise.all(loops); const scheme = runtime.scheme || 'http'; ctx = await browser.newContext({ @@ -342,6 +360,7 @@ export async function bootFixtureSession({ ctx, dev, live, + worker: externalWorker, consoleErrors, stopLiveServer: stopLiveForDeferredWork, teardown, diff --git a/tests/live-e2e/ui.mjs b/tests/live-e2e/ui.mjs index db9d60fb5..9ddf0701f 100644 --- a/tests/live-e2e/ui.mjs +++ b/tests/live-e2e/ui.mjs @@ -544,6 +544,36 @@ export async function setCount(page, count) { throw new Error(`could not cycle count to ${count}`); } +/** Select a named Impeccable sub-command from the configure-row picker. */ +export async function selectAction(page, action) { + const pickerSelector = '#impeccable-live-picker'; + const opened = await page.evaluate(({ barSel, pickerSel }) => { + const query = window.__impeccableLiveQuery || ((selector) => document.querySelector(selector)); + const bar = query(barSel); + const picker = query(pickerSel); + const actionControl = [...(bar?.querySelectorAll('button') || [])] + .find((button) => (button.textContent || '').includes('\u25BE')); + if (!actionControl || !picker) return false; + actionControl.click(); + return true; + }, { barSel: BAR_ID, pickerSel: pickerSelector }); + if (!opened) throw new Error('could not open Live action picker'); + + await page.waitForFunction((selector) => { + const picker = window.__impeccableLiveQuery(selector); + return picker && picker.style.display !== 'none'; + }, pickerSelector, { timeout: 5_000 }); + + const selected = await page.evaluate(({ pickerSel, value }) => { + const picker = window.__impeccableLiveQuery(pickerSel); + const chip = picker?.querySelector(`button[data-action="${CSS.escape(value)}"]`); + if (!chip) return false; + chip.click(); + return true; + }, { pickerSel: pickerSelector, value: action }); + if (!selected) throw new Error(`Live action ${JSON.stringify(action)} is unavailable`); +} + /** * Click Go. Browser POSTs the generate event; the agent picks it up. Headed * browser runs can occasionally accept the click without leaving configure