diff --git a/scripts/benchmark-live-codex-architecture.mjs b/scripts/benchmark-live-codex-architecture.mjs index f9978fad1..4b0bc2b76 100644 --- a/scripts/benchmark-live-codex-architecture.mjs +++ b/scripts/benchmark-live-codex-architecture.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import os from 'node:os'; import path from 'node:path'; import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; @@ -34,7 +33,7 @@ const iterations = positiveInteger(args.iterations, 2); const timeoutMs = positiveInteger(args.timeout, 300_000); const outputPath = args.output ? path.resolve(ROOT, String(args.output)) : null; const profileIds = csv(args.profiles || 'direct-sol,cold-app-server,warm-app-server'); -const taskIds = csv(args.tasks || 'editorial-bolder,operations-polish'); +const taskIds = csv(args.tasks || 'editorial-bolder,operations-polish,operations-annotated'); const judgeEnabled = args.judge !== false; const loadedEnv = loadBenchmarkEnv({ repoRoot: ROOT, explicitPath: args.envFile && path.resolve(args.envFile) }); const skillPath = path.join(ROOT, '.agents', 'skills', 'impeccable', 'SKILL.md'); @@ -79,9 +78,12 @@ if (args.dryRun) { process.exit(0); } -const scratch = await mkdtemp(path.join(os.tmpdir(), 'impeccable-codex-architecture-')); +const scratchRoot = path.join(ROOT, '.impeccable', 'live'); +await mkdir(scratchRoot, { recursive: true }); +const scratch = await mkdtemp(path.join(scratchRoot, 'architecture-')); const schemaPath = path.join(scratch, 'output-schema.json'); await writeFile(schemaPath, JSON.stringify(CODEX_QUALITY_OUTPUT_SCHEMA)); +await prepareAnnotationScreenshots(); const runs = []; try { for (const profile of profileIds) { @@ -126,20 +128,25 @@ async function runDirect({ profile, task, iteration }) { const prompt = [ '$impeccable', 'Use the attached Impeccable skill. This automated benchmark already resolved Setup context below; do not rerun setup or edit files.', + '', + buildCodexWorkerInstructions(liveSpec), + '', buildCodexQualityPrompt(task, { actionReference, fullContext: true }), ].join('\n\n'); try { + const directArgs = [ + 'exec', '--ephemeral', '--ignore-user-config', '--dangerously-bypass-hook-trust', + '-C', ROOT, '-s', 'read-only', '-m', model, + '-c', 'model_reasoning_effort="medium"', + '--output-schema', schemaPath, + '--output-last-message', outputFile, + ]; + if (task.screenshotPath) directArgs.push('-i', task.screenshotPath); + directArgs.push('--json', prompt); const result = await runCodexExecBenchmark({ cwd: ROOT, timeoutMs, - args: [ - 'exec', '--ephemeral', '--ignore-user-config', '--dangerously-bypass-hook-trust', - '-C', ROOT, '-s', 'read-only', '-m', model, - '-c', 'model_reasoning_effort="medium"', - '--output-schema', schemaPath, - '--output-last-message', outputFile, - '--json', prompt, - ], + args: directArgs, }); const output = JSON.parse(await readFile(outputFile, 'utf-8')); return finishRun({ @@ -151,6 +158,7 @@ async function runDirect({ profile, task, iteration }) { generationMs: result.firstAgentMessageMs == null || result.turnStartedMs == null ? result.durationMs : result.firstAgentMessageMs - result.turnStartedMs, + firstUsableMs: result.firstAgentMessageMs ?? result.durationMs, totalMs: result.durationMs, usage: result.usage, transport: { @@ -181,6 +189,7 @@ async function runColdAppServer({ profile, task, iteration }) { output: turn.output, startupMs, generationMs: turn.durationMs, + firstUsableMs: startupMs + turn.durationMs, totalMs: performance.now() - startedAt, usage: normalizeAppServerUsage(turn.turn), }); @@ -214,6 +223,7 @@ async function runWarmProfile(profile) { output: turn.output, startupMs: iteration === 1 && task === tasks[0] ? coldStartupMs : 0, generationMs: turn.durationMs, + firstUsableMs: turn.durationMs + (iteration === 1 && task === tasks[0] ? coldStartupMs : 0), totalMs: performance.now() - turnStartedAt + (iteration === 1 && task === tasks[0] ? coldStartupMs : 0), usage: normalizeAppServerUsage(turn.turn), transport: { persistentThread: true, coldStartupMs: round(coldStartupMs) }, @@ -244,6 +254,9 @@ function threadParams(profile) { async function runAppServerTurn(client, thread, task) { const actionReference = await readFile(path.join(referenceDir, `${task.action}.md`), 'utf-8'); const prompt = buildCodexQualityPrompt(task, { actionReference, fullContext: true }); + let output = null; + let firstAgentMessageMs = null; + const startedAt = performance.now(); const result = await client.startTurn({ threadId: thread.id, input: buildCodexWorkerTurnInputs({ prompt, skillPath, screenshotPath: task.screenshotPath, cwd: ROOT }), @@ -254,11 +267,63 @@ async function runAppServerTurn(client, thread, task) { approvalPolicy: 'never', sandboxPolicy: { type: 'readOnly' }, outputSchema: CODEX_QUALITY_OUTPUT_SCHEMA, + onAgentMessage: (message) => { + if (output) return; + output = JSON.parse(message); + firstAgentMessageMs = performance.now() - startedAt; + }, }); - return { output: JSON.parse(result.message), durationMs: result.durationMs, turn: result }; + return { + output: output || JSON.parse(result.message), + durationMs: firstAgentMessageMs ?? result.firstAgentMessageMs ?? result.durationMs, + completionMs: result.durationMs, + turn: result, + }; } -async function finishRun({ profile, task, iteration, output, startupMs, generationMs, totalMs, usage, transport = null }) { +async function prepareAnnotationScreenshots() { + const annotated = tasks.filter((task) => task.annotation); + if (annotated.length === 0) return; + const { chromium } = await import('playwright'); + const browser = await chromium.launch({ headless: true }); + try { + for (const task of annotated) { + const screenshotPath = path.join(scratch, `${task.id}.png`); + const page = await browser.newPage({ viewport: { width: 1080, height: 720 }, deviceScaleFactor: 1 }); + await page.setContent(` +
+

Monday, 14 July

Fulfillment overview

Monitor the work that can put today’s dispatch at risk.

+

Ready

184

31 due before noon

At risk

12

4 need assignment

Blocked

3

Oldest waiting 42 min

+

Priority queue

Needs attention

DispatchDestinationOwnerStatusDue
DP-2048PortlandUnassignedBlocked09:30
DP-2051OaklandM. ChenAt risk10:15
+
${escapeHtml(task.annotation.comment)}
+
+ `, { waitUntil: 'load' }); + await page.screenshot({ path: screenshotPath, fullPage: true }); + await page.close(); + task.screenshotPath = screenshotPath; + } + } finally { + await browser.close(); + } +} + +async function finishRun({ profile, task, iteration, output, startupMs, generationMs, firstUsableMs, totalMs, usage, transport = null }) { const deterministic = scoreCodexQualityOutput(task, output); const judge = judgeEnabled ? await judgeOutput(task, output) : null; return { @@ -269,6 +334,7 @@ async function finishRun({ profile, task, iteration, output, startupMs, generati effort: 'medium', startupMs: round(startupMs), generationMs: round(generationMs), + firstUsableMs: round(firstUsableMs), totalMs: round(totalMs), usage, transport, @@ -302,7 +368,7 @@ async function judgeOutput(task, output) { } function normalizeAppServerUsage(turn) { - const usage = turn?.completed?.params?.turn?.usage || turn?.turn?.usage || null; + const usage = turn?.tokenUsage?.last || turn?.completed?.params?.turn?.usage || turn?.turn?.usage || null; if (!usage) return null; return { input_tokens: usage.inputTokens ?? usage.input_tokens ?? null, @@ -347,3 +413,9 @@ function positiveInteger(value, fallback) { function round(value) { return Number.isFinite(value) ? Math.round(value * 100) / 100 : null; } + +function escapeHtml(value) { + return String(value).replace(/[&<>"']/g, (character) => ({ + '&': '&', '<': '<', '>': '>', '"': '"', "'": ''', + })[character]); +} diff --git a/scripts/lib/codex-exec-benchmark.mjs b/scripts/lib/codex-exec-benchmark.mjs index 1e977ac98..ba2dfdd2f 100644 --- a/scripts/lib/codex-exec-benchmark.mjs +++ b/scripts/lib/codex-exec-benchmark.mjs @@ -89,6 +89,8 @@ export function summarizeArchitectureRuns(runs) { return { runs: runs.length, passed: completed.filter((run) => run.passed).length, + medianFirstUsableMs: percentile(completed.map((run) => run.firstUsableMs).filter(Number.isFinite), 0.5), + p95FirstUsableMs: percentile(completed.map((run) => run.firstUsableMs).filter(Number.isFinite), 0.95), medianTotalMs: percentile(completed.map((run) => run.totalMs), 0.5), p95TotalMs: percentile(completed.map((run) => run.totalMs), 0.95), medianStartupMs: percentile(completed.map((run) => run.startupMs).filter(Number.isFinite), 0.5), diff --git a/scripts/lib/live-codex-quality-benchmark.mjs b/scripts/lib/live-codex-quality-benchmark.mjs index 81c90b48b..12dd4c017 100644 --- a/scripts/lib/live-codex-quality-benchmark.mjs +++ b/scripts/lib/live-codex-quality-benchmark.mjs @@ -137,7 +137,7 @@ th { background: var(--surface-subtle); color: var(--ink-muted); font-size: 0.75 export function createCodexQualityTasks({ repoRoot }) { const fixtureDir = path.join(repoRoot, 'tests', 'framework-fixtures', 'vite8-react-brand-fidelity', 'files', 'src'); - return [ + const tasks = [ { id: 'editorial-bolder', action: 'bolder', @@ -168,6 +168,22 @@ export function createCodexQualityTasks({ repoRoot }) { judgeFocus: 'Is this a materially more polished, efficient operations surface, with excellent scan hierarchy and interaction detail, without changing its product model or turning it into a decorative dashboard?', }, ]; + const operations = tasks[1]; + tasks.push({ + ...operations, + id: 'operations-annotated', + brief: 'Polish this fulfillment dashboard while following the attached annotation. Make the At risk state easier to identify in a fast scan without making the whole dashboard louder. Preserve the existing information architecture, components, terminology, and token palette.', + annotation: { + comment: 'Make this risk state easier to scan without making the whole dashboard louder.', + target: 'The At risk metric card and its relationship to the priority queue.', + strokes: 1, + }, + allowedForbiddenMatches: [ + /box-shadow\s*:\s*inset\s+(?:0|0?\.\d+(?:rem|px))\s+0(?:\s+0)?\s+var\(--warning\)\s*;?/gi, + ], + judgeFocus: 'Does the implementation respond precisely to the visual annotation by strengthening risk-state scanning, while remaining a calm operations UI and preserving the established component and status system?', + }); + return tasks; } export function buildCodexQualityPrompt(task, { actionReference = '', fullContext = false } = {}) { @@ -182,6 +198,7 @@ export function buildCodexQualityPrompt(task, { actionReference = '', fullContex '', task.product, '', '', task.design, '', '', actionReference, '', + '', JSON.stringify(task.annotation || {}, null, 2), '', '', JSON.stringify(task.files, null, 2), '', ].filter((line) => line !== '').join('\n'); } @@ -193,17 +210,25 @@ export function scoreCodexQualityOutput(task, output) { const source = byPath['src/App.jsx'] || ''; const css = byPath['src/styles.css'] || ''; const normalizedSource = source.replace(/[\s"']/g, ''); + const visibleSourceText = source.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim(); const checks = { exactFiles: files.length === 2 && Boolean(source) && Boolean(css), implementationChanged: source !== task.files['src/App.jsx'] || css !== task.files['src/styles.css'], - copyPreserved: task.requiredCopy.every((value) => combined.includes(value)), + copyPreserved: task.requiredCopy.every((value) => ( + combined.includes(value) || visibleSourceText.includes(value.replace(/\s+/g, ' ').trim()) + )), contractsPreserved: task.requiredSource.every((value) => normalizedSource.includes(value.replace(/[\s"']/g, ''))), tokensPreserved: task.requiredTokens.every((value) => css.includes(value)), noForbiddenDrift: task.forbidden.every((pattern) => { + const outputWithoutAllowedMatches = stripAllowedForbiddenMatches(combined, task.allowedForbiddenMatches); + const inputWithoutAllowedMatches = stripAllowedForbiddenMatches( + Object.values(task.files).join('\n'), + task.allowedForbiddenMatches, + ); pattern.lastIndex = 0; - const outputMatches = combined.match(pattern) || []; + const outputMatches = outputWithoutAllowedMatches.match(pattern) || []; pattern.lastIndex = 0; - const inputMatches = Object.values(task.files).join('\n').match(pattern) || []; + const inputMatches = inputWithoutAllowedMatches.match(pattern) || []; return outputMatches.length <= inputMatches.length; }), }; @@ -213,6 +238,13 @@ export function scoreCodexQualityOutput(task, output) { }; } +function stripAllowedForbiddenMatches(value, patterns = []) { + return patterns.reduce((result, pattern) => { + pattern.lastIndex = 0; + return result.replace(pattern, ''); + }, value); +} + export function buildJudgePrompt(task, output) { return [ 'You are an exacting independent frontend design reviewer. Score the revised code, not the prose around it.', @@ -221,6 +253,7 @@ export function buildJudgePrompt(task, output) { '', `TASK: /${task.action} — ${task.brief}`, `REVIEW FOCUS: ${task.judgeFocus}`, + '', JSON.stringify(task.annotation || {}, null, 2), '', '', task.product, '', '', task.design, '', '', JSON.stringify(task.files, null, 2), '', diff --git a/tests/live-codex-exec-benchmark.test.mjs b/tests/live-codex-exec-benchmark.test.mjs index ac6e7e46a..ca07be272 100644 --- a/tests/live-codex-exec-benchmark.test.mjs +++ b/tests/live-codex-exec-benchmark.test.mjs @@ -32,12 +32,13 @@ describe('direct Codex architecture benchmark', () => { it('summarizes startup, generation, total, quality, and token medians', () => { const summary = summarizeArchitectureRuns([ - { passed: true, startupMs: 10, generationMs: 100, totalMs: 110, usage: { input_tokens: 1000, cached_input_tokens: 500, output_tokens: 100 } }, - { passed: false, startupMs: 20, generationMs: 200, totalMs: 220, usage: { input_tokens: 2000, cached_input_tokens: 1000, output_tokens: 200 } }, + { passed: true, startupMs: 10, generationMs: 100, firstUsableMs: 110, totalMs: 110, usage: { input_tokens: 1000, cached_input_tokens: 500, output_tokens: 100 } }, + { passed: false, startupMs: 20, generationMs: 200, firstUsableMs: 220, totalMs: 220, usage: { input_tokens: 2000, cached_input_tokens: 1000, output_tokens: 200 } }, ]); assert.equal(summary.runs, 2); assert.equal(summary.passed, 1); assert.equal(summary.medianTotalMs, 165); + assert.equal(summary.medianFirstUsableMs, 165); assert.equal(summary.medianInputTokens, 1500); }); }); diff --git a/tests/live-codex-quality-benchmark.test.mjs b/tests/live-codex-quality-benchmark.test.mjs index 76a015cec..93e318460 100644 --- a/tests/live-codex-quality-benchmark.test.mjs +++ b/tests/live-codex-quality-benchmark.test.mjs @@ -18,6 +18,7 @@ describe('Codex Live quality benchmark', () => { assert.deepEqual(tasks.map((task) => `${task.action}:${task.id}`), [ 'bolder:editorial-bolder', 'polish:operations-polish', + 'polish:operations-annotated', ]); const prompt = buildCodexQualityPrompt(tasks[0], { actionReference: 'BOLDER', fullContext: true }); assert.match(prompt, /Impeccable skill is attached/); @@ -25,6 +26,7 @@ describe('Codex Live quality benchmark', () => { assert.match(prompt, //); assert.match(prompt, /BOLDER/); assert.match(prompt, /src\/App\.jsx/); + assert.equal(tasks[2].annotation.strokes, 1); }); it('rejects no-op, contract-breaking, and design-system-drifting output', () => { @@ -40,6 +42,14 @@ describe('Codex Live quality benchmark', () => { }; assert.equal(scoreCodexQualityOutput(task, cssOnly).passed, true, 'CSS-only design work is a material implementation change'); + const splitVisibleCopy = { + files: [ + { path: 'src/App.jsx', content: task.files['src/App.jsx'].replace('Field Notes', 'Field Notes') }, + { path: 'src/styles.css', content: `${task.files['src/styles.css']}\n.offer-card { min-height: 30rem; }` }, + ], + }; + assert.equal(scoreCodexQualityOutput(task, splitVisibleCopy).checks.copyPreserved, true); + const drift = { files: [ { path: 'src/App.jsx', content: task.files['src/App.jsx'].replace('Field Notes', 'Neon Notes') }, @@ -51,6 +61,20 @@ describe('Codex Live quality benchmark', () => { assert.equal(score.checks.noForbiddenDrift, false); }); + it('allows an annotation-scoped semantic risk rail but still rejects decorative shadows', () => { + const task = tasks[2]; + const withRiskRail = { + files: [ + { path: 'src/App.jsx', content: task.files['src/App.jsx'] }, + { path: 'src/styles.css', content: `${task.files['src/styles.css']}\n.metric--warning { box-shadow: inset 0.1875rem 0 0 var(--warning); }` }, + ], + }; + assert.equal(scoreCodexQualityOutput(task, withRiskRail).checks.noForbiddenDrift, true); + + withRiskRail.files[1].content += '\n.queue { box-shadow: 0 1rem 3rem rgb(0 0 0 / 0.2); }'; + assert.equal(scoreCodexQualityOutput(task, withRiskRail).checks.noForbiddenDrift, false); + }); + it('parses strict judge results and summarizes latency and quality', () => { const judge = parseJudgeResult('{"commandFidelity":8,"brandAndSystemFidelity":9,"frontendQuality":7,"taskCompletion":8,"criticalFailure":false,"summary":"Good."}'); assert.equal(judge.passed, true);