diff --git a/package.json b/package.json index fcb17746b..18ddae7c5 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "bench:live": "node scripts/benchmark-live.mjs", "bench:live:providers": "node scripts/benchmark-live-providers.mjs", "bench:live:codex-quality": "node scripts/benchmark-live-codex-worker.mjs", + "bench:live:codex-architecture": "node scripts/benchmark-live-codex-architecture.mjs", "audit": "bun audit --audit-level=moderate", "prepack": "cp README.md README.repo.md && cp README.npm.md README.md", "postpack": "cp README.repo.md README.md && rm README.repo.md", diff --git a/scripts/benchmark-live-codex-architecture.mjs b/scripts/benchmark-live-codex-architecture.mjs new file mode 100644 index 000000000..f9978fad1 --- /dev/null +++ b/scripts/benchmark-live-codex-architecture.mjs @@ -0,0 +1,349 @@ +#!/usr/bin/env node + +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { fileURLToPath } from 'node:url'; + +import { anthropic } from '@ai-sdk/anthropic'; +import { generateText } from 'ai'; + +import { + CodexAppServerClient, + selectQualityCodexModel, +} from '../skill/scripts/live/codex-app-server-client.mjs'; +import { + buildCodexWorkerInstructions, + buildCodexWorkerTurnInputs, +} from '../skill/scripts/live/codex-worker.mjs'; +import { runCodexExecBenchmark, summarizeArchitectureRuns } from './lib/codex-exec-benchmark.mjs'; +import { loadBenchmarkEnv } from './lib/live-provider-benchmark.mjs'; +import { + CODEX_QUALITY_OUTPUT_SCHEMA, + buildCodexQualityPrompt, + buildJudgePrompt, + createCodexQualityTasks, + parseJudgeResult, + scoreCodexQualityOutput, +} from './lib/live-codex-quality-benchmark.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const args = parseArgs(process.argv.slice(2)); +const iterations = positiveInteger(args.iterations, 2); +const timeoutMs = positiveInteger(args.timeout, 300_000); +const outputPath = args.output ? path.resolve(ROOT, String(args.output)) : null; +const profileIds = csv(args.profiles || 'direct-sol,cold-app-server,warm-app-server'); +const taskIds = csv(args.tasks || 'editorial-bolder,operations-polish'); +const judgeEnabled = args.judge !== false; +const loadedEnv = loadBenchmarkEnv({ repoRoot: ROOT, explicitPath: args.envFile && path.resolve(args.envFile) }); +const skillPath = path.join(ROOT, '.agents', 'skills', 'impeccable', 'SKILL.md'); +const referenceDir = path.join(ROOT, 'skill', 'reference'); +const liveSpec = await readFile(path.join(referenceDir, 'live.md'), 'utf-8'); +const tasks = createCodexQualityTasks({ repoRoot: ROOT }).filter((task) => taskIds.includes(task.id)); +if (tasks.length !== taskIds.length) throw new Error('unknown task id in --tasks'); +for (const profile of profileIds) { + if (!['direct-sol', 'cold-app-server', 'warm-app-server'].includes(profile)) throw new Error(`unknown profile ${profile}`); +} +if (judgeEnabled && !process.env.ANTHROPIC_API_KEY && !args.dryRun) { + throw new Error('ANTHROPIC_API_KEY is required unless --no-judge is passed'); +} + +let model = args.model || null; +if (!model && profileIds.some((profile) => profile.includes('app-server'))) { + const discovery = new CodexAppServerClient({ cwd: ROOT }); + await discovery.connect(); + try { + const selected = selectQualityCodexModel(await discovery.listModels()); + model = selected?.model || selected?.id || null; + } finally { + await discovery.close(); + } +} +model ||= 'gpt-5.6-sol'; + +if (args.dryRun) { + await emit({ + schemaVersion: 1, + mode: 'dry-run', + profiles: profileIds, + tasks: tasks.map((task) => task.id), + iterations, + model, + effort: 'medium', + judgeEnabled, + judgeAvailable: Boolean(process.env.ANTHROPIC_API_KEY), + envFilesLoaded: loadedEnv.length, + plannedModelRuns: profileIds.length * tasks.length * iterations, + }); + process.exit(0); +} + +const scratch = await mkdtemp(path.join(os.tmpdir(), 'impeccable-codex-architecture-')); +const schemaPath = path.join(scratch, 'output-schema.json'); +await writeFile(schemaPath, JSON.stringify(CODEX_QUALITY_OUTPUT_SCHEMA)); +const runs = []; +try { + for (const profile of profileIds) { + if (profile === 'warm-app-server') { + await runWarmProfile(profile); + continue; + } + for (let iteration = 1; iteration <= iterations; iteration += 1) { + for (const task of tasks) { + process.stderr.write(`[codex-architecture] ${profile} ${task.id} ${iteration}/${iterations}\n`); + runs.push(await (profile === 'direct-sol' + ? runDirect({ profile, task, iteration }) + : runColdAppServer({ profile, task, iteration }))); + } + } + } +} finally { + await rm(scratch, { recursive: true, force: true }); +} + +const report = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + mode: 'live', + model, + effort: 'medium', + iterations, + tasks: tasks.map((task) => ({ id: task.id, action: task.action, brief: task.brief })), + profiles: profileIds.map((profile) => ({ + id: profile, + summary: summarizeArchitectureRuns(runs.filter((run) => run.profile === profile)), + })), + judge: judgeEnabled ? { provider: 'anthropic', model: args.judgeModel || 'claude-sonnet-4-6' } : null, + runs, +}; +await emit(report); +process.exitCode = runs.every((run) => run.passed) ? 0 : 1; + +async function runDirect({ profile, task, iteration }) { + const outputFile = path.join(scratch, `${profile}-${task.id}-${iteration}.json`); + const actionReference = await readFile(path.join(referenceDir, `${task.action}.md`), 'utf-8'); + const prompt = [ + '$impeccable', + 'Use the attached Impeccable skill. This automated benchmark already resolved Setup context below; do not rerun setup or edit files.', + buildCodexQualityPrompt(task, { actionReference, fullContext: true }), + ].join('\n\n'); + try { + const result = await runCodexExecBenchmark({ + cwd: ROOT, + timeoutMs, + args: [ + 'exec', '--ephemeral', '--ignore-user-config', '--dangerously-bypass-hook-trust', + '-C', ROOT, '-s', 'read-only', '-m', model, + '-c', 'model_reasoning_effort="medium"', + '--output-schema', schemaPath, + '--output-last-message', outputFile, + '--json', prompt, + ], + }); + const output = JSON.parse(await readFile(outputFile, 'utf-8')); + return finishRun({ + profile, + task, + iteration, + output, + startupMs: result.turnStartedMs, + generationMs: result.firstAgentMessageMs == null || result.turnStartedMs == null + ? result.durationMs + : result.firstAgentMessageMs - result.turnStartedMs, + totalMs: result.durationMs, + usage: result.usage, + transport: { + threadStartedMs: round(result.threadStartedMs), + turnStartedMs: round(result.turnStartedMs), + firstAgentMessageMs: round(result.firstAgentMessageMs), + }, + }); + } catch (error) { + return failedRun({ profile, task, iteration, error }); + } +} + +async function runColdAppServer({ profile, task, iteration }) { + const startedAt = performance.now(); + const client = new CodexAppServerClient({ cwd: ROOT, turnTimeoutMs: timeoutMs }); + let thread = null; + try { + await client.connect(); + await client.listModels(); + thread = await client.startDedicatedThread(threadParams(profile)); + const startupMs = performance.now() - startedAt; + const turn = await runAppServerTurn(client, thread, task); + return finishRun({ + profile, + task, + iteration, + output: turn.output, + startupMs, + generationMs: turn.durationMs, + totalMs: performance.now() - startedAt, + usage: normalizeAppServerUsage(turn.turn), + }); + } catch (error) { + return failedRun({ profile, task, iteration, error }); + } finally { + if (thread) await client.archiveThread(thread.id).catch(() => {}); + await client.close().catch(() => {}); + } +} + +async function runWarmProfile(profile) { + const client = new CodexAppServerClient({ cwd: ROOT, turnTimeoutMs: timeoutMs }); + let thread = null; + const startedAt = performance.now(); + try { + await client.connect(); + await client.listModels(); + thread = await client.startDedicatedThread(threadParams(profile)); + const coldStartupMs = performance.now() - startedAt; + for (let iteration = 1; iteration <= iterations; iteration += 1) { + for (const task of tasks) { + process.stderr.write(`[codex-architecture] ${profile} ${task.id} ${iteration}/${iterations}\n`); + const turnStartedAt = performance.now(); + try { + const turn = await runAppServerTurn(client, thread, task); + runs.push(await finishRun({ + profile, + task, + iteration, + output: turn.output, + startupMs: iteration === 1 && task === tasks[0] ? coldStartupMs : 0, + generationMs: turn.durationMs, + totalMs: performance.now() - turnStartedAt + (iteration === 1 && task === tasks[0] ? coldStartupMs : 0), + usage: normalizeAppServerUsage(turn.turn), + transport: { persistentThread: true, coldStartupMs: round(coldStartupMs) }, + })); + } catch (error) { + runs.push(failedRun({ profile, task, iteration, error })); + } + } + } + } finally { + if (thread) await client.archiveThread(thread.id).catch(() => {}); + await client.close().catch(() => {}); + } +} + +function threadParams(profile) { + return { + model, + cwd: ROOT, + approvalPolicy: 'never', + sandbox: 'read-only', + ephemeral: profile === 'cold-app-server', + serviceName: `impeccable_live_architecture_${profile}`, + baseInstructions: buildCodexWorkerInstructions(liveSpec), + }; +} + +async function runAppServerTurn(client, thread, task) { + const actionReference = await readFile(path.join(referenceDir, `${task.action}.md`), 'utf-8'); + const prompt = buildCodexQualityPrompt(task, { actionReference, fullContext: true }); + const result = await client.startTurn({ + threadId: thread.id, + input: buildCodexWorkerTurnInputs({ prompt, skillPath, screenshotPath: task.screenshotPath, cwd: ROOT }), + cwd: ROOT, + model, + effort: 'medium', + summary: 'none', + approvalPolicy: 'never', + sandboxPolicy: { type: 'readOnly' }, + outputSchema: CODEX_QUALITY_OUTPUT_SCHEMA, + }); + return { output: JSON.parse(result.message), durationMs: result.durationMs, turn: result }; +} + +async function finishRun({ profile, task, iteration, output, startupMs, generationMs, totalMs, usage, transport = null }) { + const deterministic = scoreCodexQualityOutput(task, output); + const judge = judgeEnabled ? await judgeOutput(task, output) : null; + return { + profile, + task: task.id, + iteration, + model, + effort: 'medium', + startupMs: round(startupMs), + generationMs: round(generationMs), + totalMs: round(totalMs), + usage, + transport, + deterministic, + judge, + passed: deterministic.passed && (!judge || judge.passed), + output, + }; +} + +function failedRun({ profile, task, iteration, error }) { + return { + profile, + task: task.id, + iteration, + model, + effort: 'medium', + error: String(error?.stack || error), + passed: false, + }; +} + +async function judgeOutput(task, output) { + const response = await generateText({ + model: anthropic(args.judgeModel || 'claude-sonnet-4-6'), + system: 'Be strict, concrete, and independent. Return the requested JSON object only.', + prompt: buildJudgePrompt(task, output), + maxOutputTokens: 800, + }); + return parseJudgeResult(response.text); +} + +function normalizeAppServerUsage(turn) { + const usage = turn?.completed?.params?.turn?.usage || turn?.turn?.usage || null; + if (!usage) return null; + return { + input_tokens: usage.inputTokens ?? usage.input_tokens ?? null, + cached_input_tokens: usage.cachedInputTokens ?? usage.cached_input_tokens ?? null, + output_tokens: usage.outputTokens ?? usage.output_tokens ?? null, + reasoning_output_tokens: usage.reasoningOutputTokens ?? usage.reasoning_output_tokens ?? null, + }; +} + +async function emit(report) { + if (outputPath) { + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, JSON.stringify(report, null, 2) + '\n'); + } + process.stdout.write(JSON.stringify(report, null, 2) + '\n'); +} + +function parseArgs(values) { + const result = {}; + for (let index = 0; index < values.length; index += 1) { + const value = values[index]; + if (value === '--dry-run') result.dryRun = true; + else if (value === '--no-judge') result.judge = false; + else if (value.startsWith('--')) { + const [rawKey, inline] = value.slice(2).split('=', 2); + const key = rawKey.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()); + result[key] = inline ?? values[++index]; + } + } + return result; +} + +function csv(value) { + return String(value).split(',').map((item) => item.trim()).filter(Boolean); +} + +function positiveInteger(value, fallback) { + const parsed = Number.parseInt(value, 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function round(value) { + return Number.isFinite(value) ? Math.round(value * 100) / 100 : null; +} diff --git a/scripts/lib/codex-exec-benchmark.mjs b/scripts/lib/codex-exec-benchmark.mjs new file mode 100644 index 000000000..1e977ac98 --- /dev/null +++ b/scripts/lib/codex-exec-benchmark.mjs @@ -0,0 +1,112 @@ +import { spawn } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; + +export function runCodexExecBenchmark({ + command = 'codex', + args, + cwd = process.cwd(), + env = process.env, + timeoutMs = 300_000, + spawnFactory = spawn, +} = {}) { + if (!Array.isArray(args) || args.length === 0) throw new TypeError('args are required'); + const startedAt = performance.now(); + return new Promise((resolve, reject) => { + const child = spawnFactory(command, args, { + cwd, + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + let buffer = ''; + let threadStartedMs = null; + let turnStartedMs = null; + let firstAgentMessageMs = null; + let usage = null; + const events = []; + const timer = setTimeout(() => { + child.kill('SIGTERM'); + reject(new Error(`codex exec timed out after ${timeoutMs}ms`)); + }, timeoutMs); + timer.unref?.(); + + const consumeLine = (line) => { + if (!line.trim()) return; + let event; + try { event = JSON.parse(line); } catch { return; } + events.push(event); + const elapsed = performance.now() - startedAt; + if (event.type === 'thread.started' && threadStartedMs == null) threadStartedMs = elapsed; + if (event.type === 'turn.started' && turnStartedMs == null) turnStartedMs = elapsed; + if (event.type === 'item.completed' && event.item?.type === 'agent_message' && firstAgentMessageMs == null) { + firstAgentMessageMs = elapsed; + } + if (event.type === 'turn.completed') usage = event.usage || null; + }; + + child.stdout.on('data', (chunk) => { + const text = String(chunk); + stdout += text; + buffer += text; + let newline; + while ((newline = buffer.indexOf('\n')) !== -1) { + consumeLine(buffer.slice(0, newline)); + buffer = buffer.slice(newline + 1); + } + }); + child.stderr.on('data', (chunk) => { stderr += String(chunk); }); + child.once('error', (error) => { + clearTimeout(timer); + reject(error); + }); + child.once('exit', (code, signal) => { + clearTimeout(timer); + if (buffer) consumeLine(buffer); + const durationMs = performance.now() - startedAt; + if (code !== 0) { + const error = new Error(`codex exec failed (${code ?? signal}): ${stderr || stdout}`); + error.code = code; + reject(error); + return; + } + resolve({ + durationMs, + threadStartedMs, + turnStartedMs, + firstAgentMessageMs, + usage, + events, + stdout, + stderr, + }); + }); + }); +} + +export function summarizeArchitectureRuns(runs) { + const completed = runs.filter((run) => !run.error); + return { + runs: runs.length, + passed: completed.filter((run) => run.passed).length, + medianTotalMs: percentile(completed.map((run) => run.totalMs), 0.5), + p95TotalMs: percentile(completed.map((run) => run.totalMs), 0.95), + medianStartupMs: percentile(completed.map((run) => run.startupMs).filter(Number.isFinite), 0.5), + medianGenerationMs: percentile(completed.map((run) => run.generationMs).filter(Number.isFinite), 0.5), + medianInputTokens: percentile(completed.map((run) => run.usage?.input_tokens).filter(Number.isFinite), 0.5), + medianCachedInputTokens: percentile(completed.map((run) => run.usage?.cached_input_tokens).filter(Number.isFinite), 0.5), + medianOutputTokens: percentile(completed.map((run) => run.usage?.output_tokens).filter(Number.isFinite), 0.5), + }; +} + +function percentile(values, quantile) { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const index = (sorted.length - 1) * quantile; + const lower = Math.floor(index); + const upper = Math.ceil(index); + const value = lower === upper + ? sorted[lower] + : sorted[lower] + (sorted[upper] - sorted[lower]) * (index - lower); + return Math.round(value * 100) / 100; +} diff --git a/tests/live-codex-exec-benchmark.test.mjs b/tests/live-codex-exec-benchmark.test.mjs new file mode 100644 index 000000000..ac6e7e46a --- /dev/null +++ b/tests/live-codex-exec-benchmark.test.mjs @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; +import { describe, it } from 'node:test'; + +import { + runCodexExecBenchmark, + summarizeArchitectureRuns, +} from '../scripts/lib/codex-exec-benchmark.mjs'; + +describe('direct Codex architecture benchmark', () => { + it('records JSONL lifecycle and token events from codex exec', async () => { + const child = new EventEmitter(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = () => true; + const resultPromise = runCodexExecBenchmark({ + args: ['exec', '--json', 'work'], + spawnFactory: () => child, + }); + child.stdout.write('{"type":"thread.started","thread_id":"one"}\n'); + child.stdout.write('{"type":"turn.started"}\n'); + child.stdout.write('{"type":"item.completed","item":{"type":"agent_message","text":"done"}}\n'); + child.stdout.write('{"type":"turn.completed","usage":{"input_tokens":100,"cached_input_tokens":60,"output_tokens":20}}\n'); + child.emit('exit', 0, null); + const result = await resultPromise; + assert.equal(result.events.length, 4); + assert.equal(result.usage.input_tokens, 100); + assert.ok(result.threadStartedMs >= 0); + assert.ok(result.firstAgentMessageMs >= result.turnStartedMs); + }); + + it('summarizes startup, generation, total, quality, and token medians', () => { + const summary = summarizeArchitectureRuns([ + { passed: true, startupMs: 10, generationMs: 100, totalMs: 110, usage: { input_tokens: 1000, cached_input_tokens: 500, output_tokens: 100 } }, + { passed: false, startupMs: 20, generationMs: 200, totalMs: 220, usage: { input_tokens: 2000, cached_input_tokens: 1000, output_tokens: 200 } }, + ]); + assert.equal(summary.runs, 2); + assert.equal(summary.passed, 1); + assert.equal(summary.medianTotalMs, 165); + assert.equal(summary.medianInputTokens, 1500); + }); +});