Add Codex Live architecture benchmark

Compare direct Sol execution with cold and persistent app-server paths using identical full-task quality gates, lifecycle timings, and token metrics.\n\nAI-assisted: OpenAI Codex.

package.json

@@ -71,6 +71,7 @@
     "bench:live": "node scripts/benchmark-live.mjs",
     "bench:live:providers": "node scripts/benchmark-live-providers.mjs",
     "bench:live:codex-quality": "node scripts/benchmark-live-codex-worker.mjs",
+    "bench:live:codex-architecture": "node scripts/benchmark-live-codex-architecture.mjs",
     "audit": "bun audit --audit-level=moderate",
     "prepack": "cp README.md README.repo.md && cp README.npm.md README.md",
     "postpack": "cp README.repo.md README.md && rm README.repo.md",

scripts/benchmark-live-codex-architecture.mjs

@@ -0,0 +1,349 @@
+#!/usr/bin/env node
+
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { fileURLToPath } from 'node:url';
+
+import { anthropic } from '@ai-sdk/anthropic';
+import { generateText } from 'ai';
+
+import {
+  CodexAppServerClient,
+  selectQualityCodexModel,
+} from '../skill/scripts/live/codex-app-server-client.mjs';
+import {
+  buildCodexWorkerInstructions,
+  buildCodexWorkerTurnInputs,
+} from '../skill/scripts/live/codex-worker.mjs';
+import { runCodexExecBenchmark, summarizeArchitectureRuns } from './lib/codex-exec-benchmark.mjs';
+import { loadBenchmarkEnv } from './lib/live-provider-benchmark.mjs';
+import {
+  CODEX_QUALITY_OUTPUT_SCHEMA,
+  buildCodexQualityPrompt,
+  buildJudgePrompt,
+  createCodexQualityTasks,
+  parseJudgeResult,
+  scoreCodexQualityOutput,
+} from './lib/live-codex-quality-benchmark.mjs';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const args = parseArgs(process.argv.slice(2));
+const iterations = positiveInteger(args.iterations, 2);
+const timeoutMs = positiveInteger(args.timeout, 300_000);
+const outputPath = args.output ? path.resolve(ROOT, String(args.output)) : null;
+const profileIds = csv(args.profiles || 'direct-sol,cold-app-server,warm-app-server');
+const taskIds = csv(args.tasks || 'editorial-bolder,operations-polish');
+const judgeEnabled = args.judge !== false;
+const loadedEnv = loadBenchmarkEnv({ repoRoot: ROOT, explicitPath: args.envFile && path.resolve(args.envFile) });
+const skillPath = path.join(ROOT, '.agents', 'skills', 'impeccable', 'SKILL.md');
+const referenceDir = path.join(ROOT, 'skill', 'reference');
+const liveSpec = await readFile(path.join(referenceDir, 'live.md'), 'utf-8');
+const tasks = createCodexQualityTasks({ repoRoot: ROOT }).filter((task) => taskIds.includes(task.id));
+if (tasks.length !== taskIds.length) throw new Error('unknown task id in --tasks');
+for (const profile of profileIds) {
+  if (!['direct-sol', 'cold-app-server', 'warm-app-server'].includes(profile)) throw new Error(`unknown profile ${profile}`);
+}
+if (judgeEnabled && !process.env.ANTHROPIC_API_KEY && !args.dryRun) {
+  throw new Error('ANTHROPIC_API_KEY is required unless --no-judge is passed');
+}
+
+let model = args.model || null;
+if (!model && profileIds.some((profile) => profile.includes('app-server'))) {
+  const discovery = new CodexAppServerClient({ cwd: ROOT });
+  await discovery.connect();
+  try {
+    const selected = selectQualityCodexModel(await discovery.listModels());
+    model = selected?.model || selected?.id || null;
+  } finally {
+    await discovery.close();
+  }
+}
+model ||= 'gpt-5.6-sol';
+
+if (args.dryRun) {
+  await emit({
+    schemaVersion: 1,
+    mode: 'dry-run',
+    profiles: profileIds,
+    tasks: tasks.map((task) => task.id),
+    iterations,
+    model,
+    effort: 'medium',
+    judgeEnabled,
+    judgeAvailable: Boolean(process.env.ANTHROPIC_API_KEY),
+    envFilesLoaded: loadedEnv.length,
+    plannedModelRuns: profileIds.length * tasks.length * iterations,
+  });
+  process.exit(0);
+}
+
+const scratch = await mkdtemp(path.join(os.tmpdir(), 'impeccable-codex-architecture-'));
+const schemaPath = path.join(scratch, 'output-schema.json');
+await writeFile(schemaPath, JSON.stringify(CODEX_QUALITY_OUTPUT_SCHEMA));
+const runs = [];
+try {
+  for (const profile of profileIds) {
+    if (profile === 'warm-app-server') {
+      await runWarmProfile(profile);
+      continue;
+    }
+    for (let iteration = 1; iteration <= iterations; iteration += 1) {
+      for (const task of tasks) {
+        process.stderr.write(`[codex-architecture] ${profile} ${task.id} ${iteration}/${iterations}\n`);
+        runs.push(await (profile === 'direct-sol'
+          ? runDirect({ profile, task, iteration })
+          : runColdAppServer({ profile, task, iteration })));
+      }
+    }
+  }
+} finally {
+  await rm(scratch, { recursive: true, force: true });
+}
+
+const report = {
+  schemaVersion: 1,
+  generatedAt: new Date().toISOString(),
+  mode: 'live',
+  model,
+  effort: 'medium',
+  iterations,
+  tasks: tasks.map((task) => ({ id: task.id, action: task.action, brief: task.brief })),
+  profiles: profileIds.map((profile) => ({
+    id: profile,
+    summary: summarizeArchitectureRuns(runs.filter((run) => run.profile === profile)),
+  })),
+  judge: judgeEnabled ? { provider: 'anthropic', model: args.judgeModel || 'claude-sonnet-4-6' } : null,
+  runs,
+};
+await emit(report);
+process.exitCode = runs.every((run) => run.passed) ? 0 : 1;
+
+async function runDirect({ profile, task, iteration }) {
+  const outputFile = path.join(scratch, `${profile}-${task.id}-${iteration}.json`);
+  const actionReference = await readFile(path.join(referenceDir, `${task.action}.md`), 'utf-8');
+  const prompt = [
+    '$impeccable',
+    'Use the attached Impeccable skill. This automated benchmark already resolved Setup context below; do not rerun setup or edit files.',
+    buildCodexQualityPrompt(task, { actionReference, fullContext: true }),
+  ].join('\n\n');
+  try {
+    const result = await runCodexExecBenchmark({
+      cwd: ROOT,
+      timeoutMs,
+      args: [
+        'exec', '--ephemeral', '--ignore-user-config', '--dangerously-bypass-hook-trust',
+        '-C', ROOT, '-s', 'read-only', '-m', model,
+        '-c', 'model_reasoning_effort="medium"',
+        '--output-schema', schemaPath,
+        '--output-last-message', outputFile,
+        '--json', prompt,
+      ],
+    });
+    const output = JSON.parse(await readFile(outputFile, 'utf-8'));
+    return finishRun({
+      profile,
+      task,
+      iteration,
+      output,
+      startupMs: result.turnStartedMs,
+      generationMs: result.firstAgentMessageMs == null || result.turnStartedMs == null
+        ? result.durationMs
+        : result.firstAgentMessageMs - result.turnStartedMs,
+      totalMs: result.durationMs,
+      usage: result.usage,
+      transport: {
+        threadStartedMs: round(result.threadStartedMs),
+        turnStartedMs: round(result.turnStartedMs),
+        firstAgentMessageMs: round(result.firstAgentMessageMs),
+      },
+    });
+  } catch (error) {
+    return failedRun({ profile, task, iteration, error });
+  }
+}
+
+async function runColdAppServer({ profile, task, iteration }) {
+  const startedAt = performance.now();
+  const client = new CodexAppServerClient({ cwd: ROOT, turnTimeoutMs: timeoutMs });
+  let thread = null;
+  try {
+    await client.connect();
+    await client.listModels();
+    thread = await client.startDedicatedThread(threadParams(profile));
+    const startupMs = performance.now() - startedAt;
+    const turn = await runAppServerTurn(client, thread, task);
+    return finishRun({
+      profile,
+      task,
+      iteration,
+      output: turn.output,
+      startupMs,
+      generationMs: turn.durationMs,
+      totalMs: performance.now() - startedAt,
+      usage: normalizeAppServerUsage(turn.turn),
+    });
+  } catch (error) {
+    return failedRun({ profile, task, iteration, error });
+  } finally {
+    if (thread) await client.archiveThread(thread.id).catch(() => {});
+    await client.close().catch(() => {});
+  }
+}
+
+async function runWarmProfile(profile) {
+  const client = new CodexAppServerClient({ cwd: ROOT, turnTimeoutMs: timeoutMs });
+  let thread = null;
+  const startedAt = performance.now();
+  try {
+    await client.connect();
+    await client.listModels();
+    thread = await client.startDedicatedThread(threadParams(profile));
+    const coldStartupMs = performance.now() - startedAt;
+    for (let iteration = 1; iteration <= iterations; iteration += 1) {
+      for (const task of tasks) {
+        process.stderr.write(`[codex-architecture] ${profile} ${task.id} ${iteration}/${iterations}\n`);
+        const turnStartedAt = performance.now();
+        try {
+          const turn = await runAppServerTurn(client, thread, task);
+          runs.push(await finishRun({
+            profile,
+            task,
+            iteration,
+            output: turn.output,
+            startupMs: iteration === 1 && task === tasks[0] ? coldStartupMs : 0,
+            generationMs: turn.durationMs,
+            totalMs: performance.now() - turnStartedAt + (iteration === 1 && task === tasks[0] ? coldStartupMs : 0),
+            usage: normalizeAppServerUsage(turn.turn),
+            transport: { persistentThread: true, coldStartupMs: round(coldStartupMs) },
+          }));
+        } catch (error) {
+          runs.push(failedRun({ profile, task, iteration, error }));
+        }
+      }
+    }
+  } finally {
+    if (thread) await client.archiveThread(thread.id).catch(() => {});
+    await client.close().catch(() => {});
+  }
+}
+
+function threadParams(profile) {
+  return {
+    model,
+    cwd: ROOT,
+    approvalPolicy: 'never',
+    sandbox: 'read-only',
+    ephemeral: profile === 'cold-app-server',
+    serviceName: `impeccable_live_architecture_${profile}`,
+    baseInstructions: buildCodexWorkerInstructions(liveSpec),
+  };
+}
+
+async function runAppServerTurn(client, thread, task) {
+  const actionReference = await readFile(path.join(referenceDir, `${task.action}.md`), 'utf-8');
+  const prompt = buildCodexQualityPrompt(task, { actionReference, fullContext: true });
+  const result = await client.startTurn({
+    threadId: thread.id,
+    input: buildCodexWorkerTurnInputs({ prompt, skillPath, screenshotPath: task.screenshotPath, cwd: ROOT }),
+    cwd: ROOT,
+    model,
+    effort: 'medium',
+    summary: 'none',
+    approvalPolicy: 'never',
+    sandboxPolicy: { type: 'readOnly' },
+    outputSchema: CODEX_QUALITY_OUTPUT_SCHEMA,
+  });
+  return { output: JSON.parse(result.message), durationMs: result.durationMs, turn: result };
+}
+
+async function finishRun({ profile, task, iteration, output, startupMs, generationMs, totalMs, usage, transport = null }) {
+  const deterministic = scoreCodexQualityOutput(task, output);
+  const judge = judgeEnabled ? await judgeOutput(task, output) : null;
+  return {
+    profile,
+    task: task.id,
+    iteration,
+    model,
+    effort: 'medium',
+    startupMs: round(startupMs),
+    generationMs: round(generationMs),
+    totalMs: round(totalMs),
+    usage,
+    transport,
+    deterministic,
+    judge,
+    passed: deterministic.passed && (!judge || judge.passed),
+    output,
+  };
+}
+
+function failedRun({ profile, task, iteration, error }) {
+  return {
+    profile,
+    task: task.id,
+    iteration,
+    model,
+    effort: 'medium',
+    error: String(error?.stack || error),
+    passed: false,
+  };
+}
+
+async function judgeOutput(task, output) {
+  const response = await generateText({
+    model: anthropic(args.judgeModel || 'claude-sonnet-4-6'),
+    system: 'Be strict, concrete, and independent. Return the requested JSON object only.',
+    prompt: buildJudgePrompt(task, output),
+    maxOutputTokens: 800,
+  });
+  return parseJudgeResult(response.text);
+}
+
+function normalizeAppServerUsage(turn) {
+  const usage = turn?.completed?.params?.turn?.usage || turn?.turn?.usage || null;
+  if (!usage) return null;
+  return {
+    input_tokens: usage.inputTokens ?? usage.input_tokens ?? null,
+    cached_input_tokens: usage.cachedInputTokens ?? usage.cached_input_tokens ?? null,
+    output_tokens: usage.outputTokens ?? usage.output_tokens ?? null,
+    reasoning_output_tokens: usage.reasoningOutputTokens ?? usage.reasoning_output_tokens ?? null,
+  };
+}
+
+async function emit(report) {
+  if (outputPath) {
+    await mkdir(path.dirname(outputPath), { recursive: true });
+    await writeFile(outputPath, JSON.stringify(report, null, 2) + '\n');
+  }
+  process.stdout.write(JSON.stringify(report, null, 2) + '\n');
+}
+
+function parseArgs(values) {
+  const result = {};
+  for (let index = 0; index < values.length; index += 1) {
+    const value = values[index];
+    if (value === '--dry-run') result.dryRun = true;
+    else if (value === '--no-judge') result.judge = false;
+    else if (value.startsWith('--')) {
+      const [rawKey, inline] = value.slice(2).split('=', 2);
+      const key = rawKey.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase());
+      result[key] = inline ?? values[++index];
+    }
+  }
+  return result;
+}
+
+function csv(value) {
+  return String(value).split(',').map((item) => item.trim()).filter(Boolean);
+}
+
+function positiveInteger(value, fallback) {
+  const parsed = Number.parseInt(value, 10);
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function round(value) {
+  return Number.isFinite(value) ? Math.round(value * 100) / 100 : null;
+}

scripts/lib/codex-exec-benchmark.mjs

@@ -0,0 +1,112 @@
+import { spawn } from 'node:child_process';
+import { performance } from 'node:perf_hooks';
+
+export function runCodexExecBenchmark({
+  command = 'codex',
+  args,
+  cwd = process.cwd(),
+  env = process.env,
+  timeoutMs = 300_000,
+  spawnFactory = spawn,
+} = {}) {
+  if (!Array.isArray(args) || args.length === 0) throw new TypeError('args are required');
+  const startedAt = performance.now();
+  return new Promise((resolve, reject) => {
+    const child = spawnFactory(command, args, {
+      cwd,
+      env,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    let buffer = '';
+    let threadStartedMs = null;
+    let turnStartedMs = null;
+    let firstAgentMessageMs = null;
+    let usage = null;
+    const events = [];
+    const timer = setTimeout(() => {
+      child.kill('SIGTERM');
+      reject(new Error(`codex exec timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+    timer.unref?.();
+
+    const consumeLine = (line) => {
+      if (!line.trim()) return;
+      let event;
+      try { event = JSON.parse(line); } catch { return; }
+      events.push(event);
+      const elapsed = performance.now() - startedAt;
+      if (event.type === 'thread.started' && threadStartedMs == null) threadStartedMs = elapsed;
+      if (event.type === 'turn.started' && turnStartedMs == null) turnStartedMs = elapsed;
+      if (event.type === 'item.completed' && event.item?.type === 'agent_message' && firstAgentMessageMs == null) {
+        firstAgentMessageMs = elapsed;
+      }
+      if (event.type === 'turn.completed') usage = event.usage || null;
+    };
+
+    child.stdout.on('data', (chunk) => {
+      const text = String(chunk);
+      stdout += text;
+      buffer += text;
+      let newline;
+      while ((newline = buffer.indexOf('\n')) !== -1) {
+        consumeLine(buffer.slice(0, newline));
+        buffer = buffer.slice(newline + 1);
+      }
+    });
+    child.stderr.on('data', (chunk) => { stderr += String(chunk); });
+    child.once('error', (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+    child.once('exit', (code, signal) => {
+      clearTimeout(timer);
+      if (buffer) consumeLine(buffer);
+      const durationMs = performance.now() - startedAt;
+      if (code !== 0) {
+        const error = new Error(`codex exec failed (${code ?? signal}): ${stderr || stdout}`);
+        error.code = code;
+        reject(error);
+        return;
+      }
+      resolve({
+        durationMs,
+        threadStartedMs,
+        turnStartedMs,
+        firstAgentMessageMs,
+        usage,
+        events,
+        stdout,
+        stderr,
+      });
+    });
+  });
+}
+
+export function summarizeArchitectureRuns(runs) {
+  const completed = runs.filter((run) => !run.error);
+  return {
+    runs: runs.length,
+    passed: completed.filter((run) => run.passed).length,
+    medianTotalMs: percentile(completed.map((run) => run.totalMs), 0.5),
+    p95TotalMs: percentile(completed.map((run) => run.totalMs), 0.95),
+    medianStartupMs: percentile(completed.map((run) => run.startupMs).filter(Number.isFinite), 0.5),
+    medianGenerationMs: percentile(completed.map((run) => run.generationMs).filter(Number.isFinite), 0.5),
+    medianInputTokens: percentile(completed.map((run) => run.usage?.input_tokens).filter(Number.isFinite), 0.5),
+    medianCachedInputTokens: percentile(completed.map((run) => run.usage?.cached_input_tokens).filter(Number.isFinite), 0.5),
+    medianOutputTokens: percentile(completed.map((run) => run.usage?.output_tokens).filter(Number.isFinite), 0.5),
+  };
+}
+
+function percentile(values, quantile) {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = (sorted.length - 1) * quantile;
+  const lower = Math.floor(index);
+  const upper = Math.ceil(index);
+  const value = lower === upper
+    ? sorted[lower]
+    : sorted[lower] + (sorted[upper] - sorted[lower]) * (index - lower);
+  return Math.round(value * 100) / 100;
+}

tests/live-codex-exec-benchmark.test.mjs

@@ -0,0 +1,43 @@
+import assert from 'node:assert/strict';
+import { EventEmitter } from 'node:events';
+import { PassThrough } from 'node:stream';
+import { describe, it } from 'node:test';
+
+import {
+  runCodexExecBenchmark,
+  summarizeArchitectureRuns,
+} from '../scripts/lib/codex-exec-benchmark.mjs';
+
+describe('direct Codex architecture benchmark', () => {
+  it('records JSONL lifecycle and token events from codex exec', async () => {
+    const child = new EventEmitter();
+    child.stdout = new PassThrough();
+    child.stderr = new PassThrough();
+    child.kill = () => true;
+    const resultPromise = runCodexExecBenchmark({
+      args: ['exec', '--json', 'work'],
+      spawnFactory: () => child,
+    });
+    child.stdout.write('{"type":"thread.started","thread_id":"one"}\n');
+    child.stdout.write('{"type":"turn.started"}\n');
+    child.stdout.write('{"type":"item.completed","item":{"type":"agent_message","text":"done"}}\n');
+    child.stdout.write('{"type":"turn.completed","usage":{"input_tokens":100,"cached_input_tokens":60,"output_tokens":20}}\n');
+    child.emit('exit', 0, null);
+    const result = await resultPromise;
+    assert.equal(result.events.length, 4);
+    assert.equal(result.usage.input_tokens, 100);
+    assert.ok(result.threadStartedMs >= 0);
+    assert.ok(result.firstAgentMessageMs >= result.turnStartedMs);
+  });
+
+  it('summarizes startup, generation, total, quality, and token medians', () => {
+    const summary = summarizeArchitectureRuns([
+      { passed: true, startupMs: 10, generationMs: 100, totalMs: 110, usage: { input_tokens: 1000, cached_input_tokens: 500, output_tokens: 100 } },
+      { passed: false, startupMs: 20, generationMs: 200, totalMs: 220, usage: { input_tokens: 2000, cached_input_tokens: 1000, output_tokens: 200 } },
+    ]);
+    assert.equal(summary.runs, 2);
+    assert.equal(summary.passed, 1);
+    assert.equal(summary.medianTotalMs, 165);
+    assert.equal(summary.medianInputTokens, 1500);
+  });
+});