From 10183bd91cba67b6fd535a619689739b6f636f56 Mon Sep 17 00:00:00 2001 From: Paul Bakaus Date: Sun, 12 Jul 2026 20:52:34 -0700 Subject: [PATCH] Measure production Live worker phases Trace worker pickup and derive generation, validation, and publication latency from the durable Live session journal.\n\nAI-assisted: OpenAI Codex. --- scripts/benchmark-live.mjs | 29 +++++++++++++++++++++++++++-- scripts/lib/live-benchmark.mjs | 23 +++++++++++++++++++++++ tests/live-benchmark.test.mjs | 21 +++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/scripts/benchmark-live.mjs b/scripts/benchmark-live.mjs index e2a1164b4..5413e84d1 100644 --- a/scripts/benchmark-live.mjs +++ b/scripts/benchmark-live.mjs @@ -22,6 +22,7 @@ import { assembleSplitProgressiveOutput, createBenchmarkReport, createTraceRecorder, + deriveJournalGenerationMetrics, mergeBenchmarkReports, } from './lib/live-benchmark.mjs'; @@ -108,6 +109,7 @@ try { goStartedAt: goStarted.at, browserTiming, }); + Object.assign(run, deriveJournalGenerationMetrics(await readGenerationSnapshot(session.tmp, run.eventId))); assertScenarioEvidence(run, scenario); runs.push(run); @@ -152,6 +154,11 @@ try { await browser.close().catch(() => {}); } +async function readGenerationSnapshot(tmp, eventId) { + const file = join(tmp, '.impeccable', 'live', 'sessions', `${eventId}.snapshot.json`); + try { return JSON.parse(await readFile(file, 'utf-8')); } catch { return {}; } +} + async function resolveAgent(mode, options) { if (mode === 'fake') return { agent: createFakeAgent(), provider: 'deterministic', model: null, promptMode: null }; if (mode === 'codex') { @@ -178,7 +185,7 @@ async function resolveAgent(mode, options) { return { agent, provider: config.provider, model: config.model, promptMode: 'synthetic-element-contract' }; } -async function startCodexProductionWorker({ tmp, scriptsDir, log }, options) { +async function startCodexProductionWorker({ tmp, scriptsDir, log, trace }, options) { const script = join(scriptsDir, 'live-codex-worker.mjs'); const statePath = join(tmp, '.impeccable', 'live', 'codex-worker.json'); const child = spawn(process.execPath, [script, '--foreground'], { @@ -203,17 +210,35 @@ async function startCodexProductionWorker({ tmp, scriptsDir, log }, options) { child.stderr.on('data', capture); const done = new Promise((resolve) => child.once('exit', (code, signal) => resolve({ code, signal }))); const state = await waitForWorkerState(statePath, child, output, positiveInt(options.workerTimeoutMs, 20_000)); - return { + const handle = { child, state, done, async stop() { + clearInterval(handle.monitor); if (child.exitCode != null || child.signalCode != null) return; child.kill('SIGTERM'); await Promise.race([done, new Promise((resolve) => setTimeout(resolve, 5_000))]); if (child.exitCode == null && child.signalCode == null) child.kill('SIGKILL'); }, }; + const tracedEvents = new Set(); + let readingState = false; + handle.monitor = setInterval(async () => { + if (readingState) return; + readingState = true; + try { + const next = JSON.parse(await readFile(statePath, 'utf-8')); + handle.state = next; + if (next.status === 'working' && next.eventId && !tracedEvents.has(next.eventId)) { + tracedEvents.add(next.eventId); + trace('agent.event.received', { id: next.eventId, type: 'generate', owner: next.owner }); + } + } catch { /* state replacement is atomic but teardown may remove the fixture */ } + finally { readingState = false; } + }, 40); + handle.monitor.unref(); + return handle; } async function waitForWorkerState(statePath, child, output, timeoutMs) { diff --git a/scripts/lib/live-benchmark.mjs b/scripts/lib/live-benchmark.mjs index 7f3b7911a..99766553a 100644 --- a/scripts/lib/live-benchmark.mjs +++ b/scripts/lib/live-benchmark.mjs @@ -18,6 +18,11 @@ const METRIC_KEYS = [ 'goToAllVariantsMs', 'deliveryGapMs', 'impeccableOverheadMs', + 'workerPickupToSourceReadyMs', + 'workerFirstGenerationToReviewableMs', + 'workerFirstValidationToReviewableMs', + 'workerRemainingGenerationToReadyMs', + 'workerRemainingValidationToReadyMs', ]; export function createTraceRecorder(now = () => performance.now()) { @@ -137,6 +142,24 @@ export function buildInteractionRun(events, { iteration, scenario, goStartedAt, }; } +export function deriveJournalGenerationMetrics(snapshot = {}) { + const timings = snapshot.generationTimings || {}; + const at = (phase) => Number(timings[phase]?.at); + const delta = (start, end) => ( + Number.isFinite(at(start)) && Number.isFinite(at(end)) + ? roundMs(Math.max(0, at(end) - at(start))) + : null + ); + return { + workerPickupToSourceReadyMs: delta('picked_up', 'source_ready'), + workerFirstGenerationToReviewableMs: delta('first_variant_generating', 'first_reviewable'), + workerFirstValidationToReviewableMs: delta('first_variant_validating', 'first_reviewable'), + workerRemainingGenerationToReadyMs: delta('remaining_variants_generating', 'all_variants_ready'), + workerRemainingValidationToReadyMs: delta('remaining_variants_validating', 'all_variants_ready'), + journalGenerationTimings: timings, + }; +} + export function summarizeRuns(runs) { const metrics = {}; for (const key of METRIC_KEYS) { diff --git a/tests/live-benchmark.test.mjs b/tests/live-benchmark.test.mjs index 10dd4fc4e..bde33c5c4 100644 --- a/tests/live-benchmark.test.mjs +++ b/tests/live-benchmark.test.mjs @@ -6,11 +6,32 @@ import { buildInteractionRun, compareModelBackedReports, createTraceRecorder, + deriveJournalGenerationMetrics, durationBetween, summarizeRuns, } from '../scripts/lib/live-benchmark.mjs'; describe('live benchmark metrics', () => { + it('derives production worker phases from the durable session journal', () => { + const metrics = deriveJournalGenerationMetrics({ + generationTimings: { + picked_up: { at: 100 }, + source_ready: { at: 124 }, + first_variant_generating: { at: 130 }, + first_variant_validating: { at: 210 }, + first_reviewable: { at: 240 }, + remaining_variants_generating: { at: 245 }, + remaining_variants_validating: { at: 400 }, + all_variants_ready: { at: 430 }, + }, + }); + assert.equal(metrics.workerPickupToSourceReadyMs, 24); + assert.equal(metrics.workerFirstGenerationToReviewableMs, 110); + assert.equal(metrics.workerFirstValidationToReviewableMs, 30); + assert.equal(metrics.workerRemainingGenerationToReadyMs, 185); + assert.equal(metrics.workerRemainingValidationToReadyMs, 30); + }); + it('keeps published progressive CSS byte-stable and carries deferred params', () => { const firstCss = '@scope ([data-impeccable-variant="1"]) { .offer { color: red; } }'; const laterCss = [