import { performance } from 'node:perf_hooks'; import { basename, join, resolve } from 'node:path'; const METRIC_KEYS = [ 'browserPreparationMs', 'browserDispatchMs', 'automationClickMs', 'serverPickupMs', 'goToAgentMs', 'serverPreflightMs', 'scaffoldMs', 'generationToFirstMs', 'generationMs', 'firstVariantWriteMs', 'writeMs', 'writeToFirstVariantMs', 'replyMs', 'goToFirstVariantMs', 'goToSecondVariantMs', 'goToAllVariantsMs', 'firstToSecondGapMs', 'secondToAllGapMs', 'deliveryGapMs', 'impeccableOverheadMs', 'workerPickupToSourceReadyMs', 'workerFirstGenerationToReviewableMs', 'workerFirstValidationToReviewableMs', 'workerSecondGenerationToReviewableMs', 'workerSecondValidationToReviewableMs', 'workerRemainingGenerationToReadyMs', 'workerRemainingValidationToReadyMs', 'acceptToResetMs', 'acceptToNextGoDispatchMs', 'acceptToNextFirstVariantMs', ]; export function parseLiveBenchmarkArgs(argv) { const out = {}; for (const arg of argv) { if (!arg.startsWith('--')) continue; const body = arg.slice(2); const index = body.indexOf('='); const rawKey = index === -1 ? body : body.slice(0, index); const key = rawKey.replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); out[key] = index === -1 ? true : body.slice(index + 1); } return out; } /** * Resolve the benchmark's fixture and output paths without coupling private * evaluation fixtures to this repository. `--evidence-bundle` is deliberately * rubric-free: it packages screenshots and timings for an external evaluator * without embedding a quality judge or secret task corpus in the public repo. */ export function resolveLiveBenchmarkPaths(args, { root, fixturesDir }) { const evidenceRoot = args.evidenceBundle ? resolve(root, String(args.evidenceBundle)) : null; if (evidenceRoot && args.artifacts) { throw new Error('--evidence-bundle replaces --artifacts'); } if (evidenceRoot && args.output) { throw new Error('--evidence-bundle writes report.json itself; omit --output'); } if (evidenceRoot && args.append) { throw new Error('--evidence-bundle represents one portable run; omit --append'); } const explicitFixtureDir = args.fixtureDir ? resolve(root, String(args.fixtureDir)) : null; const fixtureName = String(args.fixture || (explicitFixtureDir ? basename(explicitFixtureDir) : 'vite8-react-plain')); const fixtureDir = explicitFixtureDir || join(fixturesDir, fixtureName); return { fixtureName, fixtureDir, fixtureOrigin: explicitFixtureDir ? 'external' : 'repository', evidenceRoot, artifactRoot: evidenceRoot || (args.artifacts ? resolve(root, String(args.artifacts)) : null), outputPath: evidenceRoot ? join(evidenceRoot, 'report.json') : args.output ? resolve(root, String(args.output)) : null, }; } export function createTraceRecorder(now = () => performance.now()) { const events = []; return { events, trace(name, data = {}) { events.push({ name, at: now(), ...data }); }, mark(name, data = {}) { const event = { name, at: now(), ...data }; events.push(event); return event; }, }; } export function durationBetween(events, startName, endName, predicate = () => true) { const start = events.find((event) => event.name === startName && predicate(event)); const end = events.find((event) => event.name === endName && predicate(event) && (!start || event.at >= start.at)); if (!start || !end) return null; return roundMs(Math.max(0, end.at - start.at)); } /** * Assemble the two model calls used by the Live benchmark's progressive path. * The first checkpoint is already visible in the browser, so both its markup * and CSS are immutable. The tail call may supply deferred params for variant * 1, but its CSS must contain only independently-scoped rules for variants 2+. */ export function assembleSplitProgressiveOutput(first, remaining) { const firstVariant = first?.variants?.[0]; if (!firstVariant) throw new Error('progressive assembly requires a first variant'); if (!Array.isArray(remaining?.variants) || remaining.variants.length < 1) { throw new Error('progressive assembly requires a complete remaining variant set'); } const firstCss = String(first.scopedCss || ''); const laterCss = String(remaining.scopedCss || ''); assertLaterVariantCss(laterCss); return { scopedCss: firstCss && laterCss ? `${firstCss}\n${laterCss}` : firstCss || laterCss, variants: [ { ...firstVariant, params: Array.isArray(remaining.variants[0]?.params) ? remaining.variants[0].params : [], }, ...remaining.variants.slice(1), ], }; } export function buildInteractionRun(events, { iteration, scenario, goStartedAt, browserTiming = null }) { const received = events.find((event) => event.name === 'agent.event.received' && event.type === 'generate' && event.at >= goStartedAt ); if (!received?.id) throw new Error(`iteration ${iteration}: no generate event was traced`); const id = received.id; const forId = (event) => event.id === id; const eventPost = events.find((event) => event.name === 'browser.generate_post' && forId(event)); const mark = (name) => events.find((event) => event.name === name && event.iteration === iteration); const first = mark('browser.first_variant'); const second = mark('browser.second_variant'); const all = mark('browser.all_variants'); const writeEnd = events.find((event) => event.name === 'agent.write.end' && forId(event)); const firstWriteEnd = events.find((event) => event.name === 'agent.first_variant.write.end' && forId(event)); const reusedScaffold = events.find((event) => event.name === 'agent.scaffold.reused' && forId(event)); const generationMs = durationBetween(events, 'agent.generate.start', 'agent.generate.end', forId); const generationToFirstMs = durationBetween(events, 'agent.generate.start', 'agent.generate.first_ready', forId); const browserPreparationMs = eventPost ? roundMs(eventPost.at - goStartedAt) : null; const browserDispatchMs = Number.isFinite(browserTiming?.goAt) && Number.isFinite(browserTiming?.generateAt) ? roundMs(Math.max(0, browserTiming.generateAt - browserTiming.goAt)) : null; const interactionStartedAt = eventPost && browserDispatchMs != null ? eventPost.at - browserDispatchMs : goStartedAt; const measuredGoToFirstVariantMs = first ? roundMs(first.at - interactionStartedAt) : null; const measuredGoToSecondVariantMs = second ? roundMs(second.at - interactionStartedAt) : null; const measuredGoToAllVariantsMs = all ? roundMs(all.at - interactionStartedAt) : null; return { iteration, scenario, eventId: id, selectionEvidence: { tagName: eventPost?.selectedTagName || null, classes: Array.isArray(eventPost?.selectedClasses) ? eventPost.selectedClasses : [], }, annotationEvidence: { screenshotPath: eventPost?.hasScreenshotPath === true, comments: Number(eventPost?.commentCount || 0), strokes: Number(eventPost?.strokeCount || 0), }, browserPreparationMs, browserDispatchMs, automationClickMs: browserPreparationMs == null || browserDispatchMs == null ? null : roundMs(Math.max(0, browserPreparationMs - browserDispatchMs)), serverPickupMs: eventPost ? roundMs(Math.max(0, received.at - eventPost.at)) : null, goToAgentMs: roundMs(received.at - interactionStartedAt), serverPreflightMs: Number.isFinite(reusedScaffold?.durationMs) ? roundMs(reusedScaffold.durationMs) : null, scaffoldMs: durationBetween(events, 'agent.scaffold.start', 'agent.scaffold.end', forId), generationToFirstMs, generationMs, firstVariantWriteMs: durationBetween(events, 'agent.first_variant.write.start', 'agent.first_variant.write.end', forId), writeMs: durationBetween(events, 'agent.write.start', 'agent.write.end', forId), writeToFirstVariantMs: first && (firstWriteEnd || writeEnd) ? roundMs(Math.max(0, first.at - (firstWriteEnd || writeEnd).at)) : null, replyMs: durationBetween(events, 'agent.reply.start', 'agent.reply.end', forId), goToFirstVariantMs: measuredGoToFirstVariantMs, goToSecondVariantMs: measuredGoToSecondVariantMs, goToAllVariantsMs: measuredGoToAllVariantsMs, firstToSecondGapMs: first && second ? roundMs(Math.max(0, second.at - first.at)) : null, secondToAllGapMs: second && all ? roundMs(Math.max(0, all.at - second.at)) : null, deliveryGapMs: first && all ? roundMs(Math.max(0, all.at - first.at)) : null, impeccableOverheadMs: measuredGoToFirstVariantMs == null || generationToFirstMs == null ? null : roundMs(Math.max(0, measuredGoToFirstVariantMs - generationToFirstMs)), }; } export function deriveJournalGenerationMetrics(snapshot = {}) { const timings = snapshot.generationTimings || {}; const at = (phase) => Number(timings[phase]?.at); const timingErrors = []; const delta = (start, end) => { if (!Number.isFinite(at(start)) || !Number.isFinite(at(end))) return null; if (at(end) < at(start)) { timingErrors.push(`${end}_before_${start}`); return null; } return roundMs(at(end) - at(start)); }; return { workerPickupToSourceReadyMs: delta('picked_up', 'source_ready'), workerFirstGenerationToReviewableMs: delta('first_variant_generating', 'first_reviewable'), workerFirstValidationToReviewableMs: delta('first_variant_validating', 'first_reviewable'), workerSecondGenerationToReviewableMs: delta('second_variant_generating', 'second_reviewable'), workerSecondValidationToReviewableMs: delta('second_variant_validating', 'second_reviewable'), workerRemainingGenerationToReadyMs: delta('remaining_variants_generating', 'all_variants_ready'), workerRemainingValidationToReadyMs: delta('remaining_variants_validating', 'all_variants_ready'), journalTimingErrors: timingErrors, journalGenerationTimings: timings, }; } export function summarizeRuns(runs) { const metrics = {}; for (const key of METRIC_KEYS) { const values = runs.map((run) => run[key]).filter(Number.isFinite).sort((a, b) => a - b); if (values.length === 0) continue; metrics[key] = { median: roundMs(percentile(values, 0.5)), p95: roundMs(percentile(values, 0.95)), min: roundMs(values[0]), max: roundMs(values[values.length - 1]), }; } return { count: runs.length, metrics }; } export function summarizeSetup(events) { const stages = [ ['dependencies', 'setup.install.start', 'setup.install.end'], ['liveServer', 'setup.live_server.start', 'setup.live_server.end'], ['codexWorker', 'setup.worker.start', 'setup.worker.end'], ['injection', 'setup.inject.start', 'setup.inject.end'], ['devServer', 'setup.dev_server.start', 'setup.dev_server.end'], ['pageLoad', 'setup.page_load.start', 'setup.page_load.end'], ['handshake', 'setup.handshake.start', 'setup.handshake.end'], ]; return Object.fromEntries(stages.map(([key, start, end]) => [key, durationBetween(events, start, end)])); } export function createBenchmarkReport({ fixture, agent, provider, model, scenario, runs, events, harnessProbe = null, delivery = 'atomic', promptMode = null, simulation = null, generatedAt = new Date().toISOString(), }) { return { schemaVersion: 1, generatedAt, benchmark: { fixture, agent, provider: provider || null, model: model || null, scenario, variants: 3, delivery, promptMode, simulation, }, setup: summarizeSetup(events), summary: summarizeRuns(runs), runs, harnessProbe, }; } export function mergeBenchmarkReports(reports, generatedAt = new Date().toISOString()) { return { schemaVersion: 1, generatedAt, reports, }; } export function compareModelBackedReports(atomic, progressive, { medianTarget = 0.35, p95Target = 0.25, minimumRuns = 3, } = {}) { assertComparableModelReport(atomic, 'atomic', minimumRuns); assertComparableModelReport(progressive, 'progressive', minimumRuns); for (const key of ['fixture', 'provider', 'model', 'scenario', 'variants', 'promptMode']) { if (atomic.benchmark[key] !== progressive.benchmark[key]) { throw new Error(`benchmark mismatch for ${key}: atomic=${atomic.benchmark[key]} progressive=${progressive.benchmark[key]}`); } } const atomicFirst = requiredMetric(atomic, 'goToFirstVariantMs'); const progressiveFirst = requiredMetric(progressive, 'goToFirstVariantMs'); const medianImprovement = improvement(atomicFirst.median, progressiveFirst.median); const p95Improvement = improvement(atomicFirst.p95, progressiveFirst.p95); const allReady = { atomic: requiredMetric(atomic, 'goToAllVariantsMs'), progressive: requiredMetric(progressive, 'goToAllVariantsMs'), }; const passed = medianImprovement >= medianTarget && p95Improvement >= p95Target; return { passed, target: { medianImprovement, p95Improvement, medianTarget, p95Target }, firstReviewable: { atomic: atomicFirst, progressive: progressiveFirst }, allVariantsReady: allReady, benchmark: { fixture: atomic.benchmark.fixture, provider: atomic.benchmark.provider, model: atomic.benchmark.model, scenario: atomic.benchmark.scenario, runs: { atomic: atomic.summary.count, progressive: progressive.summary.count }, }, }; } function assertComparableModelReport(report, delivery, minimumRuns) { if (!report?.benchmark || !report?.summary) throw new Error(`${delivery} benchmark report is missing metadata or summary`); if (report.benchmark.agent !== 'llm') throw new Error(`${delivery} benchmark must be model-backed (agent=llm)`); if (report.benchmark.delivery !== delivery) { throw new Error(`expected ${delivery} delivery report, got ${report.benchmark.delivery || 'unknown'}`); } if (report.benchmark.simulation) throw new Error(`${delivery} model benchmark must not contain simulated latency`); if (!report.benchmark.provider || !report.benchmark.model) throw new Error(`${delivery} benchmark is missing provider/model identity`); if (!Number.isInteger(report.summary.count) || report.summary.count < minimumRuns) { throw new Error(`${delivery} benchmark requires at least ${minimumRuns} runs`); } } function requiredMetric(report, key) { const metric = report.summary.metrics?.[key]; if (!Number.isFinite(metric?.median) || !Number.isFinite(metric?.p95)) { throw new Error(`${report.benchmark.delivery} benchmark is missing ${key} median/p95`); } return { median: metric.median, p95: metric.p95 }; } function improvement(baseline, candidate) { if (!(baseline > 0) || !Number.isFinite(candidate)) throw new Error('benchmark latency must be finite and baseline must be positive'); return Number((1 - (candidate / baseline)).toFixed(4)); } function assertLaterVariantCss(css) { if (!css.trim()) return; for (const prelude of topLevelCssPreludes(css)) { const variants = [...prelude.matchAll(/\[data-impeccable-variant\s*=\s*(["'])(\d+)\1[^\]]*\]/g)] .map((match) => Number(match[2])); if (variants.includes(1)) { throw new Error('progressive tail CSS must not repeat or conflict with published variant 1 CSS'); } if (variants.length === 0 || variants.some((variant) => variant < 2)) { throw new Error('progressive tail CSS must be attributable only to variants 2+'); } if (new Set(variants).size !== 1) { throw new Error('each progressive tail CSS block must target exactly one later variant'); } } } function topLevelCssPreludes(css) { const preludes = []; let cursor = 0; while (cursor < css.length) { while (cursor < css.length && /\s/.test(css[cursor])) cursor += 1; if (cursor >= css.length) break; const start = cursor; const open = findCssToken(css, cursor, '{'); if (open === -1) throw new Error('progressive tail CSS contains a rule without a block'); const prelude = css.slice(start, open).trim(); if (!prelude || prelude.includes(';')) { throw new Error('progressive tail CSS must contain scoped rule blocks only'); } preludes.push(prelude); const close = findMatchingCssBrace(css, open); if (close === -1) throw new Error('progressive tail CSS has unbalanced braces'); cursor = close + 1; } return preludes; } function findCssToken(css, start, token) { let quote = null; let comment = false; for (let index = start; index < css.length; index += 1) { const char = css[index]; const next = css[index + 1]; if (comment) { if (char === '*' && next === '/') { comment = false; index += 1; } continue; } if (!quote && char === '/' && next === '*') { comment = true; index += 1; continue; } if (quote) { if (char === '\\') index += 1; else if (char === quote) quote = null; continue; } if (char === '"' || char === "'") { quote = char; continue; } if (char === token) return index; } return -1; } function findMatchingCssBrace(css, open) { let depth = 0; let quote = null; let comment = false; for (let index = open; index < css.length; index += 1) { const char = css[index]; const next = css[index + 1]; if (comment) { if (char === '*' && next === '/') { comment = false; index += 1; } continue; } if (!quote && char === '/' && next === '*') { comment = true; index += 1; continue; } if (quote) { if (char === '\\') index += 1; else if (char === quote) quote = null; continue; } if (char === '"' || char === "'") { quote = char; continue; } if (char === '{') depth += 1; if (char === '}') { depth -= 1; if (depth === 0) return index; } } return -1; } function percentile(sortedValues, ratio) { if (sortedValues.length === 1) return sortedValues[0]; const index = (sortedValues.length - 1) * ratio; const lower = Math.floor(index); const upper = Math.ceil(index); if (lower === upper) return sortedValues[lower]; const weight = index - lower; return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight; } function roundMs(value) { if (!Number.isFinite(value)) return null; return Number(value.toFixed(2)); }