diff --git a/package.json b/package.json
index 6a4708502..6a1a3cf7f 100644
--- a/package.json
+++ b/package.json
@@ -68,7 +68,6 @@
"smoke:hooks": "node scripts/smoke-provider-hooks.mjs",
"bench:detector": "node scripts/benchmark-detector.mjs",
"bench:detector:browser": "node scripts/benchmark-detector.mjs --browser",
- "bench:live": "node scripts/benchmark-live.mjs",
"audit": "bun audit --audit-level=moderate",
"prepack": "cp README.md README.repo.md && cp README.npm.md README.md",
"postpack": "cp README.repo.md README.md && rm README.repo.md",
diff --git a/scripts/benchmark-live.mjs b/scripts/benchmark-live.mjs
deleted file mode 100644
index 126626363..000000000
--- a/scripts/benchmark-live.mjs
+++ /dev/null
@@ -1,284 +0,0 @@
-#!/usr/bin/env node
-
-import { mkdir, readFile, writeFile } from 'node:fs/promises';
-import { dirname, join, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-import { createFakeAgent } from '../tests/live-e2e/agent.mjs';
-import { createLlmAgent, resolveLlmAgentConfig } from '../tests/live-e2e/agents/llm-agent.mjs';
-import { bootFixtureSession, FIXTURES_DIR } from '../tests/live-e2e/session.mjs';
-import {
- clickDiscard,
- clickGo,
- drawAnnotationPinAndStroke,
- pickElement,
- waitForCycling,
- waitForHandshake,
-} from '../tests/live-e2e/ui.mjs';
-import { boolFlag, parseArgs, positiveIntFlag, resolveEnum } from './lib/cli-args.mjs';
-import {
- buildInteractionRun,
- assembleSplitProgressiveOutput,
- createBenchmarkReport,
- createTraceRecorder,
- mergeBenchmarkReports,
-} from './lib/live-benchmark.mjs';
-
-const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
-const args = parseArgs(process.argv.slice(2));
-const fixtureName = String(args.fixture || 'vite8-react-plain');
-const iterations = positiveIntFlag(args.iterations, 5);
-const agentMode = resolveEnum(args.agent, ['fake', 'llm'], 'fake', '--agent');
-const scenario = resolveEnum(args.scenario, ['plain', 'annotated'], 'plain', '--scenario');
-const delivery = resolveEnum(args.delivery, ['atomic', 'progressive'], 'atomic', '--delivery');
-const simulatedTailMs = positiveIntFlag(args.simulatedTailMs, 0);
-const quiet = boolFlag(args.quiet);
-const outputPath = args.output ? resolve(ROOT, String(args.output)) : null;
-const fixture = JSON.parse(await readFile(join(FIXTURES_DIR, fixtureName, 'fixture.json'), 'utf-8'));
-if (!fixture.runtime) throw new Error(`fixture ${fixtureName} has no runtime configuration`);
-if (fixture.runtime.mode === 'insert') throw new Error('live benchmark currently measures replace-mode fixtures only');
-
-const { chromium } = await import('playwright');
-const browser = await chromium.launch({ headless: !boolFlag(args.headed) });
-const recorder = createTraceRecorder();
-let session;
-
-try {
- const agentInfo = await resolveAgent(agentMode, args);
- if (delivery === 'progressive' && agentMode === 'llm') {
- agentInfo.agent = createSplitProgressiveAgent(agentInfo.agent);
- }
- session = await bootFixtureSession({
- name: fixtureName,
- fixture,
- browser,
- agent: agentInfo.agent,
- wrapTarget: wrapTargetFromPickedElement,
- trace: recorder.trace,
- progressive: delivery === 'progressive',
- progressiveDelayMs: delivery === 'progressive' ? simulatedTailMs : 0,
- atomicDelayMs: delivery === 'atomic' ? simulatedTailMs : 0,
- log: quiet ? () => {} : (message) => process.stderr.write(`[live-bench] ${message}\n`),
- });
-
- recorder.mark('setup.handshake.start');
- session.page.on('request', (request) => {
- if (!request.url().endsWith('/events') || request.method() !== 'POST') return;
- let payload;
- try { payload = request.postDataJSON(); } catch { return; }
- if (payload?.type === 'generate' && payload.id) {
- recorder.mark('browser.generate_post', {
- id: payload.id,
- hasScreenshotPath: typeof payload.screenshotPath === 'string' && payload.screenshotPath.length > 0,
- commentCount: Array.isArray(payload.comments) ? payload.comments.length : 0,
- strokeCount: Array.isArray(payload.strokes) ? payload.strokes.length : 0,
- });
- }
- });
- await waitForHandshake(session.page);
- recorder.mark('setup.handshake.end');
- await installBrowserTimingProbe(session.page);
-
- const runs = [];
- const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title';
- for (let iteration = 1; iteration <= iterations; iteration += 1) {
- await pickElement(session.page, pickSelector, { resetPickMode: iteration > 1 });
- if (scenario === 'annotated') {
- await drawAnnotationPinAndStroke(session.page, { comment: 'Benchmark annotation' });
- }
- await resetBrowserTimingProbe(session.page, iteration);
-
- const goStarted = recorder.mark('ui.go.start', { iteration, scenario });
- const firstVariant = waitForFirstVariant(session.page).then(() => {
- recorder.mark('browser.first_variant', { iteration, scenario });
- });
-
- await clickGo(session.page);
- recorder.mark('ui.generating_visible', { iteration, scenario });
- await firstVariant;
- await waitForCycling(session.page, 3, { timeout: agentMode === 'llm' ? 150_000 : 30_000 });
- recorder.mark('browser.all_variants', { iteration, scenario });
- const browserTiming = await readBrowserTimingProbe(session.page);
-
- const run = buildInteractionRun(recorder.events, {
- iteration,
- scenario,
- goStartedAt: goStarted.at,
- browserTiming,
- });
- assertScenarioEvidence(run, scenario);
- runs.push(run);
-
- if (!quiet) process.stderr.write(formatRun(runs.at(-1)) + '\n');
- await clickDiscard(session.page);
- await waitForReset(session.page);
- }
-
- const report = createBenchmarkReport({
- fixture: fixtureName,
- agent: agentMode,
- provider: agentInfo.provider,
- model: agentInfo.model,
- scenario,
- runs,
- events: recorder.events,
- harnessProbe: args.harnessProbe || null,
- delivery,
- promptMode: agentInfo.promptMode,
- simulation: simulatedTailMs > 0 ? { remainingGenerationMs: simulatedTailMs } : null,
- });
-
- let output = report;
- if (outputPath && args.append) {
- try {
- const existing = JSON.parse(await readFile(outputPath, 'utf-8'));
- const previousReports = Array.isArray(existing.reports) ? existing.reports : [existing];
- output = mergeBenchmarkReports([...previousReports, report]);
- } catch (error) {
- if (error.code !== 'ENOENT') throw error;
- }
- }
- const json = JSON.stringify(output, null, 2) + '\n';
- if (outputPath) {
- await mkdir(dirname(outputPath), { recursive: true });
- await writeFile(outputPath, json, 'utf-8');
- process.stderr.write(`[live-bench] wrote ${outputPath}\n`);
- }
- process.stdout.write(json);
-} finally {
- if (session) await session.teardown();
- await browser.close().catch(() => {});
-}
-
-async function resolveAgent(mode, options) {
- if (mode === 'fake') return { agent: createFakeAgent(), provider: 'deterministic', model: null, promptMode: null };
- const config = resolveLlmAgentConfig({
- provider: options.provider,
- model: options.model,
- });
- const agent = await createLlmAgent({
- config,
- includeLiveSpec: false,
- log: (message) => process.stderr.write(`[live-bench:llm] ${message}\n`),
- });
- if (!agent) {
- throw new Error(`LLM benchmark provider=${config.provider} requires ${config.requiredEnv}. Pass it in the environment; .env files are not read implicitly.`);
- }
- return { agent, provider: config.provider, model: config.model, promptMode: 'synthetic-element-contract' };
-}
-
-function createSplitProgressiveAgent(agent) {
- const firstBySession = new Map();
- return {
- ...agent,
- async generateFirstVariant(event, context) {
- const first = await agent.generateVariants({
- ...event,
- count: 1,
- progressive: { phase: 'first', totalCount: event.count },
- }, context);
- firstBySession.set(event.id, first);
- return first;
- },
- async generateRemainingVariants(event, context) {
- const first = firstBySession.get(event.id) || context.firstOutput;
- const remaining = await agent.generateVariants({
- ...event,
- count: event.count,
- progressive: {
- phase: 'remaining',
- totalCount: event.count,
- firstVariant: first?.variants?.[0] || null,
- omitFirstVariantCss: true,
- },
- }, context);
- firstBySession.delete(event.id);
- return assembleSplitProgressiveOutput(first, remaining);
- },
- };
-}
-
-async function waitForFirstVariant(page) {
- const handle = await page.waitForFunction(() => {
- const wrappers = [...document.querySelectorAll('[data-impeccable-variant]')];
- return wrappers.some((element) => element.getAttribute('data-impeccable-variant') !== 'original');
- }, undefined, { timeout: 150_000 });
- await handle.dispose();
-}
-
-async function waitForReset(page) {
- await page.waitForFunction(() => !document.querySelector('[data-impeccable-variants]'), undefined, { timeout: 30_000 });
- await page.waitForTimeout(100);
-}
-
-async function installBrowserTimingProbe(page) {
- await page.evaluate(() => {
- const state = { iteration: 0, goAt: null, generateAt: null };
- window.__IMPECCABLE_LIVE_BENCH_TIMING__ = state;
- const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.()
- || window.__IMPECCABLE_LIVE_UI_ROOT__
- || document;
- root.addEventListener('click', (event) => {
- const button = event.composedPath().find((node) =>
- node?.getAttribute?.('aria-label') === 'Generate variants'
- );
- if (button) state.goAt = performance.now();
- }, true);
-
- const originalFetch = window.fetch.bind(window);
- window.fetch = (input, init) => {
- try {
- const url = typeof input === 'string' ? input : input?.url;
- if (String(url || '').endsWith('/events') && init?.method === 'POST') {
- const payload = typeof init.body === 'string' ? JSON.parse(init.body) : null;
- if (payload?.type === 'generate') state.generateAt = performance.now();
- }
- } catch { /* measurement must never affect Live */ }
- return originalFetch(input, init);
- };
- });
-}
-
-async function resetBrowserTimingProbe(page, iteration) {
- await page.evaluate((nextIteration) => {
- const state = window.__IMPECCABLE_LIVE_BENCH_TIMING__;
- if (!state) return;
- state.iteration = nextIteration;
- state.goAt = null;
- state.generateAt = null;
- }, iteration);
-}
-
-async function readBrowserTimingProbe(page) {
- return page.evaluate(() => {
- const state = window.__IMPECCABLE_LIVE_BENCH_TIMING__;
- return state ? { ...state } : null;
- });
-}
-
-function assertScenarioEvidence(run, currentScenario) {
- const evidence = run.annotationEvidence;
- if (currentScenario === 'annotated') {
- if (!evidence?.screenshotPath || evidence.comments < 1 || evidence.strokes < 1) {
- throw new Error(`iteration ${run.iteration}: annotated generate payload lost screenshot/comments/strokes`);
- }
- return;
- }
- if (evidence?.screenshotPath) {
- throw new Error(`iteration ${run.iteration}: plain generate payload unexpectedly included screenshotPath`);
- }
-}
-
-function wrapTargetFromPickedElement(event) {
- const element = event.element || {};
- return {
- elementId: element.id || undefined,
- classes: Array.isArray(element.classes) ? element.classes.join(',') : undefined,
- tag: element.tagName ? String(element.tagName).toLowerCase() : undefined,
- text: element.textContent ? String(element.textContent).trim() : undefined,
- };
-}
-
-function formatRun(run) {
- return `[live-bench] run ${run.iteration}: first=${run.goToFirstVariantMs}ms all=${run.goToAllVariantsMs}ms generation=${run.generationMs}ms overhead=${run.impeccableOverheadMs}ms`;
-}
diff --git a/scripts/lib/live-benchmark.mjs b/scripts/lib/live-benchmark.mjs
deleted file mode 100644
index 7f3b7911a..000000000
--- a/scripts/lib/live-benchmark.mjs
+++ /dev/null
@@ -1,395 +0,0 @@
-import { performance } from 'node:perf_hooks';
-
-const METRIC_KEYS = [
- 'browserPreparationMs',
- 'browserDispatchMs',
- 'automationClickMs',
- 'serverPickupMs',
- 'goToAgentMs',
- 'serverPreflightMs',
- 'scaffoldMs',
- 'generationToFirstMs',
- 'generationMs',
- 'firstVariantWriteMs',
- 'writeMs',
- 'writeToFirstVariantMs',
- 'replyMs',
- 'goToFirstVariantMs',
- 'goToAllVariantsMs',
- 'deliveryGapMs',
- 'impeccableOverheadMs',
-];
-
-export function createTraceRecorder(now = () => performance.now()) {
- const events = [];
- return {
- events,
- trace(name, data = {}) {
- events.push({ name, at: now(), ...data });
- },
- mark(name, data = {}) {
- const event = { name, at: now(), ...data };
- events.push(event);
- return event;
- },
- };
-}
-
-export function durationBetween(events, startName, endName, predicate = () => true) {
- const start = events.find((event) => event.name === startName && predicate(event));
- const end = events.find((event) => event.name === endName && predicate(event) && (!start || event.at >= start.at));
- if (!start || !end) return null;
- return roundMs(Math.max(0, end.at - start.at));
-}
-
-/**
- * Assemble the two model calls used by the Live benchmark's progressive path.
- * The first checkpoint is already visible in the browser, so both its markup
- * and CSS are immutable. The tail call may supply deferred params for variant
- * 1, but its CSS must contain only independently-scoped rules for variants 2+.
- */
-export function assembleSplitProgressiveOutput(first, remaining) {
- const firstVariant = first?.variants?.[0];
- if (!firstVariant) throw new Error('progressive assembly requires a first variant');
- if (!Array.isArray(remaining?.variants) || remaining.variants.length < 1) {
- throw new Error('progressive assembly requires a complete remaining variant set');
- }
-
- const firstCss = String(first.scopedCss || '');
- const laterCss = String(remaining.scopedCss || '');
- assertLaterVariantCss(laterCss);
-
- return {
- scopedCss: firstCss && laterCss ? `${firstCss}\n${laterCss}` : firstCss || laterCss,
- variants: [
- {
- ...firstVariant,
- params: Array.isArray(remaining.variants[0]?.params)
- ? remaining.variants[0].params
- : [],
- },
- ...remaining.variants.slice(1),
- ],
- };
-}
-
-export function buildInteractionRun(events, { iteration, scenario, goStartedAt, browserTiming = null }) {
- const received = events.find((event) =>
- event.name === 'agent.event.received'
- && event.type === 'generate'
- && event.at >= goStartedAt
- );
- if (!received?.id) throw new Error(`iteration ${iteration}: no generate event was traced`);
-
- const id = received.id;
- const forId = (event) => event.id === id;
- const eventPost = events.find((event) => event.name === 'browser.generate_post' && forId(event));
- const mark = (name) => events.find((event) => event.name === name && event.iteration === iteration);
- const first = mark('browser.first_variant');
- const all = mark('browser.all_variants');
- const writeEnd = events.find((event) => event.name === 'agent.write.end' && forId(event));
- const firstWriteEnd = events.find((event) => event.name === 'agent.first_variant.write.end' && forId(event));
- const reusedScaffold = events.find((event) => event.name === 'agent.scaffold.reused' && forId(event));
- const generationMs = durationBetween(events, 'agent.generate.start', 'agent.generate.end', forId);
- const generationToFirstMs = durationBetween(events, 'agent.generate.start', 'agent.generate.first_ready', forId);
- const browserPreparationMs = eventPost ? roundMs(eventPost.at - goStartedAt) : null;
- const browserDispatchMs = Number.isFinite(browserTiming?.goAt) && Number.isFinite(browserTiming?.generateAt)
- ? roundMs(Math.max(0, browserTiming.generateAt - browserTiming.goAt))
- : null;
- const interactionStartedAt = eventPost && browserDispatchMs != null
- ? eventPost.at - browserDispatchMs
- : goStartedAt;
- const measuredGoToFirstVariantMs = first ? roundMs(first.at - interactionStartedAt) : null;
- const measuredGoToAllVariantsMs = all ? roundMs(all.at - interactionStartedAt) : null;
-
- return {
- iteration,
- scenario,
- eventId: id,
- annotationEvidence: {
- screenshotPath: eventPost?.hasScreenshotPath === true,
- comments: Number(eventPost?.commentCount || 0),
- strokes: Number(eventPost?.strokeCount || 0),
- },
- browserPreparationMs,
- browserDispatchMs,
- automationClickMs: browserPreparationMs == null || browserDispatchMs == null
- ? null
- : roundMs(Math.max(0, browserPreparationMs - browserDispatchMs)),
- serverPickupMs: eventPost ? roundMs(Math.max(0, received.at - eventPost.at)) : null,
- goToAgentMs: roundMs(received.at - interactionStartedAt),
- serverPreflightMs: Number.isFinite(reusedScaffold?.durationMs) ? roundMs(reusedScaffold.durationMs) : null,
- scaffoldMs: durationBetween(events, 'agent.scaffold.start', 'agent.scaffold.end', forId),
- generationToFirstMs,
- generationMs,
- firstVariantWriteMs: durationBetween(events, 'agent.first_variant.write.start', 'agent.first_variant.write.end', forId),
- writeMs: durationBetween(events, 'agent.write.start', 'agent.write.end', forId),
- writeToFirstVariantMs: first && (firstWriteEnd || writeEnd)
- ? roundMs(Math.max(0, first.at - (firstWriteEnd || writeEnd).at))
- : null,
- replyMs: durationBetween(events, 'agent.reply.start', 'agent.reply.end', forId),
- goToFirstVariantMs: measuredGoToFirstVariantMs,
- goToAllVariantsMs: measuredGoToAllVariantsMs,
- deliveryGapMs: first && all ? roundMs(Math.max(0, all.at - first.at)) : null,
- impeccableOverheadMs: measuredGoToFirstVariantMs == null || generationToFirstMs == null
- ? null
- : roundMs(Math.max(0, measuredGoToFirstVariantMs - generationToFirstMs)),
- };
-}
-
-export function summarizeRuns(runs) {
- const metrics = {};
- for (const key of METRIC_KEYS) {
- const values = runs.map((run) => run[key]).filter(Number.isFinite).sort((a, b) => a - b);
- if (values.length === 0) continue;
- metrics[key] = {
- median: roundMs(percentile(values, 0.5)),
- p95: roundMs(percentile(values, 0.95)),
- min: roundMs(values[0]),
- max: roundMs(values[values.length - 1]),
- };
- }
- return { count: runs.length, metrics };
-}
-
-export function summarizeSetup(events) {
- const stages = [
- ['dependencies', 'setup.install.start', 'setup.install.end'],
- ['liveServer', 'setup.live_server.start', 'setup.live_server.end'],
- ['injection', 'setup.inject.start', 'setup.inject.end'],
- ['devServer', 'setup.dev_server.start', 'setup.dev_server.end'],
- ['pageLoad', 'setup.page_load.start', 'setup.page_load.end'],
- ['handshake', 'setup.handshake.start', 'setup.handshake.end'],
- ];
- return Object.fromEntries(stages.map(([key, start, end]) => [key, durationBetween(events, start, end)]));
-}
-
-export function createBenchmarkReport({
- fixture,
- agent,
- provider,
- model,
- scenario,
- runs,
- events,
- harnessProbe = null,
- delivery = 'atomic',
- promptMode = null,
- simulation = null,
- generatedAt = new Date().toISOString(),
-}) {
- return {
- schemaVersion: 1,
- generatedAt,
- benchmark: {
- fixture,
- agent,
- provider: provider || null,
- model: model || null,
- scenario,
- variants: 3,
- delivery,
- promptMode,
- simulation,
- },
- setup: summarizeSetup(events),
- summary: summarizeRuns(runs),
- runs,
- harnessProbe,
- };
-}
-
-export function mergeBenchmarkReports(reports, generatedAt = new Date().toISOString()) {
- return {
- schemaVersion: 1,
- generatedAt,
- reports,
- };
-}
-
-export function compareModelBackedReports(atomic, progressive, {
- medianTarget = 0.35,
- p95Target = 0.25,
- minimumRuns = 3,
-} = {}) {
- assertComparableModelReport(atomic, 'atomic', minimumRuns);
- assertComparableModelReport(progressive, 'progressive', minimumRuns);
-
- for (const key of ['fixture', 'provider', 'model', 'scenario', 'variants', 'promptMode']) {
- if (atomic.benchmark[key] !== progressive.benchmark[key]) {
- throw new Error(`benchmark mismatch for ${key}: atomic=${atomic.benchmark[key]} progressive=${progressive.benchmark[key]}`);
- }
- }
-
- const atomicFirst = requiredMetric(atomic, 'goToFirstVariantMs');
- const progressiveFirst = requiredMetric(progressive, 'goToFirstVariantMs');
- const medianImprovement = improvement(atomicFirst.median, progressiveFirst.median);
- const p95Improvement = improvement(atomicFirst.p95, progressiveFirst.p95);
- const allReady = {
- atomic: requiredMetric(atomic, 'goToAllVariantsMs'),
- progressive: requiredMetric(progressive, 'goToAllVariantsMs'),
- };
- const passed = medianImprovement >= medianTarget && p95Improvement >= p95Target;
-
- return {
- passed,
- target: { medianImprovement, p95Improvement, medianTarget, p95Target },
- firstReviewable: { atomic: atomicFirst, progressive: progressiveFirst },
- allVariantsReady: allReady,
- benchmark: {
- fixture: atomic.benchmark.fixture,
- provider: atomic.benchmark.provider,
- model: atomic.benchmark.model,
- scenario: atomic.benchmark.scenario,
- runs: { atomic: atomic.summary.count, progressive: progressive.summary.count },
- },
- };
-}
-
-function assertComparableModelReport(report, delivery, minimumRuns) {
- if (!report?.benchmark || !report?.summary) throw new Error(`${delivery} benchmark report is missing metadata or summary`);
- if (report.benchmark.agent !== 'llm') throw new Error(`${delivery} benchmark must be model-backed (agent=llm)`);
- if (report.benchmark.delivery !== delivery) {
- throw new Error(`expected ${delivery} delivery report, got ${report.benchmark.delivery || 'unknown'}`);
- }
- if (report.benchmark.simulation) throw new Error(`${delivery} model benchmark must not contain simulated latency`);
- if (!report.benchmark.provider || !report.benchmark.model) throw new Error(`${delivery} benchmark is missing provider/model identity`);
- if (!Number.isInteger(report.summary.count) || report.summary.count < minimumRuns) {
- throw new Error(`${delivery} benchmark requires at least ${minimumRuns} runs`);
- }
-}
-
-function requiredMetric(report, key) {
- const metric = report.summary.metrics?.[key];
- if (!Number.isFinite(metric?.median) || !Number.isFinite(metric?.p95)) {
- throw new Error(`${report.benchmark.delivery} benchmark is missing ${key} median/p95`);
- }
- return { median: metric.median, p95: metric.p95 };
-}
-
-function improvement(baseline, candidate) {
- if (!(baseline > 0) || !Number.isFinite(candidate)) throw new Error('benchmark latency must be finite and baseline must be positive');
- return Number((1 - (candidate / baseline)).toFixed(4));
-}
-
-function assertLaterVariantCss(css) {
- if (!css.trim()) return;
- for (const prelude of topLevelCssPreludes(css)) {
- const variants = [...prelude.matchAll(/\[data-impeccable-variant\s*=\s*(["'])(\d+)\1[^\]]*\]/g)]
- .map((match) => Number(match[2]));
- if (variants.includes(1)) {
- throw new Error('progressive tail CSS must not repeat or conflict with published variant 1 CSS');
- }
- if (variants.length === 0 || variants.some((variant) => variant < 2)) {
- throw new Error('progressive tail CSS must be attributable only to variants 2+');
- }
- if (new Set(variants).size !== 1) {
- throw new Error('each progressive tail CSS block must target exactly one later variant');
- }
- }
-}
-
-function topLevelCssPreludes(css) {
- const preludes = [];
- let cursor = 0;
- while (cursor < css.length) {
- while (cursor < css.length && /\s/.test(css[cursor])) cursor += 1;
- if (cursor >= css.length) break;
- const start = cursor;
- const open = findCssToken(css, cursor, '{');
- if (open === -1) throw new Error('progressive tail CSS contains a rule without a block');
- const prelude = css.slice(start, open).trim();
- if (!prelude || prelude.includes(';')) {
- throw new Error('progressive tail CSS must contain scoped rule blocks only');
- }
- preludes.push(prelude);
- const close = findMatchingCssBrace(css, open);
- if (close === -1) throw new Error('progressive tail CSS has unbalanced braces');
- cursor = close + 1;
- }
- return preludes;
-}
-
-function findCssToken(css, start, token) {
- let quote = null;
- let comment = false;
- for (let index = start; index < css.length; index += 1) {
- const char = css[index];
- const next = css[index + 1];
- if (comment) {
- if (char === '*' && next === '/') {
- comment = false;
- index += 1;
- }
- continue;
- }
- if (!quote && char === '/' && next === '*') {
- comment = true;
- index += 1;
- continue;
- }
- if (quote) {
- if (char === '\\') index += 1;
- else if (char === quote) quote = null;
- continue;
- }
- if (char === '"' || char === "'") {
- quote = char;
- continue;
- }
- if (char === token) return index;
- }
- return -1;
-}
-
-function findMatchingCssBrace(css, open) {
- let depth = 0;
- let quote = null;
- let comment = false;
- for (let index = open; index < css.length; index += 1) {
- const char = css[index];
- const next = css[index + 1];
- if (comment) {
- if (char === '*' && next === '/') {
- comment = false;
- index += 1;
- }
- continue;
- }
- if (!quote && char === '/' && next === '*') {
- comment = true;
- index += 1;
- continue;
- }
- if (quote) {
- if (char === '\\') index += 1;
- else if (char === quote) quote = null;
- continue;
- }
- if (char === '"' || char === "'") {
- quote = char;
- continue;
- }
- if (char === '{') depth += 1;
- if (char === '}') {
- depth -= 1;
- if (depth === 0) return index;
- }
- }
- return -1;
-}
-
-function percentile(sortedValues, ratio) {
- if (sortedValues.length === 1) return sortedValues[0];
- const index = (sortedValues.length - 1) * ratio;
- const lower = Math.floor(index);
- const upper = Math.ceil(index);
- if (lower === upper) return sortedValues[lower];
- const weight = index - lower;
- return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight;
-}
-
-function roundMs(value) {
- if (!Number.isFinite(value)) return null;
- return Number(value.toFixed(2));
-}
diff --git a/scripts/test-suites.mjs b/scripts/test-suites.mjs
index 72e8d6284..181d09ccc 100644
--- a/scripts/test-suites.mjs
+++ b/scripts/test-suites.mjs
@@ -125,7 +125,6 @@ export const SUITES = {
'tests/live-browser-regression.test.mjs',
'tests/live-browser-session.test.mjs',
'tests/live-browser-source.test.mjs',
- 'tests/live-benchmark.test.mjs',
'tests/live-commit-manual-edits.test.mjs',
'tests/live-completion.test.mjs',
'tests/live-copy-edit-agent.test.mjs',
diff --git a/skill/scripts/live-wrap.mjs b/skill/scripts/live-wrap.mjs
index 4e18b70db..895217538 100644
--- a/skill/scripts/live-wrap.mjs
+++ b/skill/scripts/live-wrap.mjs
@@ -230,7 +230,6 @@ The agent should insert variant HTML at insertLine.`);
// Strip only the COMMON minimum leading whitespace across the picked lines;
// `deindentContent` on the accept side already mirrors this convention.
let originalLines = lines.slice(startLine, endLine + 1);
- const sourceOriginalLines = [...originalLines];
// Buffer-aware "original" content: if the user has pending manual edits for
// this page whose originalText appears in the picked source range, apply
diff --git a/tests/live-benchmark.test.mjs b/tests/live-benchmark.test.mjs
deleted file mode 100644
index 10dd4fc4e..000000000
--- a/tests/live-benchmark.test.mjs
+++ /dev/null
@@ -1,165 +0,0 @@
-import assert from 'node:assert/strict';
-import { describe, it } from 'node:test';
-
-import {
- assembleSplitProgressiveOutput,
- buildInteractionRun,
- compareModelBackedReports,
- createTraceRecorder,
- durationBetween,
- summarizeRuns,
-} from '../scripts/lib/live-benchmark.mjs';
-
-describe('live benchmark metrics', () => {
- it('keeps published progressive CSS byte-stable and carries deferred params', () => {
- const firstCss = '@scope ([data-impeccable-variant="1"]) { .offer { color: red; } }';
- const laterCss = [
- '@scope ([data-impeccable-variant="2"]) { .offer { color: green; } }',
- '@scope ([data-impeccable-variant="3"]) { .offer { color: blue; } }',
- ].join('\n');
- const firstVariant = { innerHtml: 'One', params: [] };
- const deferredParams = [{ name: 'density', type: 'range', min: 0, max: 1, default: 0.5 }];
- const assembled = assembleSplitProgressiveOutput(
- { scopedCss: firstCss, variants: [firstVariant] },
- {
- scopedCss: laterCss,
- variants: [
- { innerHtml: firstVariant.innerHtml, params: deferredParams },
- { innerHtml: 'Two', params: [] },
- { innerHtml: 'Three', params: [] },
- ],
- },
- );
-
- assert.equal(assembled.scopedCss, `${firstCss}\n${laterCss}`);
- assert.equal(assembled.scopedCss.slice(0, firstCss.length), firstCss);
- assert.equal(assembled.variants[0].innerHtml, firstVariant.innerHtml);
- assert.equal(assembled.variants[0].params, deferredParams);
- });
-
- it('rejects tail CSS that would reproduce published_variant_css_changed', () => {
- const first = {
- scopedCss: '@scope ([data-impeccable-variant="1"]) { .offer { color: red; } }',
- variants: [{ innerHtml: 'One', params: [] }],
- };
- const conflictingTail = {
- scopedCss: [
- '@scope ([data-impeccable-variant="1"]) { .offer { color: purple; } }',
- '@scope ([data-impeccable-variant="2"]) { .offer { color: green; } }',
- ].join('\n'),
- variants: [
- { innerHtml: first.variants[0].innerHtml, params: [] },
- { innerHtml: 'Two', params: [] },
- ],
- };
-
- assert.throws(
- () => assembleSplitProgressiveOutput(first, conflictingTail),
- /must not repeat or conflict with published variant 1 CSS/,
- );
- });
-
- it('separates model generation from Impeccable overhead', () => {
- const events = [
- { name: 'ui.go.start', at: 100, iteration: 1 },
- { name: 'browser.generate_post', at: 108, id: 'abc', hasScreenshotPath: false, commentCount: 0, strokeCount: 0 },
- { name: 'agent.event.received', at: 110, id: 'abc', type: 'generate' },
- { name: 'agent.scaffold.start', at: 112, id: 'abc' },
- { name: 'agent.scaffold.end', at: 132, id: 'abc' },
- { name: 'agent.generate.start', at: 132, id: 'abc' },
- { name: 'agent.generate.first_ready', at: 1132, id: 'abc' },
- { name: 'agent.generate.end', at: 1132, id: 'abc' },
- { name: 'agent.write.start', at: 1132, id: 'abc' },
- { name: 'agent.write.end', at: 1142, id: 'abc' },
- { name: 'agent.reply.start', at: 1142, id: 'abc' },
- { name: 'agent.reply.end', at: 1147, id: 'abc' },
- { name: 'browser.first_variant', at: 1200, iteration: 1 },
- { name: 'browser.all_variants', at: 1200, iteration: 1 },
- ];
-
- const run = buildInteractionRun(events, {
- iteration: 1,
- scenario: 'plain',
- goStartedAt: 100,
- browserTiming: { goAt: 50, generateAt: 52.5 },
- });
- assert.equal(run.goToFirstVariantMs, 1094.5);
- assert.equal(run.browserPreparationMs, 8);
- assert.equal(run.browserDispatchMs, 2.5);
- assert.equal(run.automationClickMs, 5.5);
- assert.deepEqual(run.annotationEvidence, { screenshotPath: false, comments: 0, strokes: 0 });
- assert.equal(run.serverPickupMs, 2);
- assert.equal(run.generationMs, 1000);
- assert.equal(run.impeccableOverheadMs, 94.5);
- assert.equal(run.deliveryGapMs, 0);
- assert.equal(run.scaffoldMs, 20);
- });
-
- it('reports interpolated medians and p95 values', () => {
- const summary = summarizeRuns([
- { goToFirstVariantMs: 100, generationMs: 70 },
- { goToFirstVariantMs: 200, generationMs: 140 },
- { goToFirstVariantMs: 300, generationMs: 210 },
- ]);
- assert.equal(summary.metrics.goToFirstVariantMs.median, 200);
- assert.equal(summary.metrics.goToFirstVariantMs.p95, 290);
- });
-
- it('records monotonic trace events and returns null for missing boundaries', () => {
- let now = 0;
- const recorder = createTraceRecorder(() => ++now);
- recorder.trace('start');
- recorder.trace('end');
- assert.equal(durationBetween(recorder.events, 'start', 'end'), 1);
- assert.equal(durationBetween(recorder.events, 'missing', 'end'), null);
- });
-
- it('proves model-backed first-reviewable thresholds with comparable reports', () => {
- const atomic = modelReport('atomic', 1000, 1200, 1400, 1500);
- const progressive = modelReport('progressive', 500, 700, 1450, 1550);
- const comparison = compareModelBackedReports(atomic, progressive);
- assert.equal(comparison.passed, true);
- assert.equal(comparison.target.medianImprovement, 0.5);
- assert.equal(comparison.target.p95Improvement, 0.4167);
- });
-
- it('rejects fake, simulated, and mismatched model reports', () => {
- const atomic = modelReport('atomic', 1000, 1200, 1400, 1500);
- const progressive = modelReport('progressive', 500, 700, 1450, 1550);
- assert.throws(
- () => compareModelBackedReports({ ...atomic, benchmark: { ...atomic.benchmark, agent: 'fake' } }, progressive),
- /model-backed/,
- );
- assert.throws(
- () => compareModelBackedReports(atomic, { ...progressive, benchmark: { ...progressive.benchmark, simulation: { remainingGenerationMs: 1 } } }),
- /simulated latency/,
- );
- assert.throws(
- () => compareModelBackedReports(atomic, { ...progressive, benchmark: { ...progressive.benchmark, model: 'other-model' } }),
- /benchmark mismatch for model/,
- );
- });
-});
-
-function modelReport(delivery, firstMedian, firstP95, allMedian, allP95) {
- return {
- benchmark: {
- fixture: 'vite8-react-plain',
- agent: 'llm',
- provider: 'anthropic',
- model: 'claude-haiku-4-5',
- scenario: 'plain',
- variants: 3,
- delivery,
- promptMode: 'synthetic-element-contract',
- simulation: null,
- },
- summary: {
- count: 5,
- metrics: {
- goToFirstVariantMs: { median: firstMedian, p95: firstP95 },
- goToAllVariantsMs: { median: allMedian, p95: allP95 },
- },
- },
- };
-}