From e0afd4fcea355bdb8b711b3e7a8fbcbad6249a30 Mon Sep 17 00:00:00 2001 From: Paul Bakaus Date: Sat, 18 Jul 2026 14:11:07 -0700 Subject: [PATCH] Drop the progressive benchmark, remove dead wrap scaffolding Review fallout from removing progressive publication. The Live benchmark existed to compare atomic against progressive delivery: compareModelBackedReports measures goToFirstVariantMs improvement of one over the other. With progressive gone it measures nothing against nothing. Worse, benchmark-live.mjs still passed `progressive` to bootFixtureSession, which no longer accepts it, so `--delivery progressive` was silently ignored and would have emitted reports labeled progressive that actually ran atomic. Silent wrong data is worse than a crash. It was built for progressive, so it goes with progressive: benchmark-live.mjs, its lib, its test, and the bench:live script. If an atomic latency baseline is wanted later, that is a smaller thing built on purpose. live-wrap.mjs: sourceOriginalLines was assigned and never read. Both found by review bots on #381 (Copilot). Assisted-by: Claude Code --- package.json | 1 - scripts/benchmark-live.mjs | 284 ------------------------ scripts/lib/live-benchmark.mjs | 395 --------------------------------- scripts/test-suites.mjs | 1 - skill/scripts/live-wrap.mjs | 1 - tests/live-benchmark.test.mjs | 165 -------------- 6 files changed, 847 deletions(-) delete mode 100644 scripts/benchmark-live.mjs delete mode 100644 scripts/lib/live-benchmark.mjs delete mode 100644 tests/live-benchmark.test.mjs diff --git a/package.json b/package.json index 6a4708502..6a1a3cf7f 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,6 @@ "smoke:hooks": "node scripts/smoke-provider-hooks.mjs", "bench:detector": "node scripts/benchmark-detector.mjs", "bench:detector:browser": "node scripts/benchmark-detector.mjs --browser", - "bench:live": "node scripts/benchmark-live.mjs", "audit": "bun audit --audit-level=moderate", "prepack": "cp README.md README.repo.md && cp README.npm.md README.md", "postpack": "cp README.repo.md README.md && rm README.repo.md", diff --git a/scripts/benchmark-live.mjs b/scripts/benchmark-live.mjs deleted file mode 100644 index 126626363..000000000 --- a/scripts/benchmark-live.mjs +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env node - -import { mkdir, readFile, writeFile } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -import { createFakeAgent } from '../tests/live-e2e/agent.mjs'; -import { createLlmAgent, resolveLlmAgentConfig } from '../tests/live-e2e/agents/llm-agent.mjs'; -import { bootFixtureSession, FIXTURES_DIR } from '../tests/live-e2e/session.mjs'; -import { - clickDiscard, - clickGo, - drawAnnotationPinAndStroke, - pickElement, - waitForCycling, - waitForHandshake, -} from '../tests/live-e2e/ui.mjs'; -import { boolFlag, parseArgs, positiveIntFlag, resolveEnum } from './lib/cli-args.mjs'; -import { - buildInteractionRun, - assembleSplitProgressiveOutput, - createBenchmarkReport, - createTraceRecorder, - mergeBenchmarkReports, -} from './lib/live-benchmark.mjs'; - -const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); -const args = parseArgs(process.argv.slice(2)); -const fixtureName = String(args.fixture || 'vite8-react-plain'); -const iterations = positiveIntFlag(args.iterations, 5); -const agentMode = resolveEnum(args.agent, ['fake', 'llm'], 'fake', '--agent'); -const scenario = resolveEnum(args.scenario, ['plain', 'annotated'], 'plain', '--scenario'); -const delivery = resolveEnum(args.delivery, ['atomic', 'progressive'], 'atomic', '--delivery'); -const simulatedTailMs = positiveIntFlag(args.simulatedTailMs, 0); -const quiet = boolFlag(args.quiet); -const outputPath = args.output ? resolve(ROOT, String(args.output)) : null; -const fixture = JSON.parse(await readFile(join(FIXTURES_DIR, fixtureName, 'fixture.json'), 'utf-8')); -if (!fixture.runtime) throw new Error(`fixture ${fixtureName} has no runtime configuration`); -if (fixture.runtime.mode === 'insert') throw new Error('live benchmark currently measures replace-mode fixtures only'); - -const { chromium } = await import('playwright'); -const browser = await chromium.launch({ headless: !boolFlag(args.headed) }); -const recorder = createTraceRecorder(); -let session; - -try { - const agentInfo = await resolveAgent(agentMode, args); - if (delivery === 'progressive' && agentMode === 'llm') { - agentInfo.agent = createSplitProgressiveAgent(agentInfo.agent); - } - session = await bootFixtureSession({ - name: fixtureName, - fixture, - browser, - agent: agentInfo.agent, - wrapTarget: wrapTargetFromPickedElement, - trace: recorder.trace, - progressive: delivery === 'progressive', - progressiveDelayMs: delivery === 'progressive' ? simulatedTailMs : 0, - atomicDelayMs: delivery === 'atomic' ? simulatedTailMs : 0, - log: quiet ? () => {} : (message) => process.stderr.write(`[live-bench] ${message}\n`), - }); - - recorder.mark('setup.handshake.start'); - session.page.on('request', (request) => { - if (!request.url().endsWith('/events') || request.method() !== 'POST') return; - let payload; - try { payload = request.postDataJSON(); } catch { return; } - if (payload?.type === 'generate' && payload.id) { - recorder.mark('browser.generate_post', { - id: payload.id, - hasScreenshotPath: typeof payload.screenshotPath === 'string' && payload.screenshotPath.length > 0, - commentCount: Array.isArray(payload.comments) ? payload.comments.length : 0, - strokeCount: Array.isArray(payload.strokes) ? payload.strokes.length : 0, - }); - } - }); - await waitForHandshake(session.page); - recorder.mark('setup.handshake.end'); - await installBrowserTimingProbe(session.page); - - const runs = []; - const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title'; - for (let iteration = 1; iteration <= iterations; iteration += 1) { - await pickElement(session.page, pickSelector, { resetPickMode: iteration > 1 }); - if (scenario === 'annotated') { - await drawAnnotationPinAndStroke(session.page, { comment: 'Benchmark annotation' }); - } - await resetBrowserTimingProbe(session.page, iteration); - - const goStarted = recorder.mark('ui.go.start', { iteration, scenario }); - const firstVariant = waitForFirstVariant(session.page).then(() => { - recorder.mark('browser.first_variant', { iteration, scenario }); - }); - - await clickGo(session.page); - recorder.mark('ui.generating_visible', { iteration, scenario }); - await firstVariant; - await waitForCycling(session.page, 3, { timeout: agentMode === 'llm' ? 150_000 : 30_000 }); - recorder.mark('browser.all_variants', { iteration, scenario }); - const browserTiming = await readBrowserTimingProbe(session.page); - - const run = buildInteractionRun(recorder.events, { - iteration, - scenario, - goStartedAt: goStarted.at, - browserTiming, - }); - assertScenarioEvidence(run, scenario); - runs.push(run); - - if (!quiet) process.stderr.write(formatRun(runs.at(-1)) + '\n'); - await clickDiscard(session.page); - await waitForReset(session.page); - } - - const report = createBenchmarkReport({ - fixture: fixtureName, - agent: agentMode, - provider: agentInfo.provider, - model: agentInfo.model, - scenario, - runs, - events: recorder.events, - harnessProbe: args.harnessProbe || null, - delivery, - promptMode: agentInfo.promptMode, - simulation: simulatedTailMs > 0 ? { remainingGenerationMs: simulatedTailMs } : null, - }); - - let output = report; - if (outputPath && args.append) { - try { - const existing = JSON.parse(await readFile(outputPath, 'utf-8')); - const previousReports = Array.isArray(existing.reports) ? existing.reports : [existing]; - output = mergeBenchmarkReports([...previousReports, report]); - } catch (error) { - if (error.code !== 'ENOENT') throw error; - } - } - const json = JSON.stringify(output, null, 2) + '\n'; - if (outputPath) { - await mkdir(dirname(outputPath), { recursive: true }); - await writeFile(outputPath, json, 'utf-8'); - process.stderr.write(`[live-bench] wrote ${outputPath}\n`); - } - process.stdout.write(json); -} finally { - if (session) await session.teardown(); - await browser.close().catch(() => {}); -} - -async function resolveAgent(mode, options) { - if (mode === 'fake') return { agent: createFakeAgent(), provider: 'deterministic', model: null, promptMode: null }; - const config = resolveLlmAgentConfig({ - provider: options.provider, - model: options.model, - }); - const agent = await createLlmAgent({ - config, - includeLiveSpec: false, - log: (message) => process.stderr.write(`[live-bench:llm] ${message}\n`), - }); - if (!agent) { - throw new Error(`LLM benchmark provider=${config.provider} requires ${config.requiredEnv}. Pass it in the environment; .env files are not read implicitly.`); - } - return { agent, provider: config.provider, model: config.model, promptMode: 'synthetic-element-contract' }; -} - -function createSplitProgressiveAgent(agent) { - const firstBySession = new Map(); - return { - ...agent, - async generateFirstVariant(event, context) { - const first = await agent.generateVariants({ - ...event, - count: 1, - progressive: { phase: 'first', totalCount: event.count }, - }, context); - firstBySession.set(event.id, first); - return first; - }, - async generateRemainingVariants(event, context) { - const first = firstBySession.get(event.id) || context.firstOutput; - const remaining = await agent.generateVariants({ - ...event, - count: event.count, - progressive: { - phase: 'remaining', - totalCount: event.count, - firstVariant: first?.variants?.[0] || null, - omitFirstVariantCss: true, - }, - }, context); - firstBySession.delete(event.id); - return assembleSplitProgressiveOutput(first, remaining); - }, - }; -} - -async function waitForFirstVariant(page) { - const handle = await page.waitForFunction(() => { - const wrappers = [...document.querySelectorAll('[data-impeccable-variant]')]; - return wrappers.some((element) => element.getAttribute('data-impeccable-variant') !== 'original'); - }, undefined, { timeout: 150_000 }); - await handle.dispose(); -} - -async function waitForReset(page) { - await page.waitForFunction(() => !document.querySelector('[data-impeccable-variants]'), undefined, { timeout: 30_000 }); - await page.waitForTimeout(100); -} - -async function installBrowserTimingProbe(page) { - await page.evaluate(() => { - const state = { iteration: 0, goAt: null, generateAt: null }; - window.__IMPECCABLE_LIVE_BENCH_TIMING__ = state; - const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.() - || window.__IMPECCABLE_LIVE_UI_ROOT__ - || document; - root.addEventListener('click', (event) => { - const button = event.composedPath().find((node) => - node?.getAttribute?.('aria-label') === 'Generate variants' - ); - if (button) state.goAt = performance.now(); - }, true); - - const originalFetch = window.fetch.bind(window); - window.fetch = (input, init) => { - try { - const url = typeof input === 'string' ? input : input?.url; - if (String(url || '').endsWith('/events') && init?.method === 'POST') { - const payload = typeof init.body === 'string' ? JSON.parse(init.body) : null; - if (payload?.type === 'generate') state.generateAt = performance.now(); - } - } catch { /* measurement must never affect Live */ } - return originalFetch(input, init); - }; - }); -} - -async function resetBrowserTimingProbe(page, iteration) { - await page.evaluate((nextIteration) => { - const state = window.__IMPECCABLE_LIVE_BENCH_TIMING__; - if (!state) return; - state.iteration = nextIteration; - state.goAt = null; - state.generateAt = null; - }, iteration); -} - -async function readBrowserTimingProbe(page) { - return page.evaluate(() => { - const state = window.__IMPECCABLE_LIVE_BENCH_TIMING__; - return state ? { ...state } : null; - }); -} - -function assertScenarioEvidence(run, currentScenario) { - const evidence = run.annotationEvidence; - if (currentScenario === 'annotated') { - if (!evidence?.screenshotPath || evidence.comments < 1 || evidence.strokes < 1) { - throw new Error(`iteration ${run.iteration}: annotated generate payload lost screenshot/comments/strokes`); - } - return; - } - if (evidence?.screenshotPath) { - throw new Error(`iteration ${run.iteration}: plain generate payload unexpectedly included screenshotPath`); - } -} - -function wrapTargetFromPickedElement(event) { - const element = event.element || {}; - return { - elementId: element.id || undefined, - classes: Array.isArray(element.classes) ? element.classes.join(',') : undefined, - tag: element.tagName ? String(element.tagName).toLowerCase() : undefined, - text: element.textContent ? String(element.textContent).trim() : undefined, - }; -} - -function formatRun(run) { - return `[live-bench] run ${run.iteration}: first=${run.goToFirstVariantMs}ms all=${run.goToAllVariantsMs}ms generation=${run.generationMs}ms overhead=${run.impeccableOverheadMs}ms`; -} diff --git a/scripts/lib/live-benchmark.mjs b/scripts/lib/live-benchmark.mjs deleted file mode 100644 index 7f3b7911a..000000000 --- a/scripts/lib/live-benchmark.mjs +++ /dev/null @@ -1,395 +0,0 @@ -import { performance } from 'node:perf_hooks'; - -const METRIC_KEYS = [ - 'browserPreparationMs', - 'browserDispatchMs', - 'automationClickMs', - 'serverPickupMs', - 'goToAgentMs', - 'serverPreflightMs', - 'scaffoldMs', - 'generationToFirstMs', - 'generationMs', - 'firstVariantWriteMs', - 'writeMs', - 'writeToFirstVariantMs', - 'replyMs', - 'goToFirstVariantMs', - 'goToAllVariantsMs', - 'deliveryGapMs', - 'impeccableOverheadMs', -]; - -export function createTraceRecorder(now = () => performance.now()) { - const events = []; - return { - events, - trace(name, data = {}) { - events.push({ name, at: now(), ...data }); - }, - mark(name, data = {}) { - const event = { name, at: now(), ...data }; - events.push(event); - return event; - }, - }; -} - -export function durationBetween(events, startName, endName, predicate = () => true) { - const start = events.find((event) => event.name === startName && predicate(event)); - const end = events.find((event) => event.name === endName && predicate(event) && (!start || event.at >= start.at)); - if (!start || !end) return null; - return roundMs(Math.max(0, end.at - start.at)); -} - -/** - * Assemble the two model calls used by the Live benchmark's progressive path. - * The first checkpoint is already visible in the browser, so both its markup - * and CSS are immutable. The tail call may supply deferred params for variant - * 1, but its CSS must contain only independently-scoped rules for variants 2+. - */ -export function assembleSplitProgressiveOutput(first, remaining) { - const firstVariant = first?.variants?.[0]; - if (!firstVariant) throw new Error('progressive assembly requires a first variant'); - if (!Array.isArray(remaining?.variants) || remaining.variants.length < 1) { - throw new Error('progressive assembly requires a complete remaining variant set'); - } - - const firstCss = String(first.scopedCss || ''); - const laterCss = String(remaining.scopedCss || ''); - assertLaterVariantCss(laterCss); - - return { - scopedCss: firstCss && laterCss ? `${firstCss}\n${laterCss}` : firstCss || laterCss, - variants: [ - { - ...firstVariant, - params: Array.isArray(remaining.variants[0]?.params) - ? remaining.variants[0].params - : [], - }, - ...remaining.variants.slice(1), - ], - }; -} - -export function buildInteractionRun(events, { iteration, scenario, goStartedAt, browserTiming = null }) { - const received = events.find((event) => - event.name === 'agent.event.received' - && event.type === 'generate' - && event.at >= goStartedAt - ); - if (!received?.id) throw new Error(`iteration ${iteration}: no generate event was traced`); - - const id = received.id; - const forId = (event) => event.id === id; - const eventPost = events.find((event) => event.name === 'browser.generate_post' && forId(event)); - const mark = (name) => events.find((event) => event.name === name && event.iteration === iteration); - const first = mark('browser.first_variant'); - const all = mark('browser.all_variants'); - const writeEnd = events.find((event) => event.name === 'agent.write.end' && forId(event)); - const firstWriteEnd = events.find((event) => event.name === 'agent.first_variant.write.end' && forId(event)); - const reusedScaffold = events.find((event) => event.name === 'agent.scaffold.reused' && forId(event)); - const generationMs = durationBetween(events, 'agent.generate.start', 'agent.generate.end', forId); - const generationToFirstMs = durationBetween(events, 'agent.generate.start', 'agent.generate.first_ready', forId); - const browserPreparationMs = eventPost ? roundMs(eventPost.at - goStartedAt) : null; - const browserDispatchMs = Number.isFinite(browserTiming?.goAt) && Number.isFinite(browserTiming?.generateAt) - ? roundMs(Math.max(0, browserTiming.generateAt - browserTiming.goAt)) - : null; - const interactionStartedAt = eventPost && browserDispatchMs != null - ? eventPost.at - browserDispatchMs - : goStartedAt; - const measuredGoToFirstVariantMs = first ? roundMs(first.at - interactionStartedAt) : null; - const measuredGoToAllVariantsMs = all ? roundMs(all.at - interactionStartedAt) : null; - - return { - iteration, - scenario, - eventId: id, - annotationEvidence: { - screenshotPath: eventPost?.hasScreenshotPath === true, - comments: Number(eventPost?.commentCount || 0), - strokes: Number(eventPost?.strokeCount || 0), - }, - browserPreparationMs, - browserDispatchMs, - automationClickMs: browserPreparationMs == null || browserDispatchMs == null - ? null - : roundMs(Math.max(0, browserPreparationMs - browserDispatchMs)), - serverPickupMs: eventPost ? roundMs(Math.max(0, received.at - eventPost.at)) : null, - goToAgentMs: roundMs(received.at - interactionStartedAt), - serverPreflightMs: Number.isFinite(reusedScaffold?.durationMs) ? roundMs(reusedScaffold.durationMs) : null, - scaffoldMs: durationBetween(events, 'agent.scaffold.start', 'agent.scaffold.end', forId), - generationToFirstMs, - generationMs, - firstVariantWriteMs: durationBetween(events, 'agent.first_variant.write.start', 'agent.first_variant.write.end', forId), - writeMs: durationBetween(events, 'agent.write.start', 'agent.write.end', forId), - writeToFirstVariantMs: first && (firstWriteEnd || writeEnd) - ? roundMs(Math.max(0, first.at - (firstWriteEnd || writeEnd).at)) - : null, - replyMs: durationBetween(events, 'agent.reply.start', 'agent.reply.end', forId), - goToFirstVariantMs: measuredGoToFirstVariantMs, - goToAllVariantsMs: measuredGoToAllVariantsMs, - deliveryGapMs: first && all ? roundMs(Math.max(0, all.at - first.at)) : null, - impeccableOverheadMs: measuredGoToFirstVariantMs == null || generationToFirstMs == null - ? null - : roundMs(Math.max(0, measuredGoToFirstVariantMs - generationToFirstMs)), - }; -} - -export function summarizeRuns(runs) { - const metrics = {}; - for (const key of METRIC_KEYS) { - const values = runs.map((run) => run[key]).filter(Number.isFinite).sort((a, b) => a - b); - if (values.length === 0) continue; - metrics[key] = { - median: roundMs(percentile(values, 0.5)), - p95: roundMs(percentile(values, 0.95)), - min: roundMs(values[0]), - max: roundMs(values[values.length - 1]), - }; - } - return { count: runs.length, metrics }; -} - -export function summarizeSetup(events) { - const stages = [ - ['dependencies', 'setup.install.start', 'setup.install.end'], - ['liveServer', 'setup.live_server.start', 'setup.live_server.end'], - ['injection', 'setup.inject.start', 'setup.inject.end'], - ['devServer', 'setup.dev_server.start', 'setup.dev_server.end'], - ['pageLoad', 'setup.page_load.start', 'setup.page_load.end'], - ['handshake', 'setup.handshake.start', 'setup.handshake.end'], - ]; - return Object.fromEntries(stages.map(([key, start, end]) => [key, durationBetween(events, start, end)])); -} - -export function createBenchmarkReport({ - fixture, - agent, - provider, - model, - scenario, - runs, - events, - harnessProbe = null, - delivery = 'atomic', - promptMode = null, - simulation = null, - generatedAt = new Date().toISOString(), -}) { - return { - schemaVersion: 1, - generatedAt, - benchmark: { - fixture, - agent, - provider: provider || null, - model: model || null, - scenario, - variants: 3, - delivery, - promptMode, - simulation, - }, - setup: summarizeSetup(events), - summary: summarizeRuns(runs), - runs, - harnessProbe, - }; -} - -export function mergeBenchmarkReports(reports, generatedAt = new Date().toISOString()) { - return { - schemaVersion: 1, - generatedAt, - reports, - }; -} - -export function compareModelBackedReports(atomic, progressive, { - medianTarget = 0.35, - p95Target = 0.25, - minimumRuns = 3, -} = {}) { - assertComparableModelReport(atomic, 'atomic', minimumRuns); - assertComparableModelReport(progressive, 'progressive', minimumRuns); - - for (const key of ['fixture', 'provider', 'model', 'scenario', 'variants', 'promptMode']) { - if (atomic.benchmark[key] !== progressive.benchmark[key]) { - throw new Error(`benchmark mismatch for ${key}: atomic=${atomic.benchmark[key]} progressive=${progressive.benchmark[key]}`); - } - } - - const atomicFirst = requiredMetric(atomic, 'goToFirstVariantMs'); - const progressiveFirst = requiredMetric(progressive, 'goToFirstVariantMs'); - const medianImprovement = improvement(atomicFirst.median, progressiveFirst.median); - const p95Improvement = improvement(atomicFirst.p95, progressiveFirst.p95); - const allReady = { - atomic: requiredMetric(atomic, 'goToAllVariantsMs'), - progressive: requiredMetric(progressive, 'goToAllVariantsMs'), - }; - const passed = medianImprovement >= medianTarget && p95Improvement >= p95Target; - - return { - passed, - target: { medianImprovement, p95Improvement, medianTarget, p95Target }, - firstReviewable: { atomic: atomicFirst, progressive: progressiveFirst }, - allVariantsReady: allReady, - benchmark: { - fixture: atomic.benchmark.fixture, - provider: atomic.benchmark.provider, - model: atomic.benchmark.model, - scenario: atomic.benchmark.scenario, - runs: { atomic: atomic.summary.count, progressive: progressive.summary.count }, - }, - }; -} - -function assertComparableModelReport(report, delivery, minimumRuns) { - if (!report?.benchmark || !report?.summary) throw new Error(`${delivery} benchmark report is missing metadata or summary`); - if (report.benchmark.agent !== 'llm') throw new Error(`${delivery} benchmark must be model-backed (agent=llm)`); - if (report.benchmark.delivery !== delivery) { - throw new Error(`expected ${delivery} delivery report, got ${report.benchmark.delivery || 'unknown'}`); - } - if (report.benchmark.simulation) throw new Error(`${delivery} model benchmark must not contain simulated latency`); - if (!report.benchmark.provider || !report.benchmark.model) throw new Error(`${delivery} benchmark is missing provider/model identity`); - if (!Number.isInteger(report.summary.count) || report.summary.count < minimumRuns) { - throw new Error(`${delivery} benchmark requires at least ${minimumRuns} runs`); - } -} - -function requiredMetric(report, key) { - const metric = report.summary.metrics?.[key]; - if (!Number.isFinite(metric?.median) || !Number.isFinite(metric?.p95)) { - throw new Error(`${report.benchmark.delivery} benchmark is missing ${key} median/p95`); - } - return { median: metric.median, p95: metric.p95 }; -} - -function improvement(baseline, candidate) { - if (!(baseline > 0) || !Number.isFinite(candidate)) throw new Error('benchmark latency must be finite and baseline must be positive'); - return Number((1 - (candidate / baseline)).toFixed(4)); -} - -function assertLaterVariantCss(css) { - if (!css.trim()) return; - for (const prelude of topLevelCssPreludes(css)) { - const variants = [...prelude.matchAll(/\[data-impeccable-variant\s*=\s*(["'])(\d+)\1[^\]]*\]/g)] - .map((match) => Number(match[2])); - if (variants.includes(1)) { - throw new Error('progressive tail CSS must not repeat or conflict with published variant 1 CSS'); - } - if (variants.length === 0 || variants.some((variant) => variant < 2)) { - throw new Error('progressive tail CSS must be attributable only to variants 2+'); - } - if (new Set(variants).size !== 1) { - throw new Error('each progressive tail CSS block must target exactly one later variant'); - } - } -} - -function topLevelCssPreludes(css) { - const preludes = []; - let cursor = 0; - while (cursor < css.length) { - while (cursor < css.length && /\s/.test(css[cursor])) cursor += 1; - if (cursor >= css.length) break; - const start = cursor; - const open = findCssToken(css, cursor, '{'); - if (open === -1) throw new Error('progressive tail CSS contains a rule without a block'); - const prelude = css.slice(start, open).trim(); - if (!prelude || prelude.includes(';')) { - throw new Error('progressive tail CSS must contain scoped rule blocks only'); - } - preludes.push(prelude); - const close = findMatchingCssBrace(css, open); - if (close === -1) throw new Error('progressive tail CSS has unbalanced braces'); - cursor = close + 1; - } - return preludes; -} - -function findCssToken(css, start, token) { - let quote = null; - let comment = false; - for (let index = start; index < css.length; index += 1) { - const char = css[index]; - const next = css[index + 1]; - if (comment) { - if (char === '*' && next === '/') { - comment = false; - index += 1; - } - continue; - } - if (!quote && char === '/' && next === '*') { - comment = true; - index += 1; - continue; - } - if (quote) { - if (char === '\\') index += 1; - else if (char === quote) quote = null; - continue; - } - if (char === '"' || char === "'") { - quote = char; - continue; - } - if (char === token) return index; - } - return -1; -} - -function findMatchingCssBrace(css, open) { - let depth = 0; - let quote = null; - let comment = false; - for (let index = open; index < css.length; index += 1) { - const char = css[index]; - const next = css[index + 1]; - if (comment) { - if (char === '*' && next === '/') { - comment = false; - index += 1; - } - continue; - } - if (!quote && char === '/' && next === '*') { - comment = true; - index += 1; - continue; - } - if (quote) { - if (char === '\\') index += 1; - else if (char === quote) quote = null; - continue; - } - if (char === '"' || char === "'") { - quote = char; - continue; - } - if (char === '{') depth += 1; - if (char === '}') { - depth -= 1; - if (depth === 0) return index; - } - } - return -1; -} - -function percentile(sortedValues, ratio) { - if (sortedValues.length === 1) return sortedValues[0]; - const index = (sortedValues.length - 1) * ratio; - const lower = Math.floor(index); - const upper = Math.ceil(index); - if (lower === upper) return sortedValues[lower]; - const weight = index - lower; - return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight; -} - -function roundMs(value) { - if (!Number.isFinite(value)) return null; - return Number(value.toFixed(2)); -} diff --git a/scripts/test-suites.mjs b/scripts/test-suites.mjs index 72e8d6284..181d09ccc 100644 --- a/scripts/test-suites.mjs +++ b/scripts/test-suites.mjs @@ -125,7 +125,6 @@ export const SUITES = { 'tests/live-browser-regression.test.mjs', 'tests/live-browser-session.test.mjs', 'tests/live-browser-source.test.mjs', - 'tests/live-benchmark.test.mjs', 'tests/live-commit-manual-edits.test.mjs', 'tests/live-completion.test.mjs', 'tests/live-copy-edit-agent.test.mjs', diff --git a/skill/scripts/live-wrap.mjs b/skill/scripts/live-wrap.mjs index 4e18b70db..895217538 100644 --- a/skill/scripts/live-wrap.mjs +++ b/skill/scripts/live-wrap.mjs @@ -230,7 +230,6 @@ The agent should insert variant HTML at insertLine.`); // Strip only the COMMON minimum leading whitespace across the picked lines; // `deindentContent` on the accept side already mirrors this convention. let originalLines = lines.slice(startLine, endLine + 1); - const sourceOriginalLines = [...originalLines]; // Buffer-aware "original" content: if the user has pending manual edits for // this page whose originalText appears in the picked source range, apply diff --git a/tests/live-benchmark.test.mjs b/tests/live-benchmark.test.mjs deleted file mode 100644 index 10dd4fc4e..000000000 --- a/tests/live-benchmark.test.mjs +++ /dev/null @@ -1,165 +0,0 @@ -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; - -import { - assembleSplitProgressiveOutput, - buildInteractionRun, - compareModelBackedReports, - createTraceRecorder, - durationBetween, - summarizeRuns, -} from '../scripts/lib/live-benchmark.mjs'; - -describe('live benchmark metrics', () => { - it('keeps published progressive CSS byte-stable and carries deferred params', () => { - const firstCss = '@scope ([data-impeccable-variant="1"]) { .offer { color: red; } }'; - const laterCss = [ - '@scope ([data-impeccable-variant="2"]) { .offer { color: green; } }', - '@scope ([data-impeccable-variant="3"]) { .offer { color: blue; } }', - ].join('\n'); - const firstVariant = { innerHtml: '
One
', params: [] }; - const deferredParams = [{ name: 'density', type: 'range', min: 0, max: 1, default: 0.5 }]; - const assembled = assembleSplitProgressiveOutput( - { scopedCss: firstCss, variants: [firstVariant] }, - { - scopedCss: laterCss, - variants: [ - { innerHtml: firstVariant.innerHtml, params: deferredParams }, - { innerHtml: '
Two
', params: [] }, - { innerHtml: '
Three
', params: [] }, - ], - }, - ); - - assert.equal(assembled.scopedCss, `${firstCss}\n${laterCss}`); - assert.equal(assembled.scopedCss.slice(0, firstCss.length), firstCss); - assert.equal(assembled.variants[0].innerHtml, firstVariant.innerHtml); - assert.equal(assembled.variants[0].params, deferredParams); - }); - - it('rejects tail CSS that would reproduce published_variant_css_changed', () => { - const first = { - scopedCss: '@scope ([data-impeccable-variant="1"]) { .offer { color: red; } }', - variants: [{ innerHtml: '
One
', params: [] }], - }; - const conflictingTail = { - scopedCss: [ - '@scope ([data-impeccable-variant="1"]) { .offer { color: purple; } }', - '@scope ([data-impeccable-variant="2"]) { .offer { color: green; } }', - ].join('\n'), - variants: [ - { innerHtml: first.variants[0].innerHtml, params: [] }, - { innerHtml: '
Two
', params: [] }, - ], - }; - - assert.throws( - () => assembleSplitProgressiveOutput(first, conflictingTail), - /must not repeat or conflict with published variant 1 CSS/, - ); - }); - - it('separates model generation from Impeccable overhead', () => { - const events = [ - { name: 'ui.go.start', at: 100, iteration: 1 }, - { name: 'browser.generate_post', at: 108, id: 'abc', hasScreenshotPath: false, commentCount: 0, strokeCount: 0 }, - { name: 'agent.event.received', at: 110, id: 'abc', type: 'generate' }, - { name: 'agent.scaffold.start', at: 112, id: 'abc' }, - { name: 'agent.scaffold.end', at: 132, id: 'abc' }, - { name: 'agent.generate.start', at: 132, id: 'abc' }, - { name: 'agent.generate.first_ready', at: 1132, id: 'abc' }, - { name: 'agent.generate.end', at: 1132, id: 'abc' }, - { name: 'agent.write.start', at: 1132, id: 'abc' }, - { name: 'agent.write.end', at: 1142, id: 'abc' }, - { name: 'agent.reply.start', at: 1142, id: 'abc' }, - { name: 'agent.reply.end', at: 1147, id: 'abc' }, - { name: 'browser.first_variant', at: 1200, iteration: 1 }, - { name: 'browser.all_variants', at: 1200, iteration: 1 }, - ]; - - const run = buildInteractionRun(events, { - iteration: 1, - scenario: 'plain', - goStartedAt: 100, - browserTiming: { goAt: 50, generateAt: 52.5 }, - }); - assert.equal(run.goToFirstVariantMs, 1094.5); - assert.equal(run.browserPreparationMs, 8); - assert.equal(run.browserDispatchMs, 2.5); - assert.equal(run.automationClickMs, 5.5); - assert.deepEqual(run.annotationEvidence, { screenshotPath: false, comments: 0, strokes: 0 }); - assert.equal(run.serverPickupMs, 2); - assert.equal(run.generationMs, 1000); - assert.equal(run.impeccableOverheadMs, 94.5); - assert.equal(run.deliveryGapMs, 0); - assert.equal(run.scaffoldMs, 20); - }); - - it('reports interpolated medians and p95 values', () => { - const summary = summarizeRuns([ - { goToFirstVariantMs: 100, generationMs: 70 }, - { goToFirstVariantMs: 200, generationMs: 140 }, - { goToFirstVariantMs: 300, generationMs: 210 }, - ]); - assert.equal(summary.metrics.goToFirstVariantMs.median, 200); - assert.equal(summary.metrics.goToFirstVariantMs.p95, 290); - }); - - it('records monotonic trace events and returns null for missing boundaries', () => { - let now = 0; - const recorder = createTraceRecorder(() => ++now); - recorder.trace('start'); - recorder.trace('end'); - assert.equal(durationBetween(recorder.events, 'start', 'end'), 1); - assert.equal(durationBetween(recorder.events, 'missing', 'end'), null); - }); - - it('proves model-backed first-reviewable thresholds with comparable reports', () => { - const atomic = modelReport('atomic', 1000, 1200, 1400, 1500); - const progressive = modelReport('progressive', 500, 700, 1450, 1550); - const comparison = compareModelBackedReports(atomic, progressive); - assert.equal(comparison.passed, true); - assert.equal(comparison.target.medianImprovement, 0.5); - assert.equal(comparison.target.p95Improvement, 0.4167); - }); - - it('rejects fake, simulated, and mismatched model reports', () => { - const atomic = modelReport('atomic', 1000, 1200, 1400, 1500); - const progressive = modelReport('progressive', 500, 700, 1450, 1550); - assert.throws( - () => compareModelBackedReports({ ...atomic, benchmark: { ...atomic.benchmark, agent: 'fake' } }, progressive), - /model-backed/, - ); - assert.throws( - () => compareModelBackedReports(atomic, { ...progressive, benchmark: { ...progressive.benchmark, simulation: { remainingGenerationMs: 1 } } }), - /simulated latency/, - ); - assert.throws( - () => compareModelBackedReports(atomic, { ...progressive, benchmark: { ...progressive.benchmark, model: 'other-model' } }), - /benchmark mismatch for model/, - ); - }); -}); - -function modelReport(delivery, firstMedian, firstP95, allMedian, allP95) { - return { - benchmark: { - fixture: 'vite8-react-plain', - agent: 'llm', - provider: 'anthropic', - model: 'claude-haiku-4-5', - scenario: 'plain', - variants: 3, - delivery, - promptMode: 'synthetic-element-contract', - simulation: null, - }, - summary: { - count: 5, - metrics: { - goToFirstVariantMs: { median: firstMedian, p95: firstP95 }, - goToAllVariantsMs: { median: allMedian, p95: allP95 }, - }, - }, - }; -}