Add Live performance lab and benchmarks

Measure framework and provider latency, enforce fidelity and cleanup gates, and publish reproducible results on the dev-only Live Lab.\n\nAI-assisted: OpenAI Codex.
2026-09-20 18:16:30 +03:00 · 2026-07-12 17:55:10 -07:00
parent 2106a2881f
commit 02b1040280
28 changed files with 3096 additions and 58 deletions
@@ -0,0 +1,395 @@
+import { performance } from 'node:perf_hooks';
+
+const METRIC_KEYS = [
+  'browserPreparationMs',
+  'browserDispatchMs',
+  'automationClickMs',
+  'serverPickupMs',
+  'goToAgentMs',
+  'serverPreflightMs',
+  'scaffoldMs',
+  'generationToFirstMs',
+  'generationMs',
+  'firstVariantWriteMs',
+  'writeMs',
+  'writeToFirstVariantMs',
+  'replyMs',
+  'goToFirstVariantMs',
+  'goToAllVariantsMs',
+  'deliveryGapMs',
+  'impeccableOverheadMs',
+];
+
+export function createTraceRecorder(now = () => performance.now()) {
+  const events = [];
+  return {
+    events,
+    trace(name, data = {}) {
+      events.push({ name, at: now(), ...data });
+    },
+    mark(name, data = {}) {
+      const event = { name, at: now(), ...data };
+      events.push(event);
+      return event;
+    },
+  };
+}
+
+export function durationBetween(events, startName, endName, predicate = () => true) {
+  const start = events.find((event) => event.name === startName && predicate(event));
+  const end = events.find((event) => event.name === endName && predicate(event) && (!start || event.at >= start.at));
+  if (!start || !end) return null;
+  return roundMs(Math.max(0, end.at - start.at));
+}
+
+/**
+ * Assemble the two model calls used by the Live benchmark's progressive path.
+ * The first checkpoint is already visible in the browser, so both its markup
+ * and CSS are immutable. The tail call may supply deferred params for variant
+ * 1, but its CSS must contain only independently-scoped rules for variants 2+.
+ */
+export function assembleSplitProgressiveOutput(first, remaining) {
+  const firstVariant = first?.variants?.[0];
+  if (!firstVariant) throw new Error('progressive assembly requires a first variant');
+  if (!Array.isArray(remaining?.variants) || remaining.variants.length < 1) {
+    throw new Error('progressive assembly requires a complete remaining variant set');
+  }
+
+  const firstCss = String(first.scopedCss || '');
+  const laterCss = String(remaining.scopedCss || '');
+  assertLaterVariantCss(laterCss);
+
+  return {
+    scopedCss: firstCss && laterCss ? `${firstCss}\n${laterCss}` : firstCss || laterCss,
+    variants: [
+      {
+        ...firstVariant,
+        params: Array.isArray(remaining.variants[0]?.params)
+          ? remaining.variants[0].params
+          : [],
+      },
+      ...remaining.variants.slice(1),
+    ],
+  };
+}
+
+export function buildInteractionRun(events, { iteration, scenario, goStartedAt, browserTiming = null }) {
+  const received = events.find((event) =>
+    event.name === 'agent.event.received'
+    && event.type === 'generate'
+    && event.at >= goStartedAt
+  );
+  if (!received?.id) throw new Error(`iteration ${iteration}: no generate event was traced`);
+
+  const id = received.id;
+  const forId = (event) => event.id === id;
+  const eventPost = events.find((event) => event.name === 'browser.generate_post' && forId(event));
+  const mark = (name) => events.find((event) => event.name === name && event.iteration === iteration);
+  const first = mark('browser.first_variant');
+  const all = mark('browser.all_variants');
+  const writeEnd = events.find((event) => event.name === 'agent.write.end' && forId(event));
+  const firstWriteEnd = events.find((event) => event.name === 'agent.first_variant.write.end' && forId(event));
+  const reusedScaffold = events.find((event) => event.name === 'agent.scaffold.reused' && forId(event));
+  const generationMs = durationBetween(events, 'agent.generate.start', 'agent.generate.end', forId);
+  const generationToFirstMs = durationBetween(events, 'agent.generate.start', 'agent.generate.first_ready', forId);
+  const browserPreparationMs = eventPost ? roundMs(eventPost.at - goStartedAt) : null;
+  const browserDispatchMs = Number.isFinite(browserTiming?.goAt) && Number.isFinite(browserTiming?.generateAt)
+    ? roundMs(Math.max(0, browserTiming.generateAt - browserTiming.goAt))
+    : null;
+  const interactionStartedAt = eventPost && browserDispatchMs != null
+    ? eventPost.at - browserDispatchMs
+    : goStartedAt;
+  const measuredGoToFirstVariantMs = first ? roundMs(first.at - interactionStartedAt) : null;
+  const measuredGoToAllVariantsMs = all ? roundMs(all.at - interactionStartedAt) : null;
+
+  return {
+    iteration,
+    scenario,
+    eventId: id,
+    annotationEvidence: {
+      screenshotPath: eventPost?.hasScreenshotPath === true,
+      comments: Number(eventPost?.commentCount || 0),
+      strokes: Number(eventPost?.strokeCount || 0),
+    },
+    browserPreparationMs,
+    browserDispatchMs,
+    automationClickMs: browserPreparationMs == null || browserDispatchMs == null
+      ? null
+      : roundMs(Math.max(0, browserPreparationMs - browserDispatchMs)),
+    serverPickupMs: eventPost ? roundMs(Math.max(0, received.at - eventPost.at)) : null,
+    goToAgentMs: roundMs(received.at - interactionStartedAt),
+    serverPreflightMs: Number.isFinite(reusedScaffold?.durationMs) ? roundMs(reusedScaffold.durationMs) : null,
+    scaffoldMs: durationBetween(events, 'agent.scaffold.start', 'agent.scaffold.end', forId),
+    generationToFirstMs,
+    generationMs,
+    firstVariantWriteMs: durationBetween(events, 'agent.first_variant.write.start', 'agent.first_variant.write.end', forId),
+    writeMs: durationBetween(events, 'agent.write.start', 'agent.write.end', forId),
+    writeToFirstVariantMs: first && (firstWriteEnd || writeEnd)
+      ? roundMs(Math.max(0, first.at - (firstWriteEnd || writeEnd).at))
+      : null,
+    replyMs: durationBetween(events, 'agent.reply.start', 'agent.reply.end', forId),
+    goToFirstVariantMs: measuredGoToFirstVariantMs,
+    goToAllVariantsMs: measuredGoToAllVariantsMs,
+    deliveryGapMs: first && all ? roundMs(Math.max(0, all.at - first.at)) : null,
+    impeccableOverheadMs: measuredGoToFirstVariantMs == null || generationToFirstMs == null
+      ? null
+      : roundMs(Math.max(0, measuredGoToFirstVariantMs - generationToFirstMs)),
+  };
+}
+
+export function summarizeRuns(runs) {
+  const metrics = {};
+  for (const key of METRIC_KEYS) {
+    const values = runs.map((run) => run[key]).filter(Number.isFinite).sort((a, b) => a - b);
+    if (values.length === 0) continue;
+    metrics[key] = {
+      median: roundMs(percentile(values, 0.5)),
+      p95: roundMs(percentile(values, 0.95)),
+      min: roundMs(values[0]),
+      max: roundMs(values[values.length - 1]),
+    };
+  }
+  return { count: runs.length, metrics };
+}
+
+export function summarizeSetup(events) {
+  const stages = [
+    ['dependencies', 'setup.install.start', 'setup.install.end'],
+    ['liveServer', 'setup.live_server.start', 'setup.live_server.end'],
+    ['injection', 'setup.inject.start', 'setup.inject.end'],
+    ['devServer', 'setup.dev_server.start', 'setup.dev_server.end'],
+    ['pageLoad', 'setup.page_load.start', 'setup.page_load.end'],
+    ['handshake', 'setup.handshake.start', 'setup.handshake.end'],
+  ];
+  return Object.fromEntries(stages.map(([key, start, end]) => [key, durationBetween(events, start, end)]));
+}
+
+export function createBenchmarkReport({
+  fixture,
+  agent,
+  provider,
+  model,
+  scenario,
+  runs,
+  events,
+  harnessProbe = null,
+  delivery = 'atomic',
+  promptMode = null,
+  simulation = null,
+  generatedAt = new Date().toISOString(),
+}) {
+  return {
+    schemaVersion: 1,
+    generatedAt,
+    benchmark: {
+      fixture,
+      agent,
+      provider: provider || null,
+      model: model || null,
+      scenario,
+      variants: 3,
+      delivery,
+      promptMode,
+      simulation,
+    },
+    setup: summarizeSetup(events),
+    summary: summarizeRuns(runs),
+    runs,
+    harnessProbe,
+  };
+}
+
+export function mergeBenchmarkReports(reports, generatedAt = new Date().toISOString()) {
+  return {
+    schemaVersion: 1,
+    generatedAt,
+    reports,
+  };
+}
+
+export function compareModelBackedReports(atomic, progressive, {
+  medianTarget = 0.35,
+  p95Target = 0.25,
+  minimumRuns = 3,
+} = {}) {
+  assertComparableModelReport(atomic, 'atomic', minimumRuns);
+  assertComparableModelReport(progressive, 'progressive', minimumRuns);
+
+  for (const key of ['fixture', 'provider', 'model', 'scenario', 'variants', 'promptMode']) {
+    if (atomic.benchmark[key] !== progressive.benchmark[key]) {
+      throw new Error(`benchmark mismatch for ${key}: atomic=${atomic.benchmark[key]} progressive=${progressive.benchmark[key]}`);
+    }
+  }
+
+  const atomicFirst = requiredMetric(atomic, 'goToFirstVariantMs');
+  const progressiveFirst = requiredMetric(progressive, 'goToFirstVariantMs');
+  const medianImprovement = improvement(atomicFirst.median, progressiveFirst.median);
+  const p95Improvement = improvement(atomicFirst.p95, progressiveFirst.p95);
+  const allReady = {
+    atomic: requiredMetric(atomic, 'goToAllVariantsMs'),
+    progressive: requiredMetric(progressive, 'goToAllVariantsMs'),
+  };
+  const passed = medianImprovement >= medianTarget && p95Improvement >= p95Target;
+
+  return {
+    passed,
+    target: { medianImprovement, p95Improvement, medianTarget, p95Target },
+    firstReviewable: { atomic: atomicFirst, progressive: progressiveFirst },
+    allVariantsReady: allReady,
+    benchmark: {
+      fixture: atomic.benchmark.fixture,
+      provider: atomic.benchmark.provider,
+      model: atomic.benchmark.model,
+      scenario: atomic.benchmark.scenario,
+      runs: { atomic: atomic.summary.count, progressive: progressive.summary.count },
+    },
+  };
+}
+
+function assertComparableModelReport(report, delivery, minimumRuns) {
+  if (!report?.benchmark || !report?.summary) throw new Error(`${delivery} benchmark report is missing metadata or summary`);
+  if (report.benchmark.agent !== 'llm') throw new Error(`${delivery} benchmark must be model-backed (agent=llm)`);
+  if (report.benchmark.delivery !== delivery) {
+    throw new Error(`expected ${delivery} delivery report, got ${report.benchmark.delivery || 'unknown'}`);
+  }
+  if (report.benchmark.simulation) throw new Error(`${delivery} model benchmark must not contain simulated latency`);
+  if (!report.benchmark.provider || !report.benchmark.model) throw new Error(`${delivery} benchmark is missing provider/model identity`);
+  if (!Number.isInteger(report.summary.count) || report.summary.count < minimumRuns) {
+    throw new Error(`${delivery} benchmark requires at least ${minimumRuns} runs`);
+  }
+}
+
+function requiredMetric(report, key) {
+  const metric = report.summary.metrics?.[key];
+  if (!Number.isFinite(metric?.median) || !Number.isFinite(metric?.p95)) {
+    throw new Error(`${report.benchmark.delivery} benchmark is missing ${key} median/p95`);
+  }
+  return { median: metric.median, p95: metric.p95 };
+}
+
+function improvement(baseline, candidate) {
+  if (!(baseline > 0) || !Number.isFinite(candidate)) throw new Error('benchmark latency must be finite and baseline must be positive');
+  return Number((1 - (candidate / baseline)).toFixed(4));
+}
+
+function assertLaterVariantCss(css) {
+  if (!css.trim()) return;
+  for (const prelude of topLevelCssPreludes(css)) {
+    const variants = [...prelude.matchAll(/\[data-impeccable-variant\s*=\s*(["'])(\d+)\1[^\]]*\]/g)]
+      .map((match) => Number(match[2]));
+    if (variants.includes(1)) {
+      throw new Error('progressive tail CSS must not repeat or conflict with published variant 1 CSS');
+    }
+    if (variants.length === 0 || variants.some((variant) => variant < 2)) {
+      throw new Error('progressive tail CSS must be attributable only to variants 2+');
+    }
+    if (new Set(variants).size !== 1) {
+      throw new Error('each progressive tail CSS block must target exactly one later variant');
+    }
+  }
+}
+
+function topLevelCssPreludes(css) {
+  const preludes = [];
+  let cursor = 0;
+  while (cursor < css.length) {
+    while (cursor < css.length && /\s/.test(css[cursor])) cursor += 1;
+    if (cursor >= css.length) break;
+    const start = cursor;
+    const open = findCssToken(css, cursor, '{');
+    if (open === -1) throw new Error('progressive tail CSS contains a rule without a block');
+    const prelude = css.slice(start, open).trim();
+    if (!prelude || prelude.includes(';')) {
+      throw new Error('progressive tail CSS must contain scoped rule blocks only');
+    }
+    preludes.push(prelude);
+    const close = findMatchingCssBrace(css, open);
+    if (close === -1) throw new Error('progressive tail CSS has unbalanced braces');
+    cursor = close + 1;
+  }
+  return preludes;
+}
+
+function findCssToken(css, start, token) {
+  let quote = null;
+  let comment = false;
+  for (let index = start; index < css.length; index += 1) {
+    const char = css[index];
+    const next = css[index + 1];
+    if (comment) {
+      if (char === '*' && next === '/') {
+        comment = false;
+        index += 1;
+      }
+      continue;
+    }
+    if (!quote && char === '/' && next === '*') {
+      comment = true;
+      index += 1;
+      continue;
+    }
+    if (quote) {
+      if (char === '\\') index += 1;
+      else if (char === quote) quote = null;
+      continue;
+    }
+    if (char === '"' || char === "'") {
+      quote = char;
+      continue;
+    }
+    if (char === token) return index;
+  }
+  return -1;
+}
+
+function findMatchingCssBrace(css, open) {
+  let depth = 0;
+  let quote = null;
+  let comment = false;
+  for (let index = open; index < css.length; index += 1) {
+    const char = css[index];
+    const next = css[index + 1];
+    if (comment) {
+      if (char === '*' && next === '/') {
+        comment = false;
+        index += 1;
+      }
+      continue;
+    }
+    if (!quote && char === '/' && next === '*') {
+      comment = true;
+      index += 1;
+      continue;
+    }
+    if (quote) {
+      if (char === '\\') index += 1;
+      else if (char === quote) quote = null;
+      continue;
+    }
+    if (char === '"' || char === "'") {
+      quote = char;
+      continue;
+    }
+    if (char === '{') depth += 1;
+    if (char === '}') {
+      depth -= 1;
+      if (depth === 0) return index;
+    }
+  }
+  return -1;
+}
+
+function percentile(sortedValues, ratio) {
+  if (sortedValues.length === 1) return sortedValues[0];
+  const index = (sortedValues.length - 1) * ratio;
+  const lower = Math.floor(index);
+  const upper = Math.ceil(index);
+  if (lower === upper) return sortedValues[lower];
+  const weight = index - lower;
+  return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight;
+}
+
+function roundMs(value) {
+  if (!Number.isFinite(value)) return null;
+  return Number(value.toFixed(2));
+}
@@ -0,0 +1,547 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { performance } from 'node:perf_hooks';
+
+import { anthropic } from '@ai-sdk/anthropic';
+import { google } from '@ai-sdk/google';
+import { openai } from '@ai-sdk/openai';
+import { generateText } from 'ai';
+
+import {
+  VARIANT_SYSTEM_INSTRUCTIONS,
+  parseVariantResponse,
+  validateProgressiveVariantOutput,
+  validateVariantCount,
+  validateVariantMaterialChange,
+  validateVariantVisibleCopy,
+} from '../../tests/live-e2e/agents/llm-agent.mjs';
+
+export const PROVIDER_PROFILES = Object.freeze({
+  anthropic: {
+    label: 'Anthropic',
+    model: 'claude-sonnet-4-6',
+    envKeys: ['ANTHROPIC_API_KEY'],
+    pricePerMillion: { input: 3, cachedInput: 0.3, output: 15 },
+    effort: 'low',
+    priceSource: 'https://platform.claude.com/docs/en/about-claude/pricing',
+  },
+  openai: {
+    label: 'OpenAI',
+    model: 'gpt-5.5',
+    envKeys: ['OPENAI_API_KEY'],
+    pricePerMillion: { input: 5, cachedInput: 0.5, output: 30 },
+    effort: 'low',
+    priceSource: 'https://developers.openai.com/api/docs/models/gpt-5.5',
+  },
+  google: {
+    label: 'Google',
+    model: 'gemini-3.1-flash-lite',
+    envKeys: ['GOOGLE_GENERATIVE_AI_API_KEY', 'GOOGLE_CLOUD_API_KEY', 'GEMINI_API_KEY'],
+    pricePerMillion: { input: 0.25, cachedInput: 0.025, output: 1.5 },
+    effort: 'minimal (provider default)',
+    priceSource: 'https://ai.google.dev/gemini-api/docs/pricing',
+  },
+});
+
+export const STRATEGIES = Object.freeze({
+  'atomic-full': {
+    delivery: 'atomic',
+    promptMode: 'full-live-context',
+    calls: 'one 3-variant call',
+  },
+  'progressive-full': {
+    delivery: 'progressive',
+    promptMode: 'full-live-context',
+    calls: 'one first-variant call, then one remaining-directions call; deterministic assembly preserves variant 1',
+  },
+  'progressive-compact': {
+    delivery: 'progressive',
+    promptMode: 'compact-producer-contract',
+    calls: 'one first-variant call, then one remaining-directions call; deterministic assembly preserves variant 1',
+  },
+  'parallel-compact': {
+    delivery: 'parallel-progressive',
+    promptMode: 'compact-producer-contract',
+    calls: 'three concurrent one-variant calls; first valid result publishes immediately',
+  },
+});
+
+export const BRAND_CONTRACT = Object.freeze({
+  identity: 'Warm paper, dark ink, moss and brass accents; Georgia display with a restrained sans body; editorial, practical, and quiet.',
+  requiredCopy: [
+    'Quarterly print edition',
+    'Field Notes',
+    'Four routes, annotated maps, and practical details for unhurried weekends.',
+    'Reserve issue eight',
+  ],
+  requiredClasses: [
+    'offer-card',
+    'offer-card__copy',
+    'offer-card__eyebrow',
+    'offer-card__title',
+    'offer-card__body',
+    'action-link',
+  ],
+  allowedTokens: [
+    '--color-paper',
+    '--color-paper-deep',
+    '--color-ink',
+    '--color-moss',
+    '--color-brass',
+    '--font-display',
+    '--font-body',
+    '--space-1',
+    '--space-2',
+    '--space-3',
+    '--space-4',
+    '--radius-control',
+  ],
+  sourceExcerpt: [
+    '<article className="offer-card" aria-labelledby="field-notes-title">',
+    '  <div className="offer-card__copy">',
+    '    <p className="offer-card__eyebrow">Quarterly print edition</p>',
+    '    <h2 className="offer-card__title" id="field-notes-title">Field Notes</h2>',
+    '    <p className="offer-card__body">Four routes, annotated maps, and practical details for unhurried weekends.</p>',
+    '  </div>',
+    '  <a className="action-link" href="#edition">Reserve issue eight</a>',
+    '</article>',
+  ].join('\n'),
+});
+
+const COMPACT_CONTRACT = [
+  VARIANT_SYSTEM_INSTRUCTIONS,
+  '',
+  'QUALITY GATE FOR THIS PRODUCER:',
+  '- Preserve all visible copy exactly and retain the article/component class contract.',
+  '- Stay inside the supplied identity. Reuse the supplied CSS custom properties instead of inventing colors, typefaces, spacing, or radii.',
+  '- Do not add gradients, blur, glow, glass, neon, decorative shadows, emoji, or unrelated content.',
+  '- Make each variant materially different through hierarchy, layout, density, or color-role allocation.',
+].join('\n');
+
+export function loadBenchmarkEnv({ repoRoot, explicitPath } = {}) {
+  const candidates = [
+    explicitPath,
+    repoRoot && path.join(repoRoot, '.env'),
+    path.join(os.homedir(), 'code', 'impeccable-evals', '.env'),
+  ].filter(Boolean);
+  const loaded = [];
+  for (const file of candidates) {
+    if (!fs.existsSync(file)) continue;
+    const body = fs.readFileSync(file, 'utf-8');
+    for (const line of body.split(/\r?\n/)) {
+      const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
+      if (!match || match[1].startsWith('#')) continue;
+      let value = match[2];
+      if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+        value = value.slice(1, -1);
+      }
+      if (!process.env[match[1]] && value) process.env[match[1]] = value;
+    }
+    loaded.push(file);
+  }
+  if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY) {
+    process.env.GOOGLE_GENERATIVE_AI_API_KEY = process.env.GOOGLE_CLOUD_API_KEY || process.env.GEMINI_API_KEY;
+  }
+  return loaded;
+}
+
+export function resolveProviderSelection(providerNames, modelOverrides = {}) {
+  return providerNames.map((provider) => {
+    const profile = PROVIDER_PROFILES[provider];
+    if (!profile) throw new Error(`unknown provider ${JSON.stringify(provider)}`);
+    const keyPresent = profile.envKeys.some((key) => Boolean(process.env[key]));
+    return {
+      provider,
+      label: profile.label,
+      model: modelOverrides[provider] || profile.model,
+      keyPresent,
+      pricePerMillion: profile.pricePerMillion,
+      effort: profile.effort,
+      priceSource: profile.priceSource,
+    };
+  });
+}
+
+export function createProviderLiveAgent({ provider, model, strategy, liveSpec, onRecord = () => {} }) {
+  const strategyConfig = STRATEGIES[strategy];
+  if (!strategyConfig) throw new Error(`unknown strategy ${JSON.stringify(strategy)}`);
+  const languageModel = providerModel(provider, model);
+  const system = strategyConfig.promptMode === 'full-live-context'
+    ? `${COMPACT_CONTRACT}\n\nFULL LIVE CONTEXT:\n${liveSpec}`
+    : COMPACT_CONTRACT;
+  const pendingParallel = new Map();
+  const pendingFirst = new Map();
+
+  const request = async ({ event, phase, lane = null, firstVariant = null }) => {
+    const startedAt = performance.now();
+    const expectedCount = Number(event.count);
+    const payload = benchmarkPayload(event, { phase, lane, firstVariant });
+    const basePrompt = [
+      'Produce Impeccable Live variant output for this request. Return only the JSON object.',
+      phaseInstructions(phase, expectedCount, lane),
+      '',
+      '<benchmark_context>',
+      JSON.stringify(payload, null, 2),
+      '</benchmark_context>',
+    ].join('\n');
+    let prompt = basePrompt;
+    let lastError;
+    for (let attempt = 1; attempt <= 2; attempt += 1) {
+      const attemptStartedAt = performance.now();
+      let usage = null;
+      try {
+        const response = await generateText({
+          model: languageModel,
+          system,
+          prompt,
+          maxOutputTokens: 12_000,
+          ...providerLatencyOptions(provider),
+        });
+        usage = normalizeUsage(response.usage);
+        const parsed = parseVariantResponse(response.text);
+        const validationError = validateVariantOutput(parsed, event, { phase, firstVariant });
+        if (validationError) throw new Error(validationError);
+        const record = {
+          provider,
+          model,
+          strategy,
+          phase,
+          lane,
+          attempt,
+          durationMs: roundMs(performance.now() - attemptStartedAt),
+          totalPhaseMs: roundMs(performance.now() - startedAt),
+          usage,
+          estimatedCostUsd: estimateCostUsd(usage, PROVIDER_PROFILES[provider].pricePerMillion),
+          output: parsed,
+        };
+        onRecord(record);
+        return parsed;
+      } catch (error) {
+        lastError = error;
+        onRecord({
+          provider,
+          model,
+          strategy,
+          phase,
+          lane,
+          attempt,
+          durationMs: roundMs(performance.now() - attemptStartedAt),
+          usage,
+          estimatedCostUsd: usage ? estimateCostUsd(usage, PROVIDER_PROFILES[provider].pricePerMillion) : 0,
+          error: String(error?.message || error),
+        });
+        prompt = `${basePrompt}\n\nVALIDATION ERROR:\n${String(error?.message || error)}\nReturn corrected JSON only.`;
+      }
+    }
+    throw lastError;
+  };
+
+  if (strategy === 'atomic-full') {
+    return {
+      async generateVariants(event) {
+        return request({ event, phase: 'atomic' });
+      },
+    };
+  }
+
+  if (strategy === 'parallel-compact') {
+    return {
+      async generateFirstVariant(event) {
+        const lanes = ['hierarchy', 'layout', 'density'];
+        const calls = lanes.map((lane) => {
+          const laneEvent = { ...event, count: 1 };
+          return request({ event: laneEvent, phase: 'parallel-lane', lane }).then((output) => ({ lane, output }));
+        });
+        const first = await Promise.race(calls);
+        pendingParallel.set(event.id, { calls, first });
+        return first.output;
+      },
+      async generateRemainingVariants(event) {
+        const pending = pendingParallel.get(event.id);
+        if (!pending) throw new Error(`parallel generation state missing for ${event.id}`);
+        const settled = await Promise.all(pending.calls);
+        pendingParallel.delete(event.id);
+        const ordered = [pending.first, ...settled.filter((item) => item !== pending.first)];
+        const variants = ordered.map((item) => item.output.variants[0]);
+        const scopedCss = ordered.map((item, index) => remapSingleVariantCss(item.output.scopedCss, index + 1)).join('\n');
+        const output = { scopedCss, variants };
+        onRecord({ provider, model, strategy, phase: 'parallel-assembled', lane: null, attempt: 1, usage: normalizeUsage(), estimatedCostUsd: 0, output });
+        return output;
+      },
+    };
+  }
+
+  return {
+    async generateFirstVariant(event) {
+      const first = await request({ event: { ...event, count: 1 }, phase: 'first' });
+      pendingFirst.set(event.id, first);
+      return first;
+    },
+    async generateRemainingVariants(event, context) {
+      const first = pendingFirst.get(event.id) || context.firstOutput;
+      if (!first?.variants?.[0]) throw new Error(`first variant state missing for ${event.id}`);
+      const remaining = await request({
+        event: { ...event, count: Math.max(1, event.count - 1) },
+        phase: 'remaining-directions',
+        firstVariant: first.variants[0],
+      });
+      pendingFirst.delete(event.id);
+      return assembleProgressiveOutput(first, remaining);
+    },
+  };
+}
+
+export function scoreVariantOutput(output, { validationError = null } = {}) {
+  const variants = Array.isArray(output?.variants) ? output.variants : [];
+  const css = String(output?.scopedCss || '');
+  const perVariant = variants.map((variant) => String(variant.innerHtml || ''));
+  const copyChecks = perVariant.flatMap((html) => BRAND_CONTRACT.requiredCopy.map((copy) => html.includes(copy)));
+  const componentChecks = perVariant.flatMap((html) => [
+    /^\s*<article\b/i.test(html),
+    /\bclass=["'][^"']*\boffer-card\b/.test(html),
+    ...BRAND_CONTRACT.requiredClasses.slice(1).map((className) => new RegExp(`\\b${escapeRegExp(className)}\\b`).test(html)),
+    /href=["']#edition["']/.test(html),
+    /aria-labelledby=["']field-notes-title["']/.test(html),
+  ]);
+  const usedTokens = BRAND_CONTRACT.allowedTokens.filter((token) => css.includes(`var(${token}`));
+  const rawColors = css.match(/#[0-9a-f]{3,8}\b|\b(?:rgb|hsl|oklch|lab)\s*\(/gi) || [];
+  const foreignFonts = css.match(/font-family\s*:\s*([^;}]+)/gi) || [];
+  const tokenChecks = [
+    usedTokens.length >= 3,
+    rawColors.length === 0,
+    foreignFonts.every((declaration) => /var\(--font-(?:display|body)\)|inherit|serif|sans-serif/.test(declaration)),
+    !/\b(?:margin|padding|gap|border-radius)\s*:\s*(?!var\(|0(?:\D|$))[^;}]+/i.test(css),
+  ];
+  const brandChecks = [
+    /var\(--color-(?:paper|paper-deep|ink|moss|brass)\)/.test(css),
+    !/(?:linear|radial|conic)-gradient|backdrop-filter|filter\s*:\s*blur|text-shadow|box-shadow/i.test(css),
+    !/\b(?:neon|glass|glow|purple|magenta|cyan)\b/i.test(`${css}\n${perVariant.join('\n')}`),
+    !/border-radius\s*:\s*(?:999|[5-9]\d)px/i.test(css),
+  ];
+  const sourceChecks = [
+    !validationError,
+    variants.length > 0,
+    perVariant.every((html) => !/data-impeccable-|<script|<style/i.test(html)),
+    perVariant.every((html) => /^\s*<article\b[\s\S]*<\/article>\s*$/i.test(html)),
+  ];
+  const dimensions = {
+    brandFidelity: dimension(brandChecks),
+    componentFidelity: dimension(componentChecks),
+    tokenFidelity: dimension(tokenChecks),
+    copyFidelity: dimension(copyChecks),
+    sourceValidity: dimension(sourceChecks),
+  };
+  const overall = roundScore(Object.values(dimensions).reduce((sum, value) => sum + value, 0) / Object.keys(dimensions).length);
+  return {
+    ...dimensions,
+    overall,
+    passed: overall >= 0.9 && Object.values(dimensions).every((value) => value >= 0.75),
+    diagnostics: {
+      usedTokens,
+      rawColorCount: rawColors.length,
+      validationError,
+    },
+  };
+}
+
+export function validateAcceptedCleanup({ source, browserClean, buildPassed, expectedCopy = BRAND_CONTRACT.requiredCopy }) {
+  const markerFree = !/data-impeccable-|impeccable-(?:variants|carbonize|params|original)/i.test(source);
+  const copyPreserved = expectedCopy.every((copy) => source.includes(copy));
+  const sourceShape = /<article\b[^>]*\boffer-card\b[\s\S]*<\/article>/.test(source);
+  const checks = { markerFree, copyPreserved, sourceShape, browserClean: Boolean(browserClean), buildPassed: Boolean(buildPassed) };
+  return { ...checks, passed: Object.values(checks).every(Boolean) };
+}
+
+export function applyRuntimeSourceScore(quality, cleanup) {
+  const sourceChecks = [cleanup.markerFree, cleanup.copyPreserved, cleanup.sourceShape, cleanup.browserClean, cleanup.buildPassed];
+  const sourceValidity = dimension(sourceChecks);
+  const dimensions = {
+    brandFidelity: quality.brandFidelity,
+    componentFidelity: quality.componentFidelity,
+    tokenFidelity: quality.tokenFidelity,
+    copyFidelity: quality.copyFidelity,
+    sourceValidity,
+  };
+  const overall = roundScore(Object.values(dimensions).reduce((sum, value) => sum + value, 0) / Object.keys(dimensions).length);
+  return {
+    ...quality,
+    ...dimensions,
+    overall,
+    passed: cleanup.passed === true && overall >= 0.9 && Object.values(dimensions).every((value) => value >= 0.75),
+  };
+}
+
+export function assembleProgressiveOutput(first, remaining) {
+  if (!first?.variants?.[0]) throw new Error('progressive assembly requires a first variant');
+  if (!Array.isArray(remaining?.variants) || remaining.variants.length === 0) {
+    throw new Error('progressive assembly requires remaining variants');
+  }
+  return {
+    scopedCss: [first.scopedCss, shiftVariantCss(remaining.scopedCss, 1)].filter(Boolean).join('\n'),
+    variants: [first.variants[0], ...remaining.variants],
+  };
+}
+
+export function summarizeProviderRuns(runs) {
+  const latencyKeys = ['firstReviewableMs', 'allReadyMs', 'acceptCleanupMs'];
+  const metrics = {};
+  for (const key of latencyKeys) {
+    const values = runs.map((run) => run[key]).filter(Number.isFinite).sort((a, b) => a - b);
+    if (values.length) metrics[key] = summarizeNumbers(values);
+  }
+  const qualityKeys = ['brandFidelity', 'componentFidelity', 'tokenFidelity', 'copyFidelity', 'sourceValidity', 'overall'];
+  const quality = {};
+  for (const key of qualityKeys) {
+    const values = runs.map((run) => run.quality?.[key]).filter(Number.isFinite).sort((a, b) => a - b);
+    if (values.length) quality[key] = summarizeNumbers(values);
+  }
+  return {
+    count: runs.length,
+    metrics,
+    quality,
+    cleanupPassRate: runs.length ? roundScore(runs.filter((run) => run.cleanup?.passed).length / runs.length) : 0,
+    gatePassRate: runs.length ? roundScore(runs.filter((run) => run.passed).length / runs.length) : 0,
+    estimatedCostUsd: roundUsd(runs.reduce((sum, run) => sum + Number(run.estimatedCostUsd || 0), 0)),
+  };
+}
+
+function providerModel(provider, model) {
+  if (provider === 'anthropic') return anthropic(model);
+  if (provider === 'openai') return openai(model);
+  if (provider === 'google') return google(model);
+  throw new Error(`unsupported provider ${provider}`);
+}
+
+function providerLatencyOptions(provider) {
+  if (provider === 'anthropic') return { providerOptions: { anthropic: { effort: 'low' } } };
+  if (provider === 'openai') return { providerOptions: { openai: { reasoningEffort: 'low' } } };
+  // Gemini 3.1 Flash-Lite defaults to minimal thinking; leaving the provider
+  // option unset preserves that latency-oriented default across SDK versions.
+  return {};
+}
+
+function benchmarkPayload(event, { phase, lane, firstVariant }) {
+  return {
+    request: {
+      id: event.id,
+      action: event.action,
+      freeformPrompt: event.freeformPrompt,
+      count: event.count,
+      phase,
+      lane,
+      firstVariant,
+    },
+    pickedElement: event.element,
+    identityLock: BRAND_CONTRACT.identity,
+    sourceExcerpt: BRAND_CONTRACT.sourceExcerpt,
+    availableTokens: BRAND_CONTRACT.allowedTokens,
+    componentContract: {
+      rootTag: 'article',
+      requiredClasses: BRAND_CONTRACT.requiredClasses,
+      requiredHref: '#edition',
+      requiredAriaLabelledby: 'field-notes-title',
+      exactVisibleCopy: BRAND_CONTRACT.requiredCopy,
+    },
+  };
+}
+
+function phaseInstructions(phase, count, lane) {
+  if (phase === 'first') {
+    return 'Return exactly one variant. Use params: [] so tunables stay off the first-reviewable path.';
+  }
+  if (phase === 'remaining-directions') {
+    return `Return exactly ${count} new variants for different axes. Do not reproduce request.firstVariant; assembly preserves that first output byte-for-byte.`;
+  }
+  if (phase === 'parallel-lane') {
+    return `Return exactly one complete variant whose primary difference axis is ${lane}. It must stand alone and may include 0-3 useful params.`;
+  }
+  return `Return exactly ${count} complete variants in one response.`;
+}
+
+function validateVariantOutput(parsed, event, { phase, firstVariant }) {
+  const phaseEvent = phase === 'first'
+    ? { ...event, progressive: { phase: 'first', totalCount: 3 } }
+    : event;
+  return validateVariantCount(parsed, phaseEvent)
+    || validateProgressiveVariantOutput(parsed, phaseEvent)
+    || validateVariantVisibleCopy(parsed, event.element)
+    || validateVariantMaterialChange(parsed, event.element);
+}
+
+function remapSingleVariantCss(css, variantNumber) {
+  return String(css)
+    .replaceAll('[data-impeccable-variant="1"]', `[data-impeccable-variant="${variantNumber}"]`)
+    .replaceAll("[data-impeccable-variant='1']", `[data-impeccable-variant='${variantNumber}']`);
+}
+
+function shiftVariantCss(css, amount) {
+  return String(css).replace(/(data-impeccable-variant=["'])(\d+)(["'])/g, (_, before, number, after) => {
+    return `${before}${Number(number) + amount}${after}`;
+  });
+}
+
+function normalizeUsage(usage = {}) {
+  const input = numberFrom(usage.inputTokens, usage.promptTokens, usage.inputTokenDetails?.noCacheTokens);
+  const cached = numberFrom(usage.cachedInputTokens, usage.inputTokenDetails?.cacheReadTokens, usage.inputTokenDetails?.cachedTokens);
+  const output = numberFrom(usage.outputTokens, usage.completionTokens);
+  return {
+    inputTokens: input,
+    cachedInputTokens: cached,
+    outputTokens: output,
+    totalTokens: numberFrom(usage.totalTokens, input + output),
+  };
+}
+
+export function estimateCostUsd(usage, pricing) {
+  const cached = Math.min(usage.cachedInputTokens || 0, usage.inputTokens || 0);
+  const uncached = Math.max(0, (usage.inputTokens || 0) - cached);
+  return roundUsd((
+    uncached * pricing.input
+    + cached * pricing.cachedInput
+    + (usage.outputTokens || 0) * pricing.output
+  ) / 1_000_000);
+}
+
+function numberFrom(...values) {
+  for (const value of values) if (Number.isFinite(value)) return Number(value);
+  return 0;
+}
+
+function dimension(checks) {
+  return checks.length ? roundScore(checks.filter(Boolean).length / checks.length) : 0;
+}
+
+function summarizeNumbers(values) {
+  return {
+    median: roundMs(percentile(values, 0.5)),
+    p95: roundMs(percentile(values, 0.95)),
+    min: roundMs(values[0]),
+    max: roundMs(values.at(-1)),
+  };
+}
+
+function percentile(values, ratio) {
+  if (values.length === 1) return values[0];
+  const index = (values.length - 1) * ratio;
+  const lower = Math.floor(index);
+  const upper = Math.ceil(index);
+  if (lower === upper) return values[lower];
+  return values[lower] + (values[upper] - values[lower]) * (index - lower);
+}
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function roundMs(value) {
+  return Number(Number(value).toFixed(2));
+}
+
+function roundScore(value) {
+  return Number(Number(value).toFixed(4));
+}
+
+function roundUsd(value) {
+  return Number(Number(value).toFixed(6));
+}