From e0c19eff285eb103bde0b061bda91cb5edcea5b0 Mon Sep 17 00:00:00 2001 From: Paul Bakaus Date: Mon, 13 Jul 2026 13:56:25 -0700 Subject: [PATCH] Export portable Live evidence bundles AI-assisted implementation under maintainer direction. --- scripts/benchmark-live.mjs | 34 +++++++++++++++++------ scripts/lib/live-benchmark.mjs | 40 +++++++++++++++++++++++++++ scripts/lib/live-rendered-quality.mjs | 6 +++- tests/framework-fixtures/README.md | 35 +++++++++++++++++++++++ tests/live-benchmark.test.mjs | 31 +++++++++++++++++++++ tests/live-e2e/session.mjs | 7 +++-- tests/live-rendered-quality.test.mjs | 20 ++++++++++++++ 7 files changed, 161 insertions(+), 12 deletions(-) diff --git a/scripts/benchmark-live.mjs b/scripts/benchmark-live.mjs index 3af111877..025e75cea 100644 --- a/scripts/benchmark-live.mjs +++ b/scripts/benchmark-live.mjs @@ -32,6 +32,7 @@ import { deriveJournalGenerationMetrics, mergeBenchmarkReports, parseLiveBenchmarkArgs, + resolveLiveBenchmarkPaths, } from './lib/live-benchmark.mjs'; import { loadBenchmarkEnv } from './lib/live-provider-benchmark.mjs'; import { @@ -42,7 +43,14 @@ import { const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const args = parseLiveBenchmarkArgs(process.argv.slice(2)); -const fixtureName = String(args.fixture || 'vite8-react-plain'); +const { + fixtureName, + fixtureDir, + fixtureOrigin, + evidenceRoot, + artifactRoot, + outputPath, +} = resolveLiveBenchmarkPaths(args, { root: ROOT, fixturesDir: FIXTURES_DIR }); const iterations = positiveInt(args.iterations, 5); const agentMode = args.agent === 'codex' ? 'codex' : args.agent === 'llm' ? 'llm' : 'fake'; const scenario = args.scenario === 'annotated' ? 'annotated' : 'plain'; @@ -57,14 +65,17 @@ const interactionMode = acceptVariant ? `accept-variant-${acceptVariant}-then-next-go` : 'complete-then-discard'; const simulatedTailMs = positiveInt(args.simulatedTailMs, 0); -const outputPath = args.output ? resolve(ROOT, String(args.output)) : null; -const artifactRoot = args.artifacts ? resolve(ROOT, String(args.artifacts)) : null; const judgeRendered = args.judgeRendered === true || args.judgeRendered === 'true'; const judgeModel = String(args.judgeModel || 'claude-sonnet-4-6'); -const fixture = JSON.parse(await readFile(join(FIXTURES_DIR, fixtureName, 'fixture.json'), 'utf-8')); +const fixtureSource = await readFile(join(fixtureDir, 'fixture.json'), 'utf-8'); +const fixture = JSON.parse(fixtureSource); +const captureConfig = fixture.evidenceCapture || fixture.renderedQuality || {}; if (!fixture.runtime) throw new Error(`fixture ${fixtureName} has no runtime configuration`); if (fixture.runtime.mode === 'insert') throw new Error('live benchmark currently measures replace-mode fixtures only'); if (judgeRendered && !artifactRoot) throw new Error('--judge-rendered requires --artifacts='); +if (judgeRendered && evidenceRoot) { + throw new Error('--evidence-bundle is rubric-free; run rendered quality evaluation in the external eval harness'); +} if (judgeRendered && acceptVariant) throw new Error('--judge-rendered requires complete variants; omit --accept-first/--accept-variant'); if (judgeRendered && fixture.renderedQuality?.remoteSafe !== true) { throw new Error(`fixture ${fixtureName} is not explicitly remote-safe for rendered judging`); @@ -89,6 +100,7 @@ try { session = await bootFixtureSession({ name: fixtureName, fixture, + fixtureRoot: fixtureDir, browser, agent: agentInfo.agent, startWorker: agentInfo.startWorker, @@ -101,8 +113,8 @@ try { log: args.quiet ? () => {} : (message) => process.stderr.write(`[live-bench] ${message}\n`), }); - if (fixture.renderedQuality?.viewport) { - await session.page.setViewportSize(fixture.renderedQuality.viewport); + if (captureConfig.viewport) { + await session.page.setViewportSize(captureConfig.viewport); } recorder.mark('setup.handshake.start'); @@ -289,9 +301,15 @@ try { simulation: simulatedTailMs > 0 ? { remainingGenerationMs: simulatedTailMs } : null, }); report.benchmark.interactionMode = interactionMode; + report.benchmark.fixtureOrigin = fixtureOrigin; + report.benchmark.fixtureConfigSha256 = createHash('sha256').update(fixtureSource).digest('hex'); + report.benchmark.action = renderedContext?.action || (args.action ? String(args.action) : null); if (artifactRoot) report.artifacts = { - root: artifactRoot.startsWith(`${ROOT}${sep}`) ? relative(ROOT, artifactRoot) : null, - externalRoot: !artifactRoot.startsWith(`${ROOT}${sep}`), + kind: 'impeccable-live-evidence', + schemaVersion: 1, + root: evidenceRoot ? '.' : artifactRoot.startsWith(`${ROOT}${sep}`) ? relative(ROOT, artifactRoot) : null, + externalRoot: evidenceRoot ? false : !artifactRoot.startsWith(`${ROOT}${sep}`), + report: evidenceRoot ? 'report.json' : null, screenshotScope: renderedContext.captureSelector, }; if (judgeRendered) { diff --git a/scripts/lib/live-benchmark.mjs b/scripts/lib/live-benchmark.mjs index 2c2ae966b..7b48c116d 100644 --- a/scripts/lib/live-benchmark.mjs +++ b/scripts/lib/live-benchmark.mjs @@ -1,4 +1,5 @@ import { performance } from 'node:perf_hooks'; +import { basename, join, resolve } from 'node:path'; const METRIC_KEYS = [ 'browserPreparationMs', @@ -46,6 +47,45 @@ export function parseLiveBenchmarkArgs(argv) { return out; } +/** + * Resolve the benchmark's fixture and output paths without coupling private + * evaluation fixtures to this repository. `--evidence-bundle` is deliberately + * rubric-free: it packages screenshots and timings for an external evaluator + * without embedding a quality judge or secret task corpus in the public repo. + */ +export function resolveLiveBenchmarkPaths(args, { root, fixturesDir }) { + const evidenceRoot = args.evidenceBundle + ? resolve(root, String(args.evidenceBundle)) + : null; + if (evidenceRoot && args.artifacts) { + throw new Error('--evidence-bundle replaces --artifacts'); + } + if (evidenceRoot && args.output) { + throw new Error('--evidence-bundle writes report.json itself; omit --output'); + } + if (evidenceRoot && args.append) { + throw new Error('--evidence-bundle represents one portable run; omit --append'); + } + + const explicitFixtureDir = args.fixtureDir + ? resolve(root, String(args.fixtureDir)) + : null; + const fixtureName = String(args.fixture || (explicitFixtureDir ? basename(explicitFixtureDir) : 'vite8-react-plain')); + const fixtureDir = explicitFixtureDir || join(fixturesDir, fixtureName); + return { + fixtureName, + fixtureDir, + fixtureOrigin: explicitFixtureDir ? 'external' : 'repository', + evidenceRoot, + artifactRoot: evidenceRoot || (args.artifacts ? resolve(root, String(args.artifacts)) : null), + outputPath: evidenceRoot + ? join(evidenceRoot, 'report.json') + : args.output + ? resolve(root, String(args.output)) + : null, + }; +} + export function createTraceRecorder(now = () => performance.now()) { const events = []; return { diff --git a/scripts/lib/live-rendered-quality.mjs b/scripts/lib/live-rendered-quality.mjs index d08f3c0bc..c68bde3c7 100644 --- a/scripts/lib/live-rendered-quality.mjs +++ b/scripts/lib/live-rendered-quality.mjs @@ -28,7 +28,11 @@ export function buildRenderedJudgePrompt({ action, brief, safeContext = {}, vari } export function buildRenderedReviewContext({ fixture, fixtureConfig, action, brief } = {}) { - const configured = fixtureConfig?.renderedQuality || {}; + // `evidenceCapture` is the neutral public contract used by external eval + // harnesses. `renderedQuality` remains the backwards-compatible local smoke + // judge configuration; it may carry rubric context that evidence bundles do + // not need or expose. + const configured = fixtureConfig?.evidenceCapture || fixtureConfig?.renderedQuality || {}; const selectedAction = String(action || configured.action || 'impeccable'); return { action: selectedAction, diff --git a/tests/framework-fixtures/README.md b/tests/framework-fixtures/README.md index 1a2bc4c64..662aa842e 100644 --- a/tests/framework-fixtures/README.md +++ b/tests/framework-fixtures/README.md @@ -120,3 +120,38 @@ When `preActions` is omitted, steer smoke inherits `runtime.preActions` to revea | `nuxt-csp/` | Nuxt `routeRules` with literal CSP header in `nuxt.config.ts`. CSP shape `append-string`. | Add new fixtures by cloning a directory, swapping files, and updating `fixture.json`. + +## External quality-eval fixtures + +The public Live benchmark can execute a fixture owned by another repository +without copying its task corpus or rubric into Impeccable: + +```sh +bun run bench:live -- \ + --fixture-dir=/absolute/path/to/private-fixture \ + --agent=codex \ + --action=bolder \ + --iterations=1 \ + --evidence-bundle=/absolute/path/to/output-bundle +``` + +An external fixture has the same shape as a directory in this folder: +`fixture.json`, `gitignore.txt`, and `files/`. Use the optional +`evidenceCapture` block in `fixture.json` for rubric-free capture metadata: + +```json +{ + "evidenceCapture": { + "captureSelector": "section.case-study", + "viewport": { "width": 1440, "height": 1080 }, + "action": "bolder" + } +} +``` + +The bundle contains `report.json`, the original capture, each progressively +delivered variant capture, geometry/overflow facts, hashes, and timing data. +It deliberately cannot run `--judge-rendered`; comparative rubrics, private +fixtures, human calibration, and quality decisions belong in the consuming +evaluation harness. The normal public E2E suite remains responsible for Live +protocol, framework, source-commit, cleanup, and recovery correctness. diff --git a/tests/live-benchmark.test.mjs b/tests/live-benchmark.test.mjs index a49bbf1d9..c21ab7c31 100644 --- a/tests/live-benchmark.test.mjs +++ b/tests/live-benchmark.test.mjs @@ -9,6 +9,7 @@ import { deriveJournalGenerationMetrics, durationBetween, parseLiveBenchmarkArgs, + resolveLiveBenchmarkPaths, summarizeRuns, } from '../scripts/lib/live-benchmark.mjs'; @@ -27,6 +28,36 @@ describe('live benchmark metrics', () => { }); }); + it('resolves an external fixture into a portable rubric-free evidence bundle', () => { + const paths = resolveLiveBenchmarkPaths({ + fixtureDir: '../impeccable-evals/fixtures/live/tidewater', + evidenceBundle: '/tmp/live-tidewater', + }, { + root: '/workspace/impeccable', + fixturesDir: '/workspace/impeccable/tests/framework-fixtures', + }); + assert.deepEqual(paths, { + fixtureName: 'tidewater', + fixtureDir: '/workspace/impeccable-evals/fixtures/live/tidewater', + fixtureOrigin: 'external', + evidenceRoot: '/tmp/live-tidewater', + artifactRoot: '/tmp/live-tidewater', + outputPath: '/tmp/live-tidewater/report.json', + }); + }); + + it('keeps evidence bundles atomic and unambiguous', () => { + const options = { root: '/repo', fixturesDir: '/repo/tests/framework-fixtures' }; + assert.throws( + () => resolveLiveBenchmarkPaths({ evidenceBundle: 'bundle', artifacts: 'shots' }, options), + /replaces --artifacts/, + ); + assert.throws( + () => resolveLiveBenchmarkPaths({ evidenceBundle: 'bundle', output: 'report.json' }, options), + /writes report.json itself/, + ); + }); + it('derives production worker phases from the durable session journal', () => { const metrics = deriveJournalGenerationMetrics({ generationTimings: { diff --git a/tests/live-e2e/session.mjs b/tests/live-e2e/session.mjs index 8e36cff8a..00973a1e5 100644 --- a/tests/live-e2e/session.mjs +++ b/tests/live-e2e/session.mjs @@ -32,8 +32,7 @@ export { SCRIPTS_DIR, FIXTURES_DIR, REPO_ROOT }; // Stage // --------------------------------------------------------------------------- -export function stageFixture(name, fixture) { - const fixtureRoot = join(FIXTURES_DIR, name); +export function stageFixture(name, fixture, { fixtureRoot = join(FIXTURES_DIR, name) } = {}) { const gitignore = readFileSync(join(fixtureRoot, 'gitignore.txt'), 'utf-8'); const tmp = mkdtempSync(join(tmpdir(), 'impeccable-e2e-')); @@ -216,6 +215,7 @@ export async function stopDevServer(child) { * @param {object} opts * @param {string} opts.name fixture name * @param {object} opts.fixture fixture.json contents + * @param {string=} opts.fixtureRoot fixture directory; defaults to the public framework fixture tree * @param {import('playwright').Browser} opts.browser shared browser instance * @param {object} opts.agent VariantAgent (defaults to fake) * @param {object|function=} opts.wrapTarget live-wrap target or event mapper @@ -228,6 +228,7 @@ export async function stopDevServer(child) { export async function bootFixtureSession({ name, fixture, + fixtureRoot, browser, agent, wrapTarget, @@ -244,7 +245,7 @@ export async function bootFixtureSession({ const runtime = fixture.runtime; if (!runtime) throw new Error(`fixture ${name} has no runtime block`); - const tmp = stageFixture(name, fixture); + const tmp = stageFixture(name, fixture, { fixtureRoot }); let live; let dev; let agentAbort; diff --git a/tests/live-rendered-quality.test.mjs b/tests/live-rendered-quality.test.mjs index 159332bef..a1f34fbad 100644 --- a/tests/live-rendered-quality.test.mjs +++ b/tests/live-rendered-quality.test.mjs @@ -47,6 +47,26 @@ describe('Live rendered quality judge', () => { assert.equal(context.safeContext.componentRoles.ActionLink, 'Quiet outlined control'); }); + it('prefers rubric-free evidence capture settings for external harnesses', () => { + const context = buildRenderedReviewContext({ + fixture: 'private-fixture', + fixtureConfig: { + runtime: { pickSelector: '.picked' }, + evidenceCapture: { + captureSelector: '.selected-section', + action: 'bolder', + }, + renderedQuality: { + captureSelector: '.public-smoke-only', + reviewFocus: 'Must not leak into the evidence contract.', + }, + }, + }); + assert.equal(context.captureSelector, '.selected-section'); + assert.equal(context.action, 'bolder'); + assert.equal(context.safeContext.reviewFocus, ''); + }); + it('requires every expected rendered variant to pass the strict score floor', () => { const result = parseRenderedJudgeResult(JSON.stringify({ variants: [