Export portable Live evidence bundles

AI-assisted implementation under maintainer direction.

scripts/benchmark-live.mjs

@@ -32,6 +32,7 @@ import {
   deriveJournalGenerationMetrics,
   mergeBenchmarkReports,
   parseLiveBenchmarkArgs,
+  resolveLiveBenchmarkPaths,
 } from './lib/live-benchmark.mjs';
 import { loadBenchmarkEnv } from './lib/live-provider-benchmark.mjs';
 import {
@@ -42,7 +43,14 @@ import {
 
 const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
 const args = parseLiveBenchmarkArgs(process.argv.slice(2));
-const fixtureName = String(args.fixture || 'vite8-react-plain');
+const {
+  fixtureName,
+  fixtureDir,
+  fixtureOrigin,
+  evidenceRoot,
+  artifactRoot,
+  outputPath,
+} = resolveLiveBenchmarkPaths(args, { root: ROOT, fixturesDir: FIXTURES_DIR });
 const iterations = positiveInt(args.iterations, 5);
 const agentMode = args.agent === 'codex' ? 'codex' : args.agent === 'llm' ? 'llm' : 'fake';
 const scenario = args.scenario === 'annotated' ? 'annotated' : 'plain';
@@ -57,14 +65,17 @@ const interactionMode = acceptVariant
   ? `accept-variant-${acceptVariant}-then-next-go`
   : 'complete-then-discard';
 const simulatedTailMs = positiveInt(args.simulatedTailMs, 0);
-const outputPath = args.output ? resolve(ROOT, String(args.output)) : null;
-const artifactRoot = args.artifacts ? resolve(ROOT, String(args.artifacts)) : null;
 const judgeRendered = args.judgeRendered === true || args.judgeRendered === 'true';
 const judgeModel = String(args.judgeModel || 'claude-sonnet-4-6');
-const fixture = JSON.parse(await readFile(join(FIXTURES_DIR, fixtureName, 'fixture.json'), 'utf-8'));
+const fixtureSource = await readFile(join(fixtureDir, 'fixture.json'), 'utf-8');
+const fixture = JSON.parse(fixtureSource);
+const captureConfig = fixture.evidenceCapture || fixture.renderedQuality || {};
 if (!fixture.runtime) throw new Error(`fixture ${fixtureName} has no runtime configuration`);
 if (fixture.runtime.mode === 'insert') throw new Error('live benchmark currently measures replace-mode fixtures only');
 if (judgeRendered && !artifactRoot) throw new Error('--judge-rendered requires --artifacts=<directory>');
+if (judgeRendered && evidenceRoot) {
+  throw new Error('--evidence-bundle is rubric-free; run rendered quality evaluation in the external eval harness');
+}
 if (judgeRendered && acceptVariant) throw new Error('--judge-rendered requires complete variants; omit --accept-first/--accept-variant');
 if (judgeRendered && fixture.renderedQuality?.remoteSafe !== true) {
   throw new Error(`fixture ${fixtureName} is not explicitly remote-safe for rendered judging`);
@@ -89,6 +100,7 @@ try {
   session = await bootFixtureSession({
     name: fixtureName,
     fixture,
+    fixtureRoot: fixtureDir,
     browser,
     agent: agentInfo.agent,
     startWorker: agentInfo.startWorker,
@@ -101,8 +113,8 @@ try {
     log: args.quiet ? () => {} : (message) => process.stderr.write(`[live-bench] ${message}\n`),
   });
 
-  if (fixture.renderedQuality?.viewport) {
+  if (captureConfig.viewport) {
-    await session.page.setViewportSize(fixture.renderedQuality.viewport);
+    await session.page.setViewportSize(captureConfig.viewport);
   }
 
   recorder.mark('setup.handshake.start');
@@ -289,9 +301,15 @@ try {
     simulation: simulatedTailMs > 0 ? { remainingGenerationMs: simulatedTailMs } : null,
   });
   report.benchmark.interactionMode = interactionMode;
+  report.benchmark.fixtureOrigin = fixtureOrigin;
+  report.benchmark.fixtureConfigSha256 = createHash('sha256').update(fixtureSource).digest('hex');
+  report.benchmark.action = renderedContext?.action || (args.action ? String(args.action) : null);
   if (artifactRoot) report.artifacts = {
-    root: artifactRoot.startsWith(`${ROOT}${sep}`) ? relative(ROOT, artifactRoot) : null,
+    kind: 'impeccable-live-evidence',
-    externalRoot: !artifactRoot.startsWith(`${ROOT}${sep}`),
+    schemaVersion: 1,
+    root: evidenceRoot ? '.' : artifactRoot.startsWith(`${ROOT}${sep}`) ? relative(ROOT, artifactRoot) : null,
+    externalRoot: evidenceRoot ? false : !artifactRoot.startsWith(`${ROOT}${sep}`),
+    report: evidenceRoot ? 'report.json' : null,
     screenshotScope: renderedContext.captureSelector,
   };
   if (judgeRendered) {

scripts/lib/live-benchmark.mjs

@@ -1,4 +1,5 @@
 import { performance } from 'node:perf_hooks';
+import { basename, join, resolve } from 'node:path';
 
 const METRIC_KEYS = [
   'browserPreparationMs',
@@ -46,6 +47,45 @@ export function parseLiveBenchmarkArgs(argv) {
   return out;
 }
 
+/**
+ * Resolve the benchmark's fixture and output paths without coupling private
+ * evaluation fixtures to this repository. `--evidence-bundle` is deliberately
+ * rubric-free: it packages screenshots and timings for an external evaluator
+ * without embedding a quality judge or secret task corpus in the public repo.
+ */
+export function resolveLiveBenchmarkPaths(args, { root, fixturesDir }) {
+  const evidenceRoot = args.evidenceBundle
+    ? resolve(root, String(args.evidenceBundle))
+    : null;
+  if (evidenceRoot && args.artifacts) {
+    throw new Error('--evidence-bundle replaces --artifacts');
+  }
+  if (evidenceRoot && args.output) {
+    throw new Error('--evidence-bundle writes report.json itself; omit --output');
+  }
+  if (evidenceRoot && args.append) {
+    throw new Error('--evidence-bundle represents one portable run; omit --append');
+  }
+
+  const explicitFixtureDir = args.fixtureDir
+    ? resolve(root, String(args.fixtureDir))
+    : null;
+  const fixtureName = String(args.fixture || (explicitFixtureDir ? basename(explicitFixtureDir) : 'vite8-react-plain'));
+  const fixtureDir = explicitFixtureDir || join(fixturesDir, fixtureName);
+  return {
+    fixtureName,
+    fixtureDir,
+    fixtureOrigin: explicitFixtureDir ? 'external' : 'repository',
+    evidenceRoot,
+    artifactRoot: evidenceRoot || (args.artifacts ? resolve(root, String(args.artifacts)) : null),
+    outputPath: evidenceRoot
+      ? join(evidenceRoot, 'report.json')
+      : args.output
+        ? resolve(root, String(args.output))
+        : null,
+  };
+}
+
 export function createTraceRecorder(now = () => performance.now()) {
   const events = [];
   return {

scripts/lib/live-rendered-quality.mjs

@@ -28,7 +28,11 @@ export function buildRenderedJudgePrompt({ action, brief, safeContext = {}, vari
 }
 
 export function buildRenderedReviewContext({ fixture, fixtureConfig, action, brief } = {}) {
-  const configured = fixtureConfig?.renderedQuality || {};
+  // `evidenceCapture` is the neutral public contract used by external eval
+  // harnesses. `renderedQuality` remains the backwards-compatible local smoke
+  // judge configuration; it may carry rubric context that evidence bundles do
+  // not need or expose.
+  const configured = fixtureConfig?.evidenceCapture || fixtureConfig?.renderedQuality || {};
   const selectedAction = String(action || configured.action || 'impeccable');
   return {
     action: selectedAction,

tests/framework-fixtures/README.md

@@ -120,3 +120,38 @@ When `preActions` is omitted, steer smoke inherits `runtime.preActions` to revea
 | `nuxt-csp/` | Nuxt `routeRules` with literal CSP header in `nuxt.config.ts`. CSP shape `append-string`. |
 
 Add new fixtures by cloning a directory, swapping files, and updating `fixture.json`.
+
+## External quality-eval fixtures
+
+The public Live benchmark can execute a fixture owned by another repository
+without copying its task corpus or rubric into Impeccable:
+
+```sh
+bun run bench:live -- \
+  --fixture-dir=/absolute/path/to/private-fixture \
+  --agent=codex \
+  --action=bolder \
+  --iterations=1 \
+  --evidence-bundle=/absolute/path/to/output-bundle
+```
+
+An external fixture has the same shape as a directory in this folder:
+`fixture.json`, `gitignore.txt`, and `files/`. Use the optional
+`evidenceCapture` block in `fixture.json` for rubric-free capture metadata:
+
+```json
+{
+  "evidenceCapture": {
+    "captureSelector": "section.case-study",
+    "viewport": { "width": 1440, "height": 1080 },
+    "action": "bolder"
+  }
+}
+```
+
+The bundle contains `report.json`, the original capture, each progressively
+delivered variant capture, geometry/overflow facts, hashes, and timing data.
+It deliberately cannot run `--judge-rendered`; comparative rubrics, private
+fixtures, human calibration, and quality decisions belong in the consuming
+evaluation harness. The normal public E2E suite remains responsible for Live
+protocol, framework, source-commit, cleanup, and recovery correctness.

tests/live-benchmark.test.mjs

@@ -9,6 +9,7 @@ import {
   deriveJournalGenerationMetrics,
   durationBetween,
   parseLiveBenchmarkArgs,
+  resolveLiveBenchmarkPaths,
   summarizeRuns,
 } from '../scripts/lib/live-benchmark.mjs';
 
@@ -27,6 +28,36 @@ describe('live benchmark metrics', () => {
     });
   });
 
+  it('resolves an external fixture into a portable rubric-free evidence bundle', () => {
+    const paths = resolveLiveBenchmarkPaths({
+      fixtureDir: '../impeccable-evals/fixtures/live/tidewater',
+      evidenceBundle: '/tmp/live-tidewater',
+    }, {
+      root: '/workspace/impeccable',
+      fixturesDir: '/workspace/impeccable/tests/framework-fixtures',
+    });
+    assert.deepEqual(paths, {
+      fixtureName: 'tidewater',
+      fixtureDir: '/workspace/impeccable-evals/fixtures/live/tidewater',
+      fixtureOrigin: 'external',
+      evidenceRoot: '/tmp/live-tidewater',
+      artifactRoot: '/tmp/live-tidewater',
+      outputPath: '/tmp/live-tidewater/report.json',
+    });
+  });
+
+  it('keeps evidence bundles atomic and unambiguous', () => {
+    const options = { root: '/repo', fixturesDir: '/repo/tests/framework-fixtures' };
+    assert.throws(
+      () => resolveLiveBenchmarkPaths({ evidenceBundle: 'bundle', artifacts: 'shots' }, options),
+      /replaces --artifacts/,
+    );
+    assert.throws(
+      () => resolveLiveBenchmarkPaths({ evidenceBundle: 'bundle', output: 'report.json' }, options),
+      /writes report.json itself/,
+    );
+  });
+
   it('derives production worker phases from the durable session journal', () => {
     const metrics = deriveJournalGenerationMetrics({
       generationTimings: {

tests/live-e2e/session.mjs

@@ -32,8 +32,7 @@ export { SCRIPTS_DIR, FIXTURES_DIR, REPO_ROOT };
 // Stage
 // ---------------------------------------------------------------------------
 
-export function stageFixture(name, fixture) {
+export function stageFixture(name, fixture, { fixtureRoot = join(FIXTURES_DIR, name) } = {}) {
-  const fixtureRoot = join(FIXTURES_DIR, name);
   const gitignore = readFileSync(join(fixtureRoot, 'gitignore.txt'), 'utf-8');
 
   const tmp = mkdtempSync(join(tmpdir(), 'impeccable-e2e-'));
@@ -216,6 +215,7 @@ export async function stopDevServer(child) {
  * @param {object} opts
  * @param {string} opts.name              fixture name
  * @param {object} opts.fixture           fixture.json contents
+ * @param {string=} opts.fixtureRoot      fixture directory; defaults to the public framework fixture tree
  * @param {import('playwright').Browser} opts.browser   shared browser instance
  * @param {object} opts.agent             VariantAgent (defaults to fake)
  * @param {object|function=} opts.wrapTarget live-wrap target or event mapper
@@ -228,6 +228,7 @@ export async function stopDevServer(child) {
 export async function bootFixtureSession({
   name,
   fixture,
+  fixtureRoot,
   browser,
   agent,
   wrapTarget,
@@ -244,7 +245,7 @@ export async function bootFixtureSession({
   const runtime = fixture.runtime;
   if (!runtime) throw new Error(`fixture ${name} has no runtime block`);
 
-  const tmp = stageFixture(name, fixture);
+  const tmp = stageFixture(name, fixture, { fixtureRoot });
   let live;
   let dev;
   let agentAbort;

tests/live-rendered-quality.test.mjs

@@ -47,6 +47,26 @@ describe('Live rendered quality judge', () => {
     assert.equal(context.safeContext.componentRoles.ActionLink, 'Quiet outlined control');
   });
 
+  it('prefers rubric-free evidence capture settings for external harnesses', () => {
+    const context = buildRenderedReviewContext({
+      fixture: 'private-fixture',
+      fixtureConfig: {
+        runtime: { pickSelector: '.picked' },
+        evidenceCapture: {
+          captureSelector: '.selected-section',
+          action: 'bolder',
+        },
+        renderedQuality: {
+          captureSelector: '.public-smoke-only',
+          reviewFocus: 'Must not leak into the evidence contract.',
+        },
+      },
+    });
+    assert.equal(context.captureSelector, '.selected-section');
+    assert.equal(context.action, 'bolder');
+    assert.equal(context.safeContext.reviewFocus, '');
+  });
+
   it('requires every expected rendered variant to pass the strict score floor', () => {
     const result = parseRenderedJudgeResult(JSON.stringify({
       variants: [