Calibrate Live rendered quality review

scripts/benchmark-live.mjs

@@ -35,6 +35,7 @@ import {
 } from './lib/live-benchmark.mjs';
 import { loadBenchmarkEnv } from './lib/live-provider-benchmark.mjs';
 import {
+  buildRenderedReviewContext,
   judgeRenderedVariants,
   summarizeRenderedJudgeRuns,
 } from './lib/live-rendered-quality.mjs';
@@ -120,7 +121,12 @@ try {
   const runs = [];
   const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title';
   const renderedContext = artifactRoot || judgeRendered
-    ? readRenderedContext(fixture, { fixtureName, action: args.action })
+    ? buildRenderedReviewContext({
+        fixture: fixtureName,
+        fixtureConfig: fixture,
+        action: args.action,
+        brief: args.brief,
+      })
     : null;
   for (let iteration = 1; iteration <= iterations; iteration += 1) {
     const runArtifactDir = artifactRoot
@@ -282,24 +288,6 @@ async function readGenerationSnapshot(tmp, eventId) {
   try { return JSON.parse(await readFile(file, 'utf-8')); } catch { return {}; }
 }
 
-function readRenderedContext(currentFixtureConfig, { fixtureName: currentFixture, action }) {
-  const configured = currentFixtureConfig.renderedQuality || {};
-  const selectedAction = String(action || 'impeccable');
-  const briefs = {
-    'vite8-react-brand-fidelity:bolder': 'Make the Field Notes offer card materially bolder. Keep it unmistakably Northstar: amplify hierarchy, proportion, and composition inside the existing design system. Preserve every word and the ActionLink component.',
-  };
-  return {
-    action: selectedAction,
-    brief: String(args.brief || configured.brief || briefs[`${currentFixture}:${selectedAction}`] || `Apply /${selectedAction} to the selected element while preserving its project identity and functional contract.`),
-    captureSelector: String(configured.captureSelector || currentFixtureConfig.runtime.pickSelector || 'body'),
-    safeContext: {
-      fixture: currentFixture,
-      reviewFocus: String(configured.reviewFocus || ''),
-      constraints: Array.isArray(configured.constraints) ? configured.constraints.map(String) : [],
-    },
-  };
-}
-
 async function captureRenderedElement(page, { filePath, selector = null, variantId = null }) {
   const geometry = await page.evaluate(({ targetSelector, targetVariantId }) => {
     const wrapper = targetVariantId == null ? null : document.querySelector('[data-impeccable-variants]');

scripts/judge-live-rendered.mjs

@@ -0,0 +1,110 @@
+#!/usr/bin/env node
+
+import { readdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import Anthropic from '@anthropic-ai/sdk';
+
+import { FIXTURES_DIR } from '../tests/live-e2e/session.mjs';
+import { loadBenchmarkEnv } from './lib/live-provider-benchmark.mjs';
+import {
+  buildRenderedReviewContext,
+  judgeRenderedVariants,
+  summarizeRenderedJudgeRuns,
+} from './lib/live-rendered-quality.mjs';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const args = parseArgs(process.argv.slice(2));
+const artifactRoot = resolve(ROOT, required(args, 'artifacts'));
+const fixtureName = String(args.fixture || 'vite8-react-brand-fidelity');
+const fixture = JSON.parse(await readFile(join(FIXTURES_DIR, fixtureName, 'fixture.json'), 'utf-8'));
+if (fixture.renderedQuality?.remoteSafe !== true) throw new Error(`fixture ${fixtureName} is not explicitly remote-safe`);
+
+loadBenchmarkEnv({ repoRoot: ROOT, explicitPath: args.envFile ? resolve(String(args.envFile)) : null });
+if (!process.env.ANTHROPIC_API_KEY) throw new Error('ANTHROPIC_API_KEY is required');
+
+const review = buildRenderedReviewContext({
+  fixture: fixtureName,
+  fixtureConfig: fixture,
+  action: args.action,
+  brief: args.brief,
+});
+const model = String(args.model || 'claude-sonnet-4-6');
+const scenario = String(args.scenario || 'plain');
+const scenarioRoot = join(artifactRoot, scenario);
+const runNames = (await readdir(scenarioRoot, { withFileTypes: true }))
+  .filter((entry) => entry.isDirectory() && /^run-\d+$/.test(entry.name))
+  .map((entry) => entry.name)
+  .sort();
+if (runNames.length === 0) throw new Error(`no rendered runs found under ${scenarioRoot}`);
+
+const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
+const runs = [];
+for (const runName of runNames) {
+  const runRoot = join(scenarioRoot, runName);
+  const variants = [1, 2, 3].map((variantId) => ({
+    variantId,
+    path: join(runRoot, `variant-${variantId}.png`),
+  }));
+  process.stderr.write(`[live-rendered-judge] ${runName}\n`);
+  runs.push({
+    run: runName,
+    renderedJudge: await judgeRenderedVariants({
+      client,
+      model,
+      action: review.action,
+      brief: review.brief,
+      safeContext: review.safeContext,
+      originalPath: join(runRoot, 'original.png'),
+      variants,
+    }),
+  });
+}
+
+const report = {
+  schemaVersion: 1,
+  generatedAt: new Date().toISOString(),
+  fixture: fixtureName,
+  scenario,
+  model,
+  artifacts: artifactRoot,
+  review,
+  summary: summarizeRenderedJudgeRuns(runs),
+  runs,
+};
+const json = `${JSON.stringify(report, null, 2)}\n`;
+if (args.output) await writeFile(resolve(ROOT, String(args.output)), json, 'utf-8');
+process.stdout.write(json);
+
+function required(values, key) {
+  const value = values[key];
+  if (!value) throw new Error(`--${key}=<value> is required`);
+  return String(value);
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (!arg.startsWith('--')) continue;
+    const equals = arg.indexOf('=');
+    if (equals !== -1) {
+      out[toCamel(arg.slice(2, equals))] = arg.slice(equals + 1);
+      continue;
+    }
+    const key = toCamel(arg.slice(2));
+    const next = argv[index + 1];
+    if (next && !next.startsWith('--')) {
+      out[key] = next;
+      index += 1;
+    } else {
+      out[key] = true;
+    }
+  }
+  return out;
+}
+
+function toCamel(value) {
+  return value.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase());
+}

scripts/lib/live-rendered-quality.mjs

@@ -16,6 +16,7 @@ export function buildRenderedJudgePrompt({ action, brief, safeContext = {}, vari
     'Use integer scores from 1-10. A 7 means clearly shippable and materially improved. Mark criticalFailure for illegible, broken, clipped, off-brand, generic-AI, or task-contradicting output.',
     'Judge every supplied variant independently. Do not reward novelty that violates the existing identity.',
     'Treat the remote-safe constraints as authoritative. Never call a color, typeface, component, or primitive off-system when the constraints explicitly allow it, even if its rendered hue has another everyday name.',
+    'A palette allowlist permits those colors in any visually sound role unless the constraints explicitly restrict a role. Do not infer dark-ink-only typography, no filled surfaces, or no brass rules from a general palette list.',
     'Do not invent prohibitions from adjectives such as restrained, editorial, bold, or quiet. If an allowed primitive is used poorly, score that under renderedQuality or commandFidelity and describe the actual visual problem; do not misreport it as a system violation.',
     'Use the original screenshot as evidence for established roles, but allow the requested action to materially change hierarchy, proportion, composition, and the placement of explicitly allowed colors.',
     '',
@@ -26,6 +27,23 @@ export function buildRenderedJudgePrompt({ action, brief, safeContext = {}, vari
   ].join('\n');
 }
 
+export function buildRenderedReviewContext({ fixture, fixtureConfig, action, brief } = {}) {
+  const configured = fixtureConfig?.renderedQuality || {};
+  const selectedAction = String(action || configured.action || 'impeccable');
+  return {
+    action: selectedAction,
+    brief: String(brief || configured.brief || `Apply /${selectedAction} to the selected element while preserving its project identity and functional contract.`),
+    captureSelector: String(configured.captureSelector || fixtureConfig?.runtime?.pickSelector || 'body'),
+    safeContext: {
+      fixture: String(fixture || ''),
+      reviewFocus: String(configured.reviewFocus || ''),
+      constraints: Array.isArray(configured.constraints) ? configured.constraints.map(String) : [],
+      tokens: sanitizeReviewObject(configured.tokens),
+      componentRoles: sanitizeReviewObject(configured.componentRoles),
+    },
+  };
+}
+
 export async function judgeRenderedVariants({
   client,
   model = 'claude-sonnet-4-6',
@@ -140,3 +158,10 @@ function normalizeUsage(usage) {
 function round(value) {
   return Math.round(value * 100) / 100;
 }
+
+function sanitizeReviewObject(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return {};
+  return Object.fromEntries(Object.entries(value)
+    .filter(([key, entry]) => key.length <= 80 && (typeof entry === 'string' || typeof entry === 'number' || typeof entry === 'boolean'))
+    .map(([key, entry]) => [String(key), entry]));
+}