/** * Oracle harness: records the observable behavior of every impeccable verb * (stdout, stderr, exit code, files written) against a fixed corpus, and * replays the same corpus against an alternate implementation to diff. * * Two implementations are addressable: * - js (default): the Node scripts in skill/scripts and cli/bin * - bin: an executable at $IMPECCABLE_BIN invoked as ` ...args` * * A case is { id, verb, args, cwd?, stdin?, env?, files?, workspace? }: * - workspace: name of a dir under tests/oracle/workspaces to copy into a * temp dir and use as cwd (so writes never touch the repo) * - cwd: subpath inside the staged workspace (default '.') * - files: globs (relative to staged workspace) to snapshot after the run * - args may contain and placeholders * * Normalization replaces the staged workspace path with , the repo root * with , $HOME with , and masks ISO timestamps, so goldens are * stable across machines and runs. */ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { spawnSync } from 'node:child_process'; import { fileURLToPath, pathToFileURL } from 'node:url'; export const ORACLE_DIR = path.dirname(fileURLToPath(import.meta.url)); export const REPO_ROOT = path.resolve(ORACLE_DIR, '..', '..'); export const GOLDEN_DIR = path.join(ORACLE_DIR, 'golden'); export const CASES_DIR = path.join(ORACLE_DIR, 'cases'); export const WORKSPACES_DIR = path.join(ORACLE_DIR, 'workspaces'); /** verb -> how the JS implementation is invoked */ export const JS_VERBS = { detect: ['node', path.join(REPO_ROOT, 'cli', 'bin', 'cli.js'), 'detect'], 'cli-help': ['node', path.join(REPO_ROOT, 'cli', 'bin', 'cli.js'), '--help'], 'cli-version': ['node', path.join(REPO_ROOT, 'cli', 'bin', 'cli.js'), '--version'], ignores: ['node', path.join(REPO_ROOT, 'cli', 'bin', 'cli.js'), 'ignores'], }; for (const script of [ 'context', 'doctor', 'pin', 'surface-brief', 'critique-storage', 'palette', 'embed-prompt', 'context-signals', 'detect-csp', 'concept-seed', 'hook', 'hook-before-edit', 'hook-admin', 'live', 'live-server', 'live-poll', 'live-status', 'live-resume', 'live-complete', 'live-accept', 'live-wrap', 'live-insert', 'live-inject', 'live-target', 'live-commit-manual-edits', 'live-discard-manual-edits', 'live-manual-edit-evidence', ]) { JS_VERBS[script] = ['node', path.join(REPO_ROOT, 'skill', 'scripts', `${script}.mjs`)]; } /** verb -> argv for the binary implementation (verb name is the subcommand) */ export function binArgv(bin, verb) { if (verb === 'cli-help') return [bin, '--help']; if (verb === 'cli-version') return [bin, '--version']; return [bin, verb]; } export async function allCases() { const out = []; for (const f of fs.readdirSync(CASES_DIR).sort()) { if (!f.endsWith('.mjs')) continue; const mod = await import(pathToFileURL(path.join(CASES_DIR, f)).href); const list = typeof mod.default === 'function' ? await mod.default() : mod.default; for (const item of Array.isArray(list) ? list : [list]) out.push({ ...item, sourceFile: f }); } const seen = new Set(); for (const c of out) { if (seen.has(c.id)) throw new Error(`duplicate oracle case id: ${c.id}`); seen.add(c.id); } return out; } export function stageWorkspace(name) { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-oracle-')); if (name) { const src = path.join(WORKSPACES_DIR, name); if (!fs.existsSync(src)) throw new Error(`oracle workspace not found: ${name}`); fs.cpSync(src, tmp, { recursive: true }); } return tmp; } export function normalize(text, { ws, home = os.homedir() }) { if (typeof text !== 'string') return text; let out = text; let wsReal = null; try { wsReal = ws ? fs.realpathSync(ws) : null; } catch { /* staged dir already gone */ } for (const [needle, tag] of [ [wsReal, ''], [ws, ''], [REPO_ROOT, ''], [home, ''], ]) { if (needle) out = out.split(needle).join(tag); } // ISO timestamps and epoch millis are run-dependent. out = out.replace(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z/g, ''); out = out.replace(/"(updatedAt|createdAt|checkedAt|lastCheck|lastChecked|timestamp|ts|mtimeMs|mtime|startedAt|endedAt)":\s*\d{10,}/g, '"$1": '); return out; } function globToRegex(glob) { let re = ""; for (let i = 0; i < glob.length; i++) { const ch = glob[i]; if (ch === "*") { if (glob[i + 1] === "*") { i++; if (glob[i + 1] === "/") { i++; re += "(?:.*/)?"; } else re += ".*"; } else re += "[^/]*"; } else if (ch === "?") re += "[^/]"; else re += ch.replace(/[.+^${}()|[\]\\]/g, "\\$&"); } return new RegExp("^" + re + "$"); } export function snapshotFiles(ws, globs) { const out = {}; if (!globs || !globs.length) return out; const regs = globs.map(globToRegex); const walk = (dir) => { for (const ent of fs.readdirSync(dir, { withFileTypes: true })) { const full = path.join(dir, ent.name); const rel = path.relative(ws, full).split(path.sep).join('/'); if (ent.isDirectory()) { if (ent.name === 'node_modules' || ent.name === '.git') continue; walk(full); } else if (regs.some(r => r.test(rel))) { const buf = fs.readFileSync(full); out[rel] = isProbablyText(buf) ? buf.toString('utf8') : ``; } } }; walk(ws); return Object.fromEntries(Object.entries(out).sort(([a], [b]) => a.localeCompare(b))); } function isProbablyText(buf) { const n = Math.min(buf.length, 512); for (let i = 0; i < n; i++) if (buf[i] === 0) return false; return true; } /** * Run one case with the given implementation ('js' | 'bin'). * Returns { stdout, stderr, exit, signal, files } normalized. */ export function runCase(c, { impl = 'js', bin = process.env.IMPECCABLE_BIN } = {}) { const ws = stageWorkspace(c.workspace); try { const cwd = path.join(ws, c.cwd || '.'); let argv; if (impl === 'js') { const base = JS_VERBS[c.verb]; if (!base) throw new Error(`no JS invocation for verb ${c.verb}`); argv = [...base]; } else { if (!bin) throw new Error('IMPECCABLE_BIN not set'); argv = binArgv(bin, c.verb); } const args = (c.args || []).map(a => String(a).replaceAll('', ws).replaceAll('', REPO_ROOT)); argv.push(...args); const isolatedHome = path.join(ws, '.oracle-home'); const env = { ...process.env, NO_COLOR: '1', FORCE_COLOR: '0', IMPECCABLE_NO_UPDATE_CHECK: '1', IMPECCABLE_NO_TELEMETRY: '1', DO_NOT_TRACK: '1', ...(c.isolateHome === false ? {} : { HOME: isolatedHome, USERPROFILE: isolatedHome }), ...(c.env || {}), }; if (c.isolateHome !== false) fs.mkdirSync(isolatedHome, { recursive: true }); const res = spawnSync(argv[0], argv.slice(1), { cwd, env, input: c.stdin ?? '', encoding: 'utf8', timeout: c.timeoutMs || 60_000, windowsHide: true, maxBuffer: 64 * 1024 * 1024, }); const files = snapshotFiles(ws, c.files); const ctx = { ws }; return { stdout: normalize(res.stdout ?? '', ctx), stderr: normalize(res.stderr ?? '', ctx), exit: res.status, signal: res.signal || null, files: Object.fromEntries(Object.entries(files).map(([k, v]) => [k, normalize(v, ctx)])), }; } finally { fs.rmSync(ws, { recursive: true, force: true }); } } export function goldenPath(id) { return path.join(GOLDEN_DIR, `${id}.json`); } export function writeGolden(id, result) { fs.mkdirSync(GOLDEN_DIR, { recursive: true }); fs.writeFileSync(goldenPath(id), JSON.stringify(result, null, 2) + '\n'); } export function readGolden(id) { const p = goldenPath(id); if (!fs.existsSync(p)) return null; return JSON.parse(fs.readFileSync(p, 'utf8')); } /** Return a list of human-readable differences, empty if equal. */ export function diffResults(golden, actual) { const diffs = []; for (const k of ['exit', 'signal']) { if (golden[k] !== actual[k]) diffs.push(`${k}: expected ${golden[k]}, got ${actual[k]}`); } for (const k of ['stdout', 'stderr']) { if (golden[k] !== actual[k]) diffs.push(`${k} differs:\n${firstDiff(golden[k], actual[k])}`); } const keys = new Set([...Object.keys(golden.files || {}), ...Object.keys(actual.files || {})]); for (const k of [...keys].sort()) { const g = golden.files?.[k], a = actual.files?.[k]; if (g === undefined) diffs.push(`file ${k}: unexpected (written by actual only)`); else if (a === undefined) diffs.push(`file ${k}: missing (golden has it)`); else if (g !== a) diffs.push(`file ${k} differs:\n${firstDiff(g, a)}`); } return diffs; } function firstDiff(a, b) { const al = String(a).split('\n'), bl = String(b).split('\n'); const n = Math.max(al.length, bl.length); for (let i = 0; i < n; i++) { if (al[i] !== bl[i]) { return ` line ${i + 1}\n - ${JSON.stringify(al[i] ?? '')}\n + ${JSON.stringify(bl[i] ?? '')}`; } } return ' (lengths differ)'; }