Tests: gate behavior on the oracle and the engine binary

Unit tests of the deleted Node scripts and the JS detector are removed;
their behavior is pinned by tests/oracle goldens (frozen JS behavior plus
reviewed deltas) and the engine's own tests. tests/oracle.test.mjs replays
the corpus against the binary (IMPECCABLE_BIN or skill/scripts/bin/<target>/,
via tests/lib/engine-bin.mjs) and skips cleanly without one; the framework
fixture sweep drives live-inject, live-wrap, and detect-csp through the
binary the same way. record.mjs learns --bin. The function-level vectors
under tests/oracle/vectors/calls are committed as the frozen snapshot they
can no longer be regenerated from. Suites: core trimmed to build and
transformer tests, oracle added to the default run, detector/live reduced to
packaging and reference checks, the live-e2e helper tests move to the opt-in
live-e2e lane pending its retarget, cli-remote-e2e is an empty placeholder.

Prepared with AI assistance (Claude Code).

scripts/ci-test-plan.mjs

@@ -15,17 +15,31 @@ const changedFiles = localNoChanges || isSchedule ? [] : getChangedFiles();
 const forceDeterministic = localNoChanges || isSchedule || eventName === 'push' || eventName === 'workflow_dispatch';
 const forceOptIn = eventName === 'workflow_dispatch';
 
-const plan = {
-  core: true,
-  detector: forceDeterministic || matchesSuiteTriggers('detector', changedFiles),
-  live: forceDeterministic || matchesSuiteTriggers('live', changedFiles),
-  framework: forceDeterministic || matchesSuiteTriggers('framework', changedFiles),
-  cli_remote_e2e: forceOptIn,
-  live_e2e: isSchedule || forceOptIn || matchesSuiteTriggers('live-e2e', changedFiles),
-  live_e2e_accept_cleanup: forceOptIn || matchesSuiteTriggers('live-e2e-accept-cleanup', changedFiles),
-  skill_behavior: forceOptIn || matchesSuiteTriggers('skill-behavior', changedFiles),
-  live_svelte_adapter_deepseek: forceOptIn || matchesSuiteTriggers('live-svelte-adapter-deepseek', changedFiles),
-};
+const plan = isSchedule
+  ? {
+    core: true,
+    oracle: true,
+    detector: true,
+    live: true,
+    framework: true,
+    cli_remote_e2e: false,
+    live_e2e: true,
+    live_e2e_accept_cleanup: false,
+    skill_behavior: false,
+    live_svelte_adapter_deepseek: false,
+  }
+  : {
+    core: true,
+    oracle: forceDeterministic || matchesSuiteTriggers('oracle', changedFiles),
+    detector: forceDeterministic || matchesSuiteTriggers('detector', changedFiles),
+    live: forceDeterministic || matchesSuiteTriggers('live', changedFiles),
+    framework: forceDeterministic || matchesSuiteTriggers('framework', changedFiles),
+    cli_remote_e2e: forceOptIn,
+    live_e2e: forceOptIn || matchesSuiteTriggers('live-e2e', changedFiles),
+    live_e2e_accept_cleanup: forceOptIn || matchesSuiteTriggers('live-e2e-accept-cleanup', changedFiles),
+    skill_behavior: forceOptIn || matchesSuiteTriggers('skill-behavior', changedFiles),
+    live_svelte_adapter_deepseek: forceOptIn || matchesSuiteTriggers('live-svelte-adapter-deepseek', changedFiles),
+  };
 
 writeGithubOutputs(plan);
 printSummary(plan, changedFiles);

scripts/lib/cli-args.mjs

@@ -1,87 +0,0 @@
-/**
- * One argv parser for the Live benchmark / judging scripts.
- *
- * These scripts had four subtly different hand-rolled parsers, and the gaps
- * failed silently rather than loudly: a parser without the `argv[i + 1]`
- * lookahead turned `--iterations 20` into `iterations: true` and benchmarked
- * the default 5 runs; a parser without kebab→camel mapping turned
- * `--median-target=0.4` into a key nothing read, so the comparison ran against
- * the default threshold. Both produce a clean-looking report of the wrong thing.
- *
- * Supported forms, per flag:
- *   --flag              → true
- *   --flag=value        → 'value'
- *   --flag value        → 'value'   (unless `value` itself starts with `--`)
- *
- * Keys are camel-cased, so `--simulated-tail-ms` and `--simulatedTailMs` both
- * land on `simulatedTailMs`.
- */
-export function parseArgs(argv) {
-  const out = {};
-  for (let index = 0; index < argv.length; index += 1) {
-    const arg = argv[index];
-    if (!arg.startsWith('--')) continue;
-    const body = arg.slice(2);
-    if (!body) continue;
-    const equals = body.indexOf('=');
-    if (equals !== -1) {
-      out[toCamel(body.slice(0, equals))] = body.slice(equals + 1);
-      continue;
-    }
-    const next = argv[index + 1];
-    if (next !== undefined && !next.startsWith('--')) {
-      out[toCamel(body)] = next;
-      index += 1;
-    } else {
-      out[toCamel(body)] = true;
-    }
-  }
-  return out;
-}
-
-export function toCamel(value) {
-  return String(value).replace(/-([a-z0-9])/gi, (_, char) => char.toUpperCase());
-}
-
-/**
- * Read a boolean flag. `--headed` and `--headed=true` must mean the same thing;
- * comparing the raw value against `true` silently ignores the second form.
- */
-export function boolFlag(value, fallback = false) {
-  if (value === undefined) return fallback;
-  if (typeof value === 'boolean') return value;
-  const normalized = String(value).trim().toLowerCase();
-  if (['', 'true', '1', 'yes', 'on'].includes(normalized)) return true;
-  if (['false', '0', 'no', 'off'].includes(normalized)) return false;
-  return fallback;
-}
-
-/**
- * Parse a positive integer flag, falling back when absent. Throws on a value
- * that was clearly meant as a number but isn't one, so `--iterations abc`
- * fails instead of quietly benchmarking the default.
- */
-export function positiveIntFlag(value, fallback) {
-  if (value === undefined || value === true) return fallback;
-  const parsed = Number.parseInt(String(value), 10);
-  if (!Number.isFinite(parsed) || parsed <= 0 || String(parsed) !== String(value).trim()) {
-    throw new Error(`expected a positive integer, got: ${value}`);
-  }
-  return parsed;
-}
-
-/**
- * Resolve a flag that must be one of a fixed set.
- *
- * A silent `x === 'known' ? 'known' : fallback` is the trap this replaces: the
- * private evals Live runner passes `--agent=codex`, which fell through to the
- * canned fake agent and produced a clean-looking report of a deterministic stub
- * labelled as a real harness run. An unrecognized value is a mistake, not a
- * request for the default.
- */
-export function resolveEnum(value, allowed, fallback, flagName) {
-  if (value === undefined || value === true) return fallback;
-  const normalized = String(value).trim().toLowerCase();
-  if (allowed.includes(normalized)) return normalized;
-  throw new Error(`${flagName} must be one of ${allowed.join(', ')}; got: ${value}`);
-}

scripts/test-suites.mjs

@@ -1,7 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 
-export const DEFAULT_SUITES = ['core', 'detector', 'live', 'framework', 'plugin-e2e'];
+export const DEFAULT_SUITES = ['core', 'oracle', 'detector', 'live', 'framework', 'plugin-e2e'];
 export const OPT_IN_SUITES = [
   'cli-remote-e2e',
   'live-e2e',
@@ -22,11 +22,12 @@ const COMMON_INFRA_PATTERNS = [
 
 export const SUITES = {
   core: {
-    description: 'Build, provider transforms, CLI helpers, context, and storage unit tests.',
+    description: 'Build, provider transforms, hook manifests, plugin validators, and prose gates.',
     triggers: [
       ...COMMON_INFRA_PATTERNS,
-      /^scripts\/(?!benchmark-detector|build-browser-detector|build-extension)/,
-      /^skill\/(SKILL\.src\.md|agents\/|reference\/|scripts\/(cleanup-deprecated|comp-diff|comp-spec|build-phase|font-match|data\/font-index|concept-seed|generate-image|context|context-signals|critique-storage|design-parser|doctor|hook|impeccable-paths|is-generated|lib\/(artifact-schema|png|raster|image-metrics|font-fingerprint|font-index|hero-checks|composition-catalog|concept-catalog|provider|staleness|staleness-deep|staleness-notice|surface-briefs|target-slug|template-extensions)|pin|surface-brief))/,
+      /^scripts\/(?!build-browser-detector|build-extension)/,
+      /^skill\/(SKILL\.src\.md|agents\/|reference\/|scripts\/)/,
+      /^ENGINE_VERSION$/,
       /^README(\.npm)?\.md$/,
       /^cli\/bin\//,
     ],
@@ -35,15 +36,11 @@ export const SUITES = {
         runner: 'bun',
         files: [
           'tests/build.test.js',
-          'tests/cli-ignores.test.js',
-          'tests/windows-path-fix.test.js',
           'tests/lib/provider-blocks.test.js',
           'tests/lib/transformers/provider-blocks.test.js',
           'tests/lib/utils.test.js',
-          'tests/lib/impeccable-config.test.js',
           'tests/lib/transformers/factory.test.js',
           'tests/lib/transformers/providers.test.js',
-          'tests/skills-cli.test.js',
           'tests/validate-plugin-versions.test.js',
           'tests/validate-plugin-manifest.test.js',
           'tests/plugin-paths.test.js',
@@ -52,147 +49,89 @@ export const SUITES = {
       {
         runner: 'node',
         files: [
-          'tests/ci-test-plan.test.mjs',
-          'tests/cli-args.test.mjs',
-          'tests/concept-seed.test.mjs',
-          'tests/generate-image-embed.test.mjs',
-          'tests/comp-diff.test.mjs',
           'tests/build-phase.test.mjs',
+          'tests/ci-test-plan.test.mjs',
+          'tests/comp-diff.test.mjs',
           'tests/font-match.test.mjs',
-          'tests/hero-checks.test.mjs',
-          'tests/serve-question.test.mjs',
-          'tests/context.test.mjs',
-          'tests/context-signals.test.mjs',
-          'tests/critique-storage.test.mjs',
-          'tests/design-parser.test.mjs',
           'tests/github-sheriff.test.mjs',
+          'tests/hero-checks.test.mjs',
           'tests/hook-build.test.mjs',
-          'tests/hook.test.mjs',
-          'tests/impeccable-paths.test.mjs',
           'tests/openai-plugin.test.mjs',
-          'tests/pin.test.mjs',
           'tests/release.test.mjs',
-          'tests/doctor.test.mjs',
-          'tests/staleness.test.mjs',
           'tests/skill-reference.test.mjs',
           'tests/readme-gitignore.test.mjs',
-          'tests/target-args.test.mjs',
-          'tests/surface-brief.test.mjs',
-          'tests/template-extensions.test.mjs',
           'tests/test-suites.test.mjs',
-          'tests/zip.test.mjs',
         ],
       },
     ],
   },
+  // The verbs live in the engine binary; this repo pins its behavior with the
+  // oracle goldens (tests/oracle) and drives its live-mode verbs over the
+  // framework fixtures. Both skip when no binary is present (bun run
+  // fetch:engine, or IMPECCABLE_BIN).
+  oracle: {
+    description: 'Oracle corpus replay against the engine binary; skips without a binary.',
+    triggers: [
+      ...COMMON_INFRA_PATTERNS,
+      /^ENGINE_VERSION$/,
+      /^tests\/oracle\//,
+      /^tests\/fixtures\//,
+      /^tests\/lib\/engine-bin\.mjs$/,
+      /^skill\/(reference\/|scripts\/)/,
+    ],
+    commands: [
+      {
+        runner: 'node',
+        timeoutMs: 900000,
+        files: ['tests/oracle.test.mjs'],
+      },
+    ],
+  },
   detector: {
-    description: 'Anti-pattern detector tests across text, jsdom fixtures, and Puppeteer browser paths.',
-    needsPuppeteer: true,
+    description: 'Extension packaging checks (the detector engine itself is tested in the engine repo and by the oracle).',
     triggers: [
       ...COMMON_INFRA_PATTERNS,
-      /^cli\/engine\//,
       /^extension\/(background|content|detector|devtools|popup|manifest\.json)/,
-      /^scripts\/(benchmark-detector|build-browser-detector|build-extension)\.js$/,
-      /^site\/(pages\/detector|public\/antipattern|data\/anti-patterns-catalog\.js)/,
-      /^tests\/fixtures\/antipatterns/,
+      /^scripts\/(build-browser-detector|build-extension)\.js$/,
     ],
     commands: [
-      {
-        runner: 'bun',
-        files: [
-          'tests/detect-antipatterns.test.js',
-          'tests/detect-url-launch.test.mjs',
-          'tests/inline-ignores.test.mjs',
-          'tests/lib/detector-bundle.test.js',
-        ],
-      },
       {
         runner: 'node',
-        files: [
-          'tests/extension-build.test.mjs',
-          'tests/design-system.test.mjs',
-          'tests/detect-antipatterns-fixtures.test.mjs',
-          'tests/detect-antipatterns-browser.test.mjs',
-          'tests/detect-cli-design-contamination.test.mjs',
-          'tests/detect-cli-design-monorepo.test.mjs',
-          'tests/detect-cli-stdin-dispatch.test.mjs',
-        ],
+        files: ['tests/extension-build.test.mjs'],
       },
     ],
   },
   live: {
-    description: 'Fast live-mode unit and local-server integration tests, excluding full browser fixture sweeps.',
+    description: 'Live-mode reference contract checks plus the live-e2e helper units (agent output, CLI options, LLM agent parsing, steer loop against the binary); the live verbs themselves are covered by the oracle and framework suites.',
     triggers: [
       ...COMMON_INFRA_PATTERNS,
-      // `palette` is deliberately absent: skill/scripts/palette.mjs has no
-      // test anywhere, and listing it here made edits run a suite that never
-      // touches it, which reads as coverage that does not exist.
-      /^skill\/(reference\/live\.md|scripts\/(detect-csp|lib\/is-generated|lib\/template-extensions|live\/|live|live-|modern-screenshot|pin))/,
-      /^tests\/live-/,
+      /^skill\/(reference\/live\.md|scripts\/live-browser)/,
+      /^tests\/live-e2e\//,
+      /^tests\/lib\/engine-bin\.mjs$/,
     ],
     commands: [
       {
         runner: 'node',
         files: [
-          'tests/live-accept.test.mjs',
-          'tests/live-accept-css.test.mjs',
-          'tests/live-accept-scrub.test.mjs',
-          'tests/live-browser-dom.test.mjs',
+          'tests/live-reference.test.mjs',
           'tests/live-browser-ignores.test.mjs',
-          'tests/live-browser-script-parts.test.mjs',
-          'tests/live-browser-regression.test.mjs',
-          'tests/live-browser-session.test.mjs',
-          'tests/live-browser-source.test.mjs',
-          'tests/live-commit-manual-edits.test.mjs',
-          'tests/live-completion.test.mjs',
-          'tests/live-copy-edit-agent.test.mjs',
-          'tests/live-discard-manual-edits.test.mjs',
           'tests/live-e2e-agent-output.test.mjs',
           'tests/live-e2e-cli-options.test.mjs',
           'tests/live-e2e-llm-agent.test.mjs',
           'tests/live-e2e-steer-agent.test.mjs',
           'tests/live-e2e/agent-insert.test.mjs',
-          'tests/live-event-validation.test.mjs',
-          'tests/live-frameworks.test.mjs',
-          'tests/live-generation-preflight.test.mjs',
-          'tests/live-inject.test.mjs',
-          'tests/live-insert.test.mjs',
-          'tests/live-insert-ui.test.mjs',
-          'tests/live-manual-edits-buffer.test.mjs',
-          'tests/live-poll.test.mjs',
-          'tests/live-project-ignores.test.mjs',
-          'tests/live-poll-lanes.test.mjs',
-          'tests/live-poll-stream.test.mjs',
-          'tests/live-recovery-commands.test.mjs',
-          'tests/live-reference.test.mjs',
-          'tests/live-roots.test.mjs',
-          'tests/live-server.test.mjs',
-          'tests/live-session-store.test.mjs',
-          'tests/live-source-lock.test.mjs',
-          'tests/live-source-search.test.mjs',
-          'tests/live-svelte-ast.test.mjs',
-          'tests/live-svelte-component-accept.test.mjs',
-          'tests/live-svelte-props-script.test.mjs',
-          'tests/live-tanstack-adapter.test.mjs',
-          'tests/live-target-context.test.mjs',
-          'tests/live-ui-surfaces.test.mjs',
-          'tests/live-wrap.test.mjs',
-          'tests/live-wrap-buffer-aware.test.mjs',
         ],
       },
     ],
   },
   framework: {
-    description: 'Framework fixture coverage for live injection, CSP, generated-file detection, and wrapping.',
+    description: 'Framework fixture coverage for live injection, CSP detection, and wrapping through the engine binary; skips without a binary.',
     triggers: [
       ...COMMON_INFRA_PATTERNS,
+      /^ENGINE_VERSION$/,
       /^tests\/framework-fixtures/,
       /^tests\/framework-fixtures\.test\.mjs$/,
-      /^skill\/scripts\/(detect-csp|live-inject|live-wrap)\.mjs$/,
-      /^skill\/scripts\/lib\/is-generated\.mjs$/,
-      /^skill\/scripts\/lib\/template-extensions\.mjs$/,
-      /^skill\/scripts\/live\/(source-search|sveltekit-adapter|tanstack-adapter)\.mjs$/,
-      /^skill\/scripts\/live\/frameworks\//,
+      /^tests\/lib\/engine-bin\.mjs$/,
     ],
     commands: [
       {
@@ -201,30 +140,14 @@ export const SUITES = {
       },
     ],
   },
-  'cli-e2e': {
-    description: 'Deterministic CLI install/update tests against a local universal bundle.',
-    commands: [
-      {
-        runner: 'bun',
-        files: ['tests/skills-cli.test.js'],
-      },
-    ],
-  },
+  // `impeccable install/update/check` and their remote smoke moved into the
+  // engine binary and its repo; the deterministic coverage here is the oracle
+  // corpus. The lane name stays so ci.yml and package.json keep resolving.
   'cli-remote-e2e': {
-    description: 'Remote CLI install/update smoke tests against impeccable.style.',
+    description: 'Remote CLI install/update smoke (moved to the engine repo; no tests here).',
     optIn: true,
-    triggers: [
-      ...COMMON_INFRA_PATTERNS,
-      /^cli\/bin\/commands\/skills\.mjs$/,
-      /^tests\/skills-cli\.test\.js$/,
-    ],
-    commands: [
-      {
-        runner: 'bun',
-        env: { IMPECCABLE_CLI_REMOTE_E2E: '1' },
-        files: ['tests/skills-cli.test.js'],
-      },
-    ],
+    triggers: [...COMMON_INFRA_PATTERNS],
+    commands: [],
   },
   'plugin-e2e': {
     description: 'Install the committed ./plugin subtree into a real (sandboxed) Claude Code and assert skills, agents, and hooks all load. Skips when the claude CLI is not on PATH.',
@@ -234,7 +157,6 @@ export const SUITES = {
       /^skill\/agents\//,
       /^scripts\/build\.js$/,
       /^scripts\/lib\/validate-plugin-manifest\.js$/,
-      /^scripts\/lib\/plugin-paths\.js$/,
       /^tests\/plugin-e2e\.test\.mjs$/,
     ],
     commands: [
@@ -252,7 +174,8 @@ export const SUITES = {
     needsPlaywright: true,
     triggers: [
       ...COMMON_INFRA_PATTERNS,
-      /^skill\/scripts\/live/,
+      /^skill\/scripts\/live-browser/,
+      /^ENGINE_VERSION$/,
       /^tests\/framework-fixtures/,
       /^tests\/live-e2e(\.test\.mjs|\/)/,
     ],
@@ -271,7 +194,7 @@ export const SUITES = {
     needsPlaywright: true,
     triggers: [
       ...COMMON_INFRA_PATTERNS,
-      /^skill\/scripts\/(serve-question|generate-image|concept-seed)\.mjs$/,
+      /^ENGINE_VERSION$/,
       /^tests\/new-work-e2e(\.test\.mjs|\/)/,
     ],
     commands: [
@@ -289,8 +212,7 @@ export const SUITES = {
     needsPlaywright: true,
     triggers: [
       ...COMMON_INFRA_PATTERNS,
-      /^skill\/scripts\/(live-accept|live-browser|live-server|live-wrap)\.mjs$/,
-      /^skill\/scripts\/live\/sveltekit-adapter\.mjs$/,
+      /^ENGINE_VERSION$/,
       /^tests\/live-e2e-accept-cleanup-regression\.test\.mjs$/,
       /^tests\/live-e2e\//,
     ],
@@ -318,7 +240,7 @@ export const SUITES = {
       ...COMMON_INFRA_PATTERNS,
       /^skill\/SKILL\.src\.md$/,
       /^skill\/reference\/(init|document|brand|product|shape|craft|audit|polish|live)\.md$/,
-      /^skill\/scripts\/(context|context-signals|detect|detect-csp)\.mjs$/,
+      /^ENGINE_VERSION$/,
       /^tests\/skill-behavior\//,
     ],
     commands: [
@@ -346,8 +268,7 @@ export const SUITES = {
     needsPlaywright: true,
     triggers: [
       ...COMMON_INFRA_PATTERNS,
-      /^skill\/scripts\/(live-server|live-wrap)\.mjs$/,
-      /^skill\/scripts\/live\/(sveltekit-adapter|svelte-component)\.mjs$/,
+      /^ENGINE_VERSION$/,
       /^tests\/framework-fixtures\/vite8-sveltekit-stateful\//,
       /^tests\/live-svelte-adapter-deepseek\.test\.mjs$/,
     ],