mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-14 15:16:35 +03:00
Paul Bakaus
6a93a35293
Carry native comp capture and completion integrity fixes into the reviewed component workflow. Bump the skill to 4.4.0 and engine/platform pins to 0.1.6; keep publication separate from this release candidate. AI assistance: implemented and validated with OpenAI Codex.
407 lines
13 KiB
JavaScript
407 lines
13 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
|
|
export const DEFAULT_SUITES = ['core', 'oracle', 'detector', 'live', 'framework', 'plugin-e2e'];
|
|
export const OPT_IN_SUITES = [
|
|
'cli-remote-e2e',
|
|
'live-e2e',
|
|
'live-e2e-accept-cleanup',
|
|
'new-work-e2e',
|
|
'skill-behavior',
|
|
'skill-workflow',
|
|
'live-svelte-adapter-deepseek',
|
|
];
|
|
|
|
const COMMON_INFRA_PATTERNS = [
|
|
/^package\.json$/,
|
|
/^bun\.lock$/,
|
|
/^scripts\/run-tests\.mjs$/,
|
|
/^scripts\/test-suites\.mjs$/,
|
|
/^scripts\/ci-test-plan\.mjs$/,
|
|
/^scripts\/lib\/(live-server-processes|process-group|test-orphan-reaper)\.mjs$/,
|
|
/^tests\/lib\/live-servers\.mjs$/,
|
|
/^scripts\/lib\/(live-server-processes|process-group|test-orphan-reaper)\.mjs$/,
|
|
/^tests\/lib\/live-servers\.mjs$/,
|
|
/^\.github\/workflows\/ci\.yml$/,
|
|
];
|
|
|
|
export const SUITES = {
|
|
core: {
|
|
description: 'Build, provider transforms, hook manifests, plugin validators, and prose gates.',
|
|
triggers: [
|
|
/^ui\/component-review\//,
|
|
/^crates\/context\/assets\/component-review\.js$/,
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^scripts\/(?!build-extension)/,
|
|
/^skill\/(SKILL\.src\.md|agents\/|reference\/|scripts\/)/,
|
|
/^ENGINE_VERSION$/,
|
|
/^README(\.npm)?\.md$/,
|
|
/^vscode\//,
|
|
/^\.github\/workflows\/release-engine\.yml$/,
|
|
/^cli\/bin\//,
|
|
],
|
|
commands: [
|
|
{
|
|
runner: 'bun',
|
|
files: [
|
|
'tests/build.test.js',
|
|
'tests/component-review-bundle.test.js',
|
|
'ui/component-review/model.test.ts',
|
|
'ui/component-review/viewport.test.ts',
|
|
'tests/lib/provider-blocks.test.js',
|
|
'tests/lib/transformers/provider-blocks.test.js',
|
|
'tests/lib/utils.test.js',
|
|
'tests/lib/transformers/factory.test.js',
|
|
'tests/lib/transformers/opencode-commands.test.js',
|
|
'tests/lib/transformers/providers.test.js',
|
|
'tests/root-commands-sync.test.js',
|
|
'tests/validate-plugin-versions.test.js',
|
|
'tests/validate-plugin-manifest.test.js',
|
|
'tests/plugin-paths.test.js',
|
|
'tests/release-engine-workflow.test.js',
|
|
'tests/workflow-security.test.js',
|
|
],
|
|
},
|
|
{
|
|
runner: 'node',
|
|
// A finite per-test cap so an async hang is cancelled and reported
|
|
// rather than left running with `--test-timeout` unset (Infinity).
|
|
// Note: this timer lives in the event loop, so it cannot interrupt a
|
|
// test blocked in a synchronous spawnSync; the runner's wall-clock
|
|
// group-kill covers that case. The slowest core test is ~11s, so 180s
|
|
// is safe.
|
|
timeoutMs: 180000,
|
|
files: [
|
|
'tests/ci-test-plan.test.mjs',
|
|
'tests/cli-shim.test.mjs',
|
|
'tests/launcher-download.test.mjs',
|
|
'tests/publish-platform-packages.test.mjs',
|
|
'tests/github-sheriff.test.mjs',
|
|
'tests/hook-build.test.mjs',
|
|
'tests/openai-plugin.test.mjs',
|
|
'tests/cursor-plugin.test.mjs',
|
|
'tests/vscode-extension.test.mjs',
|
|
'tests/process-group.test.mjs',
|
|
'tests/release.test.mjs',
|
|
'tests/bundle-signing.test.mjs',
|
|
'tests/skill-reference.test.mjs',
|
|
'tests/skill-behavior-harness.test.mjs',
|
|
'tests/readme-gitignore.test.mjs',
|
|
'tests/test-suites.test.mjs',
|
|
],
|
|
},
|
|
],
|
|
},
|
|
// The verbs live in the engine binary; this repo pins its behavior with the
|
|
// oracle goldens (tests/oracle) and drives its live-mode verbs over the
|
|
// framework fixtures. Both skip when no binary is present (bun run
|
|
// fetch:engine, or IMPECCABLE_BIN).
|
|
oracle: {
|
|
description: 'Oracle corpus replay against the engine binary; skips without a binary.',
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^ENGINE_VERSION$/,
|
|
/^tests\/oracle\//,
|
|
/^tests\/fixtures\//,
|
|
/^tests\/lib\/engine-bin\.mjs$/,
|
|
/^skill\/(reference\/|scripts\/)/,
|
|
],
|
|
commands: [
|
|
{
|
|
runner: 'node',
|
|
timeoutMs: 900000,
|
|
files: ['tests/oracle.test.mjs'],
|
|
},
|
|
],
|
|
},
|
|
detector: {
|
|
description: 'Extension packaging checks (the rule logic itself is covered by the crate tests and the oracle).',
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^extension\/(background|content|detector|devtools|offscreen|popup|shared|manifest\.json)/,
|
|
/^scripts\/build-extension\.js$/,
|
|
/^browser-bundle\//,
|
|
// Everything `cargo xtask bundle` reads: the rules and the registry
|
|
// rows (core, foundation), the wasm module (wasm), the assembly and the
|
|
// registry serialization (bundle), and the task itself (xtask). Leaving
|
|
// one out means a PR that changes what the bundle emits never rebuilds
|
|
// it, and the tracked-output check in ci.yml then compares a committed
|
|
// artifact against an untouched tree and passes on stale bytes.
|
|
/^crates\/(bundle|core|foundation|wasm|xtask)\//,
|
|
// The tracked artifacts themselves, so a hand-edit is regenerated over.
|
|
/^crates\/live\/assets\//,
|
|
],
|
|
commands: [
|
|
{
|
|
runner: 'node',
|
|
files: ['tests/extension-build.test.mjs'],
|
|
},
|
|
],
|
|
},
|
|
live: {
|
|
description: 'Live-mode reference contract checks plus the live-e2e helper units (agent output, CLI options, LLM agent parsing, steer loop against the binary); the live verbs themselves are covered by the oracle and framework suites.',
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^skill\/(reference\/live\.md|scripts\/live-browser)/,
|
|
/^tests\/live-e2e\//,
|
|
/^tests\/lib\/engine-bin\.mjs$/,
|
|
],
|
|
commands: [
|
|
{
|
|
runner: 'node',
|
|
files: [
|
|
'tests/live-reference.test.mjs',
|
|
'tests/live-browser-ignores.test.mjs',
|
|
'tests/live-browser-source.test.mjs',
|
|
'tests/live-e2e-agent-output.test.mjs',
|
|
'tests/live-e2e-cli-options.test.mjs',
|
|
'tests/live-e2e-llm-agent.test.mjs',
|
|
'tests/live-e2e-steer-agent.test.mjs',
|
|
'tests/live-e2e/agent-insert.test.mjs',
|
|
'tests/live-server-leak.test.mjs',
|
|
],
|
|
},
|
|
],
|
|
},
|
|
framework: {
|
|
description: 'Framework fixture coverage for live injection, CSP detection, and wrapping through the engine binary; skips without a binary.',
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^ENGINE_VERSION$/,
|
|
/^tests\/framework-fixtures/,
|
|
/^tests\/framework-fixtures\.test\.mjs$/,
|
|
/^tests\/lib\/engine-bin\.mjs$/,
|
|
],
|
|
commands: [
|
|
{
|
|
runner: 'node',
|
|
files: ['tests/framework-fixtures.test.mjs'],
|
|
},
|
|
],
|
|
},
|
|
// `impeccable install/update/check` and their remote smoke moved into the
|
|
// engine binary and its repo; the deterministic coverage here is the oracle
|
|
// corpus. The lane name stays so ci.yml and package.json keep resolving.
|
|
'cli-remote-e2e': {
|
|
description: 'Remote CLI install/update smoke (moved to the engine repo; no tests here).',
|
|
optIn: true,
|
|
triggers: [...COMMON_INFRA_PATTERNS],
|
|
commands: [],
|
|
},
|
|
'plugin-e2e': {
|
|
description: 'Install the committed ./plugin subtree into a real (sandboxed) Claude Code and assert skills, agents, and hooks all load. Skips when the claude CLI is not on PATH.',
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^plugin\//,
|
|
/^skill\/agents\//,
|
|
/^scripts\/build\.js$/,
|
|
/^scripts\/lib\/validate-plugin-manifest\.js$/,
|
|
/^tests\/plugin-e2e\.test\.mjs$/,
|
|
],
|
|
commands: [
|
|
{
|
|
runner: 'node',
|
|
timeoutMs: 300000,
|
|
forceExit: true,
|
|
files: ['tests/plugin-e2e.test.mjs'],
|
|
},
|
|
],
|
|
},
|
|
'live-e2e': {
|
|
description: 'Full Playwright live-mode click-to-accept sweep across runtime framework fixtures.',
|
|
optIn: true,
|
|
needsPlaywright: true,
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^skill\/scripts\/live-browser/,
|
|
/^ENGINE_VERSION$/,
|
|
/^tests\/framework-fixtures/,
|
|
/^tests\/live-e2e(\.test\.mjs|\/)/,
|
|
],
|
|
commands: [
|
|
{
|
|
runner: 'node',
|
|
timeoutMs: 600000,
|
|
forceExit: true,
|
|
files: ['tests/live-e2e.test.mjs'],
|
|
},
|
|
],
|
|
},
|
|
'new-work-e2e': {
|
|
description: 'Playwright smoke sweep of the new-work concept/serve-question decision page plus the offline fake image generator.',
|
|
optIn: true,
|
|
needsPlaywright: true,
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^ENGINE_VERSION$/,
|
|
/^tests\/new-work-e2e(\.test\.mjs|\/)/,
|
|
],
|
|
commands: [
|
|
{
|
|
runner: 'node',
|
|
timeoutMs: 600000,
|
|
forceExit: true,
|
|
files: ['tests/new-work-e2e.test.mjs'],
|
|
},
|
|
],
|
|
},
|
|
'live-e2e-accept-cleanup': {
|
|
description: 'Provider-backed post-accept cleanup regression.',
|
|
optIn: true,
|
|
needsPlaywright: true,
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^ENGINE_VERSION$/,
|
|
/^tests\/live-e2e-accept-cleanup-regression\.test\.mjs$/,
|
|
/^tests\/live-e2e\//,
|
|
],
|
|
commands: [
|
|
{
|
|
runner: 'node',
|
|
timeoutMs: 600000,
|
|
files: ['tests/live-e2e-accept-cleanup-regression.test.mjs'],
|
|
},
|
|
],
|
|
},
|
|
'live-e2e-agent': {
|
|
description: 'Focused insert-mode fake-agent helper tests.',
|
|
commands: [
|
|
{
|
|
runner: 'node',
|
|
files: ['tests/live-e2e/agent-insert.test.mjs'],
|
|
},
|
|
],
|
|
},
|
|
'skill-behavior': {
|
|
description: 'LLM-backed protocol checkpoints, not full builds.',
|
|
optIn: true,
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^skill\/SKILL\.src\.md$/,
|
|
/^skill\/reference\//,
|
|
/^ENGINE_VERSION$/,
|
|
/^tests\/skill-behavior\//,
|
|
],
|
|
commands: [{
|
|
runner: 'node',
|
|
timeoutMs: 240000,
|
|
wallClockMs: 1_800_000,
|
|
files: ['tests/skill-behavior/scenarios.test.mjs'],
|
|
}],
|
|
},
|
|
'skill-workflow': {
|
|
description: 'Explicitly opt-in completed workflows with a preflighted browser.',
|
|
optIn: true,
|
|
needsPlaywright: true,
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^skill\//,
|
|
/^ENGINE_VERSION$/,
|
|
/^tests\/skill-workflow\//,
|
|
/^tests\/skill-behavior\//,
|
|
],
|
|
commands: [
|
|
{ runner: 'node', files: ['tests/skill-workflow-browser.test.mjs'] },
|
|
{
|
|
runner: 'node', timeoutMs: 240000, wallClockMs: 600000,
|
|
files: ['tests/skill-workflow/finish-handoff.test.mjs'],
|
|
},
|
|
{
|
|
runner: 'node',
|
|
timeoutMs: 900000,
|
|
wallClockMs: 3_600_000,
|
|
files: ['tests/skill-workflow/full-build.test.mjs'],
|
|
},
|
|
],
|
|
},
|
|
'live-svelte-adapter-deepseek': {
|
|
description: 'DeepSeek-backed Svelte adapter browser sweep.',
|
|
optIn: true,
|
|
needsPlaywright: true,
|
|
triggers: [
|
|
...COMMON_INFRA_PATTERNS,
|
|
/^ENGINE_VERSION$/,
|
|
/^tests\/framework-fixtures\/vite8-sveltekit-stateful\//,
|
|
/^tests\/live-svelte-adapter-deepseek\.test\.mjs$/,
|
|
],
|
|
commands: [
|
|
{
|
|
runner: 'node',
|
|
timeoutMs: 1200000,
|
|
files: ['tests/live-svelte-adapter-deepseek.test.mjs'],
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
function escapeRegExp(value) {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
}
|
|
|
|
// Every suite must select itself when one of its own test files changes.
|
|
// Generated from the files lists so the hand-written trigger patterns above
|
|
// only carry source paths and fixture directories; before this, four test
|
|
// files were registered in a suite that change-based CI could never select
|
|
// by editing them (serve-question, ci-test-plan, both validate-plugin-*),
|
|
// and tests/lib/detector-bundle.test.js triggered core while running in
|
|
// detector. The meta-test in tests/test-suites.test.mjs pins this invariant.
|
|
for (const suite of Object.values(SUITES)) {
|
|
const ownFiles = suite.commands.flatMap((command) => command.files);
|
|
suite.triggers = [
|
|
...(suite.triggers ?? []),
|
|
...ownFiles.map((file) => new RegExp(`^${escapeRegExp(file)}$`)),
|
|
];
|
|
}
|
|
|
|
export function expandSuites(requested) {
|
|
const names = requested.length === 0 ? ['default'] : requested;
|
|
const expanded = [];
|
|
for (const name of names) {
|
|
if (name === 'default' || name === 'all-local') {
|
|
expanded.push(...DEFAULT_SUITES);
|
|
} else if (name === 'all') {
|
|
expanded.push(...DEFAULT_SUITES, ...OPT_IN_SUITES);
|
|
} else if (SUITES[name]) {
|
|
expanded.push(name);
|
|
} else {
|
|
throw new Error(`Unknown test suite "${name}". Run: node scripts/run-tests.mjs --list`);
|
|
}
|
|
}
|
|
return [...new Set(expanded)];
|
|
}
|
|
|
|
export function suiteFiles(suiteNames) {
|
|
const files = [];
|
|
for (const name of suiteNames) {
|
|
const suite = SUITES[name];
|
|
if (!suite) throw new Error(`Unknown test suite "${name}"`);
|
|
for (const command of suite.commands) {
|
|
files.push(...command.files);
|
|
}
|
|
}
|
|
return files;
|
|
}
|
|
|
|
export function findTestFiles(root = process.cwd()) {
|
|
const out = [];
|
|
const stack = [path.join(root, 'tests')];
|
|
while (stack.length) {
|
|
const dir = stack.pop();
|
|
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
const abs = path.join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
stack.push(abs);
|
|
} else if (/\.test\.(js|mjs)$/.test(entry.name)) {
|
|
out.push(path.relative(root, abs).split(path.sep).join('/'));
|
|
}
|
|
}
|
|
}
|
|
return out.sort();
|
|
}
|
|
|
|
export function matchesSuiteTriggers(suiteName, changedFiles) {
|
|
const suite = SUITES[suiteName];
|
|
if (!suite) throw new Error(`Unknown test suite "${suiteName}"`);
|
|
return changedFiles.some((file) => suite.triggers?.some((pattern) => pattern.test(file)));
|
|
}
|