mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-11 21:57:14 +03:00
Harden the test strategy: self-verifying triggers, 40% faster runner, release guards (#501)
* test: harden the test strategy (triggers, runner speed, release guards) Follow-ups from an end-to-end testing strategy review: - Suite triggers are now auto-generated from each suite's own file list, so change-based CI can never miss a test file again (four files were unreachable by their own edits, and tests/lib/detector-bundle.test.js triggered core while running in detector). Two new meta-tests pin the invariant. Hand-written trigger patterns now carry only source paths and fixture dirs; palette dropped from the live triggers since no suite tests it. - The node runner batches all files into one node --test invocation at concurrency 4 instead of spawning per file. Default suite drops from ~159s to ~100s; the live suite soaked clean three times. - scripts/release.mjs gets its first tests: 12 scenarios spawning the real script inside a disposable git repo with a local bare origin, covering every refusal guard plus notes/tweet rendering, all under --dry-run. - skill/scripts/live/ui-core.mjs deleted: zero references repo-wide, superseded by the July live rewrite, yet still shipping to users. cli/lib/download-providers.js annotated with its cross-repo consumers (impeccable-site Pages Functions) so it is not mistaken for dead code. - CLAUDE.md gains an area-to-suite table for the opt-in suites a change owes; AGENTS.md syncs the plugin-e2e commands and obligations. AI-assisted via Claude Code under maintainer direction. Co-Authored-By: Claude Code <noreply@anthropic.com> * fix: exclude peeled tag lines from release-test origin cleanup Copilot: git ls-remote --tags emits ^{} peel lines for annotated tags, which are not deletable refs; --refs filters them so the cleanup loop survives a future scenario that pushes an annotated tag. AI-assisted via Claude Code under maintainer direction. Co-Authored-By: Claude Code <noreply@anthropic.com> --------- Co-authored-by: Claude Code <noreply@anthropic.com>
This commit is contained in:
Paul Bakaus
co-authored by
Claude Code
Claude Code
parent
14d2641685
commit
667095d216
+11
-7
@@ -40,13 +40,17 @@ function runCommand(command) {
|
||||
}
|
||||
|
||||
if (command.runner === 'node') {
|
||||
for (const file of command.files) {
|
||||
const args = ['--test'];
|
||||
if (command.timeoutMs) args.push(`--test-timeout=${command.timeoutMs}`);
|
||||
if (command.forceExit) args.push('--test-force-exit');
|
||||
args.push(file);
|
||||
runProcess(process.execPath, args, { env });
|
||||
}
|
||||
// One invocation for the whole file list: node --test runs each file in
|
||||
// its own child process regardless, so isolation is unchanged, but the
|
||||
// runner-per-file spawn overhead is gone and files execute concurrently.
|
||||
// Measured on the live suite (38 files): 52s serial-per-file vs 18s
|
||||
// batched at concurrency 4. Suites can pin `concurrency: 1` if their
|
||||
// tests ever contend for a shared resource.
|
||||
const args = ['--test', `--test-concurrency=${command.concurrency ?? 4}`];
|
||||
if (command.timeoutMs) args.push(`--test-timeout=${command.timeoutMs}`);
|
||||
if (command.forceExit) args.push('--test-force-exit');
|
||||
args.push(...command.files);
|
||||
runProcess(process.execPath, args, { env });
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
+25
-7
@@ -29,8 +29,6 @@ export const SUITES = {
|
||||
/^skill\/(SKILL\.src\.md|agents\/|reference\/|scripts\/(cleanup-deprecated|concept-seed|context|context-signals|critique-storage|design-parser|doctor|hook|impeccable-paths|is-generated|lib\/(artifact-schema|composition-catalog|concept-catalog|provider|staleness|staleness-deep|staleness-notice|surface-briefs|target-slug|template-extensions)|pin|surface-brief))/,
|
||||
/^README(\.npm)?\.md$/,
|
||||
/^cli\/bin\//,
|
||||
/^tests\/(build|cleanup-deprecated|cli-args|cli-ignores|concept-seed|context|context-signals|critique-storage|design-parser|doctor|github-sheriff|hook|hook-build|impeccable-paths|openai-plugin|pin|skills-cli|staleness|surface-brief|target-args|template-extensions|test-suites|windows-path-fix|zip)\.test\.(js|mjs)$/,
|
||||
/^tests\/lib\//,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
@@ -67,6 +65,7 @@ export const SUITES = {
|
||||
'tests/impeccable-paths.test.mjs',
|
||||
'tests/openai-plugin.test.mjs',
|
||||
'tests/pin.test.mjs',
|
||||
'tests/release.test.mjs',
|
||||
'tests/doctor.test.mjs',
|
||||
'tests/staleness.test.mjs',
|
||||
'tests/target-args.test.mjs',
|
||||
@@ -87,8 +86,7 @@ export const SUITES = {
|
||||
/^extension\/(background|content|detector|devtools|popup|manifest\.json)/,
|
||||
/^scripts\/(benchmark-detector|build-browser-detector|build-extension)\.js$/,
|
||||
/^site\/(pages\/detector|public\/antipattern|data\/anti-patterns-catalog\.js)/,
|
||||
/^tests\/design-system\.test\.mjs$/,
|
||||
/^tests\/(detect-antipatterns|detect-cli-design-contamination|detect-url-launch|inline-ignores|extension-build|fixtures\/antipatterns)/,
|
||||
/^tests\/fixtures\/antipatterns/,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
@@ -116,10 +114,11 @@ export const SUITES = {
|
||||
description: 'Fast live-mode unit and local-server integration tests, excluding full browser fixture sweeps.',
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^skill\/(reference\/live\.md|scripts\/(detect-csp|lib\/is-generated|lib\/template-extensions|live\/|live|live-|modern-screenshot|pin|palette))/,
|
||||
// `palette` is deliberately absent: skill/scripts/palette.mjs has no
|
||||
// test anywhere, and listing it here made edits run a suite that never
|
||||
// touches it, which reads as coverage that does not exist.
|
||||
/^skill\/(reference\/live\.md|scripts\/(detect-csp|lib\/is-generated|lib\/template-extensions|live\/|live|live-|modern-screenshot|pin))/,
|
||||
/^tests\/live-/,
|
||||
/^tests\/live-e2e\/(agent|agents\/llm-agent|cli-options|preactions|session|steer|ui)\.mjs$/,
|
||||
/^tests\/live-e2e\/agent-insert\.test\.mjs$/,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
@@ -339,6 +338,25 @@ export const SUITES = {
|
||||
},
|
||||
};
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
// Every suite must select itself when one of its own test files changes.
|
||||
// Generated from the files lists so the hand-written trigger patterns above
|
||||
// only carry source paths and fixture directories; before this, four test
|
||||
// files were registered in a suite that change-based CI could never select
|
||||
// by editing them (serve-question, ci-test-plan, both validate-plugin-*),
|
||||
// and tests/lib/detector-bundle.test.js triggered core while running in
|
||||
// detector. The meta-test in tests/test-suites.test.mjs pins this invariant.
|
||||
for (const suite of Object.values(SUITES)) {
|
||||
const ownFiles = suite.commands.flatMap((command) => command.files);
|
||||
suite.triggers = [
|
||||
...(suite.triggers ?? []),
|
||||
...ownFiles.map((file) => new RegExp(`^${escapeRegExp(file)}$`)),
|
||||
];
|
||||
}
|
||||
|
||||
export function expandSuites(requested) {
|
||||
const names = requested.length === 0 ? ['default'] : requested;
|
||||
const expanded = [];
|
||||
|
||||
Reference in New Issue
Block a user