mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-19 09:36:59 +03:00
Harden the test strategy: self-verifying triggers, 40% faster runner, release guards (#501)
* test: harden the test strategy (triggers, runner speed, release guards) Follow-ups from an end-to-end testing strategy review: - Suite triggers are now auto-generated from each suite's own file list, so change-based CI can never miss a test file again (four files were unreachable by their own edits, and tests/lib/detector-bundle.test.js triggered core while running in detector). Two new meta-tests pin the invariant. Hand-written trigger patterns now carry only source paths and fixture dirs; palette dropped from the live triggers since no suite tests it. - The node runner batches all files into one node --test invocation at concurrency 4 instead of spawning per file. Default suite drops from ~159s to ~100s; the live suite soaked clean three times. - scripts/release.mjs gets its first tests: 12 scenarios spawning the real script inside a disposable git repo with a local bare origin, covering every refusal guard plus notes/tweet rendering, all under --dry-run. - skill/scripts/live/ui-core.mjs deleted: zero references repo-wide, superseded by the July live rewrite, yet still shipping to users. cli/lib/download-providers.js annotated with its cross-repo consumers (impeccable-site Pages Functions) so it is not mistaken for dead code. - CLAUDE.md gains an area-to-suite table for the opt-in suites a change owes; AGENTS.md syncs the plugin-e2e commands and obligations. AI-assisted via Claude Code under maintainer direction. Co-Authored-By: Claude Code <noreply@anthropic.com> * fix: exclude peeled tag lines from release-test origin cleanup Copilot: git ls-remote --tags emits ^{} peel lines for annotated tags, which are not deletable refs; --refs filters them so the cleanup loop survives a future scenario that pushes an annotated tag. AI-assisted via Claude Code under maintainer direction. Co-Authored-By: Claude Code <noreply@anthropic.com> --------- Co-authored-by: Claude Code <noreply@anthropic.com>
This commit is contained in:
Paul Bakaus
co-authored by
Claude Code
Claude Code
parent
14d2641685
commit
667095d216
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* Guard tests for scripts/release.mjs, the tagging/publishing script for the
|
||||
* three independently versioned components. Until now it had zero coverage
|
||||
* while owning every refusal that protects a public release: dirty tree,
|
||||
* unpushed HEAD, existing tag, disagreeing manifests, missing changelog
|
||||
* entry, missing artifacts.
|
||||
*
|
||||
* The script resolves repoRoot from its own file location and runs top-level
|
||||
* code on import, so these tests copy it into a disposable git repo (with a
|
||||
* local bare `origin`) and spawn it exactly as a maintainer would. Every run
|
||||
* uses --dry-run, which skips all mutating steps (tag, push, gh release,
|
||||
* builds) but exercises every guard on the way there.
|
||||
*/
|
||||
import { describe, it, before, after, beforeEach } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const RELEASE_SCRIPT = path.join(REPO_ROOT, 'scripts', 'release.mjs');
|
||||
|
||||
const CHANGELOG = `---
|
||||
---
|
||||
<article>
|
||||
<div class="changelog-version-header"><span class="cf-version">v1.2.3</span></div>
|
||||
<ul class="cf-items">
|
||||
<li><strong>Loader contract pinned.</strong> Uses <code>plugin.json</code> checks & a <a href="https://example.com/docs">guide</a>.</li>
|
||||
<li><strong>Faster runner.</strong> Batched invocations cut wall time.</li>
|
||||
</ul>
|
||||
</article>
|
||||
<article>
|
||||
<div class="changelog-version-header"><span class="cf-version">CLI v9.9.9</span></div>
|
||||
<ul class="cf-items">
|
||||
<li><strong>New detect flags.</strong> Adds <code>--fast</code>.</li>
|
||||
</ul>
|
||||
</article>
|
||||
`;
|
||||
|
||||
function git(cwd, ...args) {
|
||||
return execFileSync('git', args, { cwd, encoding: 'utf-8' }).trim();
|
||||
}
|
||||
|
||||
function runRelease(cwd, ...args) {
|
||||
try {
|
||||
const stdout = execFileSync(process.execPath, ['scripts/release.mjs', ...args, '--dry-run'], {
|
||||
cwd,
|
||||
encoding: 'utf-8',
|
||||
timeout: 60000,
|
||||
});
|
||||
return { code: 0, stdout, stderr: '' };
|
||||
} catch (err) {
|
||||
return { code: err.status ?? 1, stdout: err.stdout ?? '', stderr: err.stderr ?? '' };
|
||||
}
|
||||
}
|
||||
|
||||
describe('release.mjs guards', () => {
|
||||
let root;
|
||||
let workDir;
|
||||
let bareDir;
|
||||
let baselineSha;
|
||||
|
||||
const write = (rel, contents) => {
|
||||
const abs = path.join(workDir, rel);
|
||||
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
||||
fs.writeFileSync(abs, contents);
|
||||
};
|
||||
|
||||
before(() => {
|
||||
root = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-release-'));
|
||||
bareDir = path.join(root, 'origin.git');
|
||||
workDir = path.join(root, 'work');
|
||||
execFileSync('git', ['init', '--bare', bareDir]);
|
||||
fs.mkdirSync(workDir);
|
||||
git(workDir, 'init', '-b', 'main');
|
||||
git(workDir, 'config', 'user.email', 'test@example.com');
|
||||
git(workDir, 'config', 'user.name', 'Release Test');
|
||||
|
||||
fs.mkdirSync(path.join(workDir, 'scripts'));
|
||||
fs.copyFileSync(RELEASE_SCRIPT, path.join(workDir, 'scripts', 'release.mjs'));
|
||||
write('.claude-plugin/plugin.json', JSON.stringify({ name: 'impeccable', version: '1.2.3' }));
|
||||
write('.claude-plugin/marketplace.json', JSON.stringify({ plugins: [{ name: 'impeccable', version: '1.2.3' }] }));
|
||||
write('package.json', JSON.stringify({ name: 'impeccable', version: '9.9.9' }));
|
||||
write('extension/manifest.json', JSON.stringify({ version: '2.0.0' }));
|
||||
write('site/pages/changelog.astro', CHANGELOG);
|
||||
write('dist/universal.zip', 'zip');
|
||||
write('dist/extension.zip', 'zip');
|
||||
write('dist/extension-firefox.zip', 'zip');
|
||||
|
||||
git(workDir, 'add', '-A');
|
||||
git(workDir, 'commit', '-m', 'fixture');
|
||||
git(workDir, 'remote', 'add', 'origin', bareDir);
|
||||
git(workDir, 'push', '-u', 'origin', 'main');
|
||||
baselineSha = git(workDir, 'rev-parse', 'HEAD');
|
||||
});
|
||||
|
||||
after(() => {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
// Undo whatever the previous scenario staged, on both ends: local tree
|
||||
// and tags back to the baseline commit, and origin force-reset too, since
|
||||
// several scenarios push commits or tags that would poison later ones.
|
||||
git(workDir, 'checkout', '--', '.');
|
||||
git(workDir, 'clean', '-fd');
|
||||
git(workDir, 'reset', '--hard', baselineSha);
|
||||
git(workDir, 'push', '--force', 'origin', 'main');
|
||||
for (const tag of git(workDir, 'tag').split('\n').filter(Boolean)) {
|
||||
git(workDir, 'tag', '-d', tag);
|
||||
}
|
||||
// --refs excludes the peeled `^{}` lines annotated tags produce, which
|
||||
// are not deletable refs and would abort the cleanup.
|
||||
for (const line of git(workDir, 'ls-remote', '--refs', '--tags', 'origin').split('\n').filter(Boolean)) {
|
||||
const ref = line.split('\t')[1];
|
||||
if (ref) git(workDir, 'push', 'origin', `:${ref}`);
|
||||
}
|
||||
});
|
||||
|
||||
it('dry-runs a clean skill release end to end', () => {
|
||||
const { code, stdout } = runRelease(workDir, 'skill');
|
||||
assert.equal(code, 0, stdout);
|
||||
assert.match(stdout, /Skill 1\.2\.3/);
|
||||
assert.match(stdout, /tag is free/);
|
||||
assert.match(stdout, /\[dry-run\] git tag -a skill-v1\.2\.3/);
|
||||
assert.match(stdout, /\[dry-run\] gh release create skill-v1\.2\.3/);
|
||||
});
|
||||
|
||||
it('converts the changelog entry to markdown release notes', () => {
|
||||
const { code, stdout } = runRelease(workDir, 'skill');
|
||||
assert.equal(code, 0, stdout);
|
||||
assert.match(stdout, /- \*\*Loader contract pinned\.\*\* Uses `plugin\.json` checks & a \[guide\]\(https:\/\/example\.com\/docs\)\./);
|
||||
assert.match(stdout, /- \*\*Faster runner\.\*\*/);
|
||||
});
|
||||
|
||||
it('renders a tweet within the 280-char limit with the release URL', () => {
|
||||
const { code, stdout } = runRelease(workDir, 'skill');
|
||||
assert.equal(code, 0, stdout);
|
||||
const tweetMatch = stdout.match(/--- Tweet \((\d+)\/280 chars\)[^\n]*---\n([\s\S]*?)\n--- end tweet ---/);
|
||||
assert.ok(tweetMatch, `no tweet block in output:\n${stdout}`);
|
||||
assert.ok(Number(tweetMatch[1]) <= 280);
|
||||
assert.match(tweetMatch[2], /Impeccable v1\.2\.3 is out\./);
|
||||
assert.match(tweetMatch[2], /releases\/tag\/skill-v1\.2\.3/);
|
||||
assert.match(tweetMatch[2], /• Loader contract pinned/);
|
||||
});
|
||||
|
||||
it('matches the prefixed changelog label for the CLI component', () => {
|
||||
const { code, stdout } = runRelease(workDir, 'cli');
|
||||
assert.equal(code, 0, stdout);
|
||||
assert.match(stdout, /CLI 9\.9\.9/);
|
||||
assert.match(stdout, /- \*\*New detect flags\.\*\*/);
|
||||
});
|
||||
|
||||
it('refuses an unknown component', () => {
|
||||
const { code, stderr } = runRelease(workDir, 'website');
|
||||
assert.equal(code, 1);
|
||||
assert.match(stderr, /usage: release\.mjs/);
|
||||
});
|
||||
|
||||
it('refuses a dirty working tree', () => {
|
||||
write('README.md', 'uncommitted');
|
||||
const { code, stderr } = runRelease(workDir, 'skill');
|
||||
assert.equal(code, 1);
|
||||
assert.match(stderr, /Working tree is dirty/);
|
||||
});
|
||||
|
||||
it('refuses when HEAD is ahead of origin', () => {
|
||||
write('note.txt', 'ahead');
|
||||
git(workDir, 'add', '-A');
|
||||
git(workDir, 'commit', '-m', 'unpushed');
|
||||
const { code, stderr } = runRelease(workDir, 'skill');
|
||||
assert.equal(code, 1);
|
||||
assert.match(stderr, /Push your commits first/);
|
||||
});
|
||||
|
||||
it('refuses when the tag already exists locally', () => {
|
||||
git(workDir, 'tag', 'skill-v1.2.3');
|
||||
const { code, stderr } = runRelease(workDir, 'skill');
|
||||
assert.equal(code, 1);
|
||||
assert.match(stderr, /already exists locally/);
|
||||
});
|
||||
|
||||
it('refuses when the tag already exists on origin', () => {
|
||||
git(workDir, 'tag', 'skill-v1.2.3');
|
||||
git(workDir, 'push', 'origin', 'skill-v1.2.3');
|
||||
git(workDir, 'tag', '-d', 'skill-v1.2.3');
|
||||
const { code, stderr } = runRelease(workDir, 'skill');
|
||||
assert.equal(code, 1);
|
||||
assert.match(stderr, /already exists on origin/);
|
||||
});
|
||||
|
||||
it('refuses when plugin.json and marketplace.json disagree', () => {
|
||||
write('.claude-plugin/marketplace.json', JSON.stringify({ plugins: [{ name: 'impeccable', version: '1.0.0' }] }));
|
||||
git(workDir, 'add', '-A');
|
||||
git(workDir, 'commit', '-m', 'mismatch');
|
||||
git(workDir, 'push', 'origin', 'main');
|
||||
const { code, stderr } = runRelease(workDir, 'skill');
|
||||
assert.equal(code, 1);
|
||||
assert.match(stderr, /disagree\. Bump both\./);
|
||||
});
|
||||
|
||||
it('refuses when the changelog entry is missing', () => {
|
||||
write('extension/manifest.json', JSON.stringify({ version: '3.0.0' }));
|
||||
git(workDir, 'add', '-A');
|
||||
git(workDir, 'commit', '-m', 'bump without changelog');
|
||||
git(workDir, 'push', 'origin', 'main');
|
||||
const { code, stderr } = runRelease(workDir, 'extension');
|
||||
assert.equal(code, 1);
|
||||
assert.match(stderr, /No changelog entry found for "Extension v3\.0\.0"/);
|
||||
});
|
||||
|
||||
it('refuses when a release artifact is missing', () => {
|
||||
fs.rmSync(path.join(workDir, 'dist/universal.zip'));
|
||||
git(workDir, 'add', '-A');
|
||||
git(workDir, 'commit', '-m', 'drop artifact');
|
||||
git(workDir, 'push', 'origin', 'main');
|
||||
const { code, stderr } = runRelease(workDir, 'skill');
|
||||
assert.equal(code, 1);
|
||||
assert.match(stderr, /Missing artifact: dist\/universal\.zip/);
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
SUITES,
|
||||
expandSuites,
|
||||
findTestFiles,
|
||||
matchesSuiteTriggers,
|
||||
suiteFiles,
|
||||
} from '../scripts/test-suites.mjs';
|
||||
|
||||
@@ -36,4 +37,33 @@ describe('test suite registry', () => {
|
||||
assert.equal(SUITES[suite].optIn, undefined, `${suite} should not be opt-in`);
|
||||
}
|
||||
});
|
||||
|
||||
it('selects every suite when one of its own test files changes', () => {
|
||||
// Change-based CI (ci-test-plan.mjs) picks suites via matchesSuiteTriggers.
|
||||
// A test file whose edits select no suite, or only a suite that does not
|
||||
// run it, is a silent CI gap; the generated own-file triggers close it.
|
||||
for (const [name, suite] of Object.entries(SUITES)) {
|
||||
for (const file of suiteFiles([name])) {
|
||||
assert.equal(
|
||||
matchesSuiteTriggers(name, [file]),
|
||||
true,
|
||||
`editing ${file} must select suite "${name}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('never lets a test file trigger only suites that do not run it', () => {
|
||||
const allSuiteNames = Object.keys(SUITES);
|
||||
for (const file of findTestFiles()) {
|
||||
const triggered = allSuiteNames.filter((name) => matchesSuiteTriggers(name, [file]));
|
||||
const runsIn = allSuiteNames.filter((name) => suiteFiles([name]).includes(file));
|
||||
const useful = triggered.filter((name) => runsIn.includes(name));
|
||||
assert.ok(
|
||||
useful.length > 0,
|
||||
`${file} triggers [${triggered.join(', ')}] but runs in [${runsIn.join(', ')}]; ` +
|
||||
'at least one triggered suite must actually run it',
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user