From 667095d2160dbe7cba128e2d2e08295d4830c0cd Mon Sep 17 00:00:00 2001 From: Paul Bakaus Date: Mon, 3 Aug 2026 17:33:10 -0700 Subject: [PATCH] Harden the test strategy: self-verifying triggers, 40% faster runner, release guards (#501) * test: harden the test strategy (triggers, runner speed, release guards) Follow-ups from an end-to-end testing strategy review: - Suite triggers are now auto-generated from each suite's own file list, so change-based CI can never miss a test file again (four files were unreachable by their own edits, and tests/lib/detector-bundle.test.js triggered core while running in detector). Two new meta-tests pin the invariant. Hand-written trigger patterns now carry only source paths and fixture dirs; palette dropped from the live triggers since no suite tests it. - The node runner batches all files into one node --test invocation at concurrency 4 instead of spawning per file. Default suite drops from ~159s to ~100s; the live suite soaked clean three times. - scripts/release.mjs gets its first tests: 12 scenarios spawning the real script inside a disposable git repo with a local bare origin, covering every refusal guard plus notes/tweet rendering, all under --dry-run. - skill/scripts/live/ui-core.mjs deleted: zero references repo-wide, superseded by the July live rewrite, yet still shipping to users. cli/lib/download-providers.js annotated with its cross-repo consumers (impeccable-site Pages Functions) so it is not mistaken for dead code. - CLAUDE.md gains an area-to-suite table for the opt-in suites a change owes; AGENTS.md syncs the plugin-e2e commands and obligations. AI-assisted via Claude Code under maintainer direction. Co-Authored-By: Claude Code * fix: exclude peeled tag lines from release-test origin cleanup Copilot: git ls-remote --tags emits ^{} peel lines for annotated tags, which are not deletable refs; --refs filters them so the cleanup loop survives a future scenario that pushes an annotated tag. AI-assisted via Claude Code under maintainer direction. Co-Authored-By: Claude Code --------- Co-authored-by: Claude Code --- AGENTS.md | 5 +- CLAUDE.md | 14 +++ cli/lib/download-providers.js | 5 + scripts/run-tests.mjs | 18 +-- scripts/test-suites.mjs | 32 +++-- skill/scripts/live/ui-core.mjs | 180 -------------------------- tests/release.test.mjs | 223 +++++++++++++++++++++++++++++++++ tests/test-suites.test.mjs | 30 +++++ 8 files changed, 312 insertions(+), 195 deletions(-) delete mode 100644 skill/scripts/live/ui-core.mjs create mode 100644 tests/release.test.mjs diff --git a/AGENTS.md b/AGENTS.md index 1223f901f..8029d45ed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,9 +12,10 @@ - `bun run rebuild` - clean and rebuild everything from scratch without syncing tracked harness folders. - `bun run rebuild:release` - clean and rebuild everything, including tracked harness output sync. - `bun test tests/build.test.js` - run a focused Bun test. -- `bun run test` - run the full Bun + Node test suite. +- `bun run test` - run the full Bun + Node test suite (includes the plugin loader E2E, which installs the committed `plugin/` subtree into a sandboxed real Claude Code and skips cleanly when the `claude` CLI is absent). - `bun run test:live-e2e` - opt-in live-mode E2E against framework fixtures (~2 min; needs `npx playwright install chromium` once). - `bun run test:skill-behavior` - opt-in LLM-backed checks that the SKILL.md Setup flow actually drives the agent (runs claude-sonnet-5 / gpt-5.6-luna / gemini-3.5-flash / deepseek-v4-flash; needs `.env` with provider keys). +- `bun run test:plugin-e2e` - just the plugin loader E2E, for fast iteration on `plugin/`, `skill/agents/`, or `scripts/build.js` changes. - `bun run build:browser` / `bun run build:extension` - rebuild browser-specific bundles. Run `bun run build` after changing anything in `skill/`, transformer code, or user-facing counts. It validates the generated distribution under `dist/` without touching tracked root harness outputs. Use `bun run build:release` only when intentionally refreshing generated provider permutations for release/main-sync or build-system work. @@ -48,6 +49,8 @@ Set `IMPECCABLE_E2E_AGENT=llm` to swap the deterministic fake agent for an API-b For changes to `skill/SKILL.src.md`'s Setup section, `skill/scripts/context.mjs`, or any Setup-touching reference file (`init.md`, `document.md`, `brand.md`, `product.md`, sub-command refs), also run `bun run test:skill-behavior`. The suite spawns current real models (claude-sonnet-5, gpt-5.6-luna, gemini-3.5-flash, deepseek-v4-flash) with the source SKILL.md inlined as system prompt and a workspace-scoped tool set, then asserts on the tool-call trace. Provider keys live in repo-root `.env`; missing keys skip cleanly. Scope to one provider with `IMPECCABLE_SKILL_BEHAVIOR_MODELS=`; add `IMPECCABLE_SKILL_BEHAVIOR_VERBOSE=1` to dump per-scenario traces. Baseline and per-scenario assertions live in `tests/skill-behavior/README.md`. +Other area-to-suite obligations (the canonical mapping is the `triggers` lists in `scripts/test-suites.mjs`; CLAUDE.md carries the full table): `serve-question.mjs` / `generate-image.mjs` / `concept-seed.mjs` changes owe `bun run test:new-work-e2e` (Playwright, offline); `cli/bin/commands/skills.mjs` changes owe `bun run test:cli-remote-e2e` (hits impeccable.style); accept/browser/server/wrap or SvelteKit adapter changes owe `bun run test:live-e2e-accept-cleanup` (provider-billed), and Svelte adapter/component changes owe `bun run test:live-svelte-adapter-deepseek` (DeepSeek-billed). + ## Anti-pattern detection rules `cli/engine/detect-antipatterns.mjs` is the source of truth for the rule engine. It feeds the CLI, the site overlay (`cli/engine/detect-antipatterns-browser.js`, regenerated by `bun run build:browser`), the Chrome extension (`extension/detector/`, regenerated by `bun run build:extension`), and the homepage `DETECTION_COUNT` in `site/public/js/generated/counts.js` (regenerated by `bun run build`). After any rule change run all three builds plus `bun run test` so nothing drifts. diff --git a/CLAUDE.md b/CLAUDE.md index 4f95fd2ce..993e0e133 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -144,6 +144,20 @@ bun run test:plugin-e2e # Just the plugin loader E2E (also part of the def Unit tests (build orchestration, detector logic) run via `bun test`. Fixture tests (jsdom-based HTML detection) run via `node --test` because bun is too slow with jsdom. The `test` script handles this split automatically. +### Which opt-in suite a change owes + +The default suite does not cover everything. When a change touches one of these areas, run the matching opt-in suite before shipping. The canonical mapping is the `triggers` lists in `scripts/test-suites.mjs`; this table mirrors it for the areas that need a manual run. + +| Area touched | Run | Cost | +|---|---|---| +| `skill/scripts/live-*.{mjs,js}`, `skill/scripts/live/**` | `bun run test:live-e2e` | ~2 min, real npm installs + dev servers, needs Playwright Chromium | +| `live-accept` / `live-browser` / `live-server` / `live-wrap` / `live/sveltekit-adapter` | also `bun run test:live-e2e-accept-cleanup` | bills a provider API key | +| `live/sveltekit-adapter.mjs`, `live/svelte-component.mjs` | `bun run test:live-svelte-adapter-deepseek` | bills DeepSeek | +| `SKILL.src.md` Setup, `context.mjs`, Setup-adjacent reference files | `bun run test:skill-behavior` | ~5 min, bills all four provider keys | +| `serve-question.mjs`, `generate-image.mjs`, `concept-seed.mjs` | `bun run test:new-work-e2e` | Playwright, offline, no API cost | +| `cli/bin/commands/skills.mjs` | `bun run test:cli-remote-e2e` | hits impeccable.style | +| `plugin/`, `skill/agents/`, `scripts/build.js`, plugin manifest validator | `bun run test:plugin-e2e` | ~1 s; already in the default suite, needs the `claude` CLI | + **Plugin loader E2E** (`tests/plugin-e2e.test.mjs`, in the default suite): installs the committed `./plugin` subtree into a real Claude Code, sandboxed via `CLAUDE_CONFIG_DIR` in a temp dir, and asserts the component inventory from `claude plugin details`: the skill parses, every `plugin/agents/*.md` is visible, hooks are discovered. This is the only check that catches loader-contract surprises the unit guards can't know about (PR #494 shipped an `agents` manifest key that silently loaded zero agents; `claude plugin validate` never flags plugin-manifest problems). Runs in about a second; skips cleanly when the `claude` CLI is not on PATH. The known contract itself (allowed manifest keys, no `agents` key, trailing-slash `skills` path, source agents shipped) is pinned deterministically by `scripts/lib/validate-plugin-manifest.js`, unit-tested in `tests/validate-plugin-manifest.test.js` and enforced as a `bun run build` gate. Never add a key to the generated plugin manifest without verifying it against a real install and extending `KNOWN_LOADER_KEYS`. **Important:** `tests/build.test.js` uses `spyOn(transformers, 'transformCursor')` with the named exports from `scripts/lib/transformers/index.js`. Those named exports (`transformCursor`, `transformClaudeCode`, etc.) are kept specifically for test spying, even though `build.js` itself uses `createTransformer + PROVIDERS` directly. **Do not delete them as "dead code"** — I made that mistake once and broke 8 tests. diff --git a/cli/lib/download-providers.js b/cli/lib/download-providers.js index b54a88430..afd2abee6 100644 --- a/cli/lib/download-providers.js +++ b/cli/lib/download-providers.js @@ -1,3 +1,8 @@ +// Nothing in this repo imports this module. Its consumers are the Cloudflare +// Pages Functions in the private impeccable-site repo (functions/api/download/ +// bundle/[provider].js and [type]/[provider]/[id].js), which share cli/lib/. +// Do not remove it as dead code; keep the provider list in sync with the +// harness dirs the build emits. export const FILE_DOWNLOAD_PROVIDER_CONFIG_DIRS = Object.freeze({ cursor: '.cursor', 'claude-code': '.claude', diff --git a/scripts/run-tests.mjs b/scripts/run-tests.mjs index 0841d3da8..556d5a54e 100644 --- a/scripts/run-tests.mjs +++ b/scripts/run-tests.mjs @@ -40,13 +40,17 @@ function runCommand(command) { } if (command.runner === 'node') { - for (const file of command.files) { - const args = ['--test']; - if (command.timeoutMs) args.push(`--test-timeout=${command.timeoutMs}`); - if (command.forceExit) args.push('--test-force-exit'); - args.push(file); - runProcess(process.execPath, args, { env }); - } + // One invocation for the whole file list: node --test runs each file in + // its own child process regardless, so isolation is unchanged, but the + // runner-per-file spawn overhead is gone and files execute concurrently. + // Measured on the live suite (38 files): 52s serial-per-file vs 18s + // batched at concurrency 4. Suites can pin `concurrency: 1` if their + // tests ever contend for a shared resource. + const args = ['--test', `--test-concurrency=${command.concurrency ?? 4}`]; + if (command.timeoutMs) args.push(`--test-timeout=${command.timeoutMs}`); + if (command.forceExit) args.push('--test-force-exit'); + args.push(...command.files); + runProcess(process.execPath, args, { env }); return; } diff --git a/scripts/test-suites.mjs b/scripts/test-suites.mjs index a29b74c05..9bb675c1f 100644 --- a/scripts/test-suites.mjs +++ b/scripts/test-suites.mjs @@ -29,8 +29,6 @@ export const SUITES = { /^skill\/(SKILL\.src\.md|agents\/|reference\/|scripts\/(cleanup-deprecated|concept-seed|context|context-signals|critique-storage|design-parser|doctor|hook|impeccable-paths|is-generated|lib\/(artifact-schema|composition-catalog|concept-catalog|provider|staleness|staleness-deep|staleness-notice|surface-briefs|target-slug|template-extensions)|pin|surface-brief))/, /^README(\.npm)?\.md$/, /^cli\/bin\//, - /^tests\/(build|cleanup-deprecated|cli-args|cli-ignores|concept-seed|context|context-signals|critique-storage|design-parser|doctor|github-sheriff|hook|hook-build|impeccable-paths|openai-plugin|pin|skills-cli|staleness|surface-brief|target-args|template-extensions|test-suites|windows-path-fix|zip)\.test\.(js|mjs)$/, - /^tests\/lib\//, ], commands: [ { @@ -67,6 +65,7 @@ export const SUITES = { 'tests/impeccable-paths.test.mjs', 'tests/openai-plugin.test.mjs', 'tests/pin.test.mjs', + 'tests/release.test.mjs', 'tests/doctor.test.mjs', 'tests/staleness.test.mjs', 'tests/target-args.test.mjs', @@ -87,8 +86,7 @@ export const SUITES = { /^extension\/(background|content|detector|devtools|popup|manifest\.json)/, /^scripts\/(benchmark-detector|build-browser-detector|build-extension)\.js$/, /^site\/(pages\/detector|public\/antipattern|data\/anti-patterns-catalog\.js)/, - /^tests\/design-system\.test\.mjs$/, - /^tests\/(detect-antipatterns|detect-cli-design-contamination|detect-url-launch|inline-ignores|extension-build|fixtures\/antipatterns)/, + /^tests\/fixtures\/antipatterns/, ], commands: [ { @@ -116,10 +114,11 @@ export const SUITES = { description: 'Fast live-mode unit and local-server integration tests, excluding full browser fixture sweeps.', triggers: [ ...COMMON_INFRA_PATTERNS, - /^skill\/(reference\/live\.md|scripts\/(detect-csp|lib\/is-generated|lib\/template-extensions|live\/|live|live-|modern-screenshot|pin|palette))/, + // `palette` is deliberately absent: skill/scripts/palette.mjs has no + // test anywhere, and listing it here made edits run a suite that never + // touches it, which reads as coverage that does not exist. + /^skill\/(reference\/live\.md|scripts\/(detect-csp|lib\/is-generated|lib\/template-extensions|live\/|live|live-|modern-screenshot|pin))/, /^tests\/live-/, - /^tests\/live-e2e\/(agent|agents\/llm-agent|cli-options|preactions|session|steer|ui)\.mjs$/, - /^tests\/live-e2e\/agent-insert\.test\.mjs$/, ], commands: [ { @@ -339,6 +338,25 @@ export const SUITES = { }, }; +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Every suite must select itself when one of its own test files changes. +// Generated from the files lists so the hand-written trigger patterns above +// only carry source paths and fixture directories; before this, four test +// files were registered in a suite that change-based CI could never select +// by editing them (serve-question, ci-test-plan, both validate-plugin-*), +// and tests/lib/detector-bundle.test.js triggered core while running in +// detector. The meta-test in tests/test-suites.test.mjs pins this invariant. +for (const suite of Object.values(SUITES)) { + const ownFiles = suite.commands.flatMap((command) => command.files); + suite.triggers = [ + ...(suite.triggers ?? []), + ...ownFiles.map((file) => new RegExp(`^${escapeRegExp(file)}$`)), + ]; +} + export function expandSuites(requested) { const names = requested.length === 0 ? ['default'] : requested; const expanded = []; diff --git a/skill/scripts/live/ui-core.mjs b/skill/scripts/live/ui-core.mjs deleted file mode 100644 index bbb2c5c35..000000000 --- a/skill/scripts/live/ui-core.mjs +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Framework-neutral Impeccable live chrome contract. - * - * The production browser bundle is intentionally plain DOM so Svelte, React, - * Vue, and static adapters can all mount the same chrome. This module is the - * testable contract/inventory for that bundle; live-browser.js mirrors these - * values at runtime because it is served as a standalone script. - */ - -export const LIVE_CHROME_MOUNT_CONTRACT = Object.freeze([ - 'root', - 'transport', - 'state', - 'actions', -]); - -export const LIVE_UI_SURFACES = Object.freeze([ - { - key: 'global-bottom-bar', - ids: [ - 'impeccable-live-global-bar', - 'impeccable-live-global-bar-brand', - 'impeccable-live-pick-toggle', - 'impeccable-live-insert-toggle', - 'impeccable-live-detect-toggle', - 'impeccable-live-detect-badge', - 'impeccable-live-design-toggle', - 'impeccable-live-page-chat', - 'impeccable-live-page-chat-input', - 'impeccable-live-page-chat-voice', - ], - states: ['rest', 'hover', 'focus-visible', 'pressed', 'active', 'tooltip'], - }, - { - key: 'pending-copy-edit-dock', - ids: ['impeccable-live-pending-dock'], - states: ['closed', 'open', 'hover', 'pressed', 'loading', 'rollback', 'keep-fixing'], - }, - { - key: 'element-selection-chrome', - ids: [ - 'impeccable-live-highlight', - 'impeccable-live-tooltip', - 'impeccable-live-bar', - 'impeccable-live-selection-pill', - 'impeccable-live-input', - 'impeccable-live-configure-voice', - 'impeccable-live-configure-bar-tooltip', - ], - states: ['rest', 'hover', 'focus-visible', 'pressed', 'disabled'], - }, - { - key: 'action-picker', - ids: ['impeccable-live-picker'], - states: ['closed', 'open', 'option-hover', 'option-focus'], - }, - { - key: 'edit-chrome', - ids: ['impeccable-live-edit-badge'], - states: ['enabled', 'disabled', 'editing', 'cancel', 'save', 'edited-content'], - }, - { - key: 'generating-row', - ids: ['impeccable-live-bar', 'impeccable-live-shader'], - states: ['action-label', 'animated-dots', 'generating', 'done'], - }, - { - key: 'variant-cycling-row', - ids: ['impeccable-live-bar', 'impeccable-live-params-panel'], - states: ['variant-1', 'variant-2', 'variant-3', 'left-disabled', 'right-disabled', 'dot-click', 'accept', 'discard'], - }, - { - key: 'variant-params-panel', - ids: ['impeccable-live-params-panel'], - states: ['closed', 'open-above', 'open-below', 'range', 'steps', 'toggle'], - }, - { - key: 'saving-confirmed-rows', - ids: ['impeccable-live-bar'], - states: ['saving', 'applying-variant', 'confirmed'], - }, - { - key: 'insert-mode-chrome', - ids: [ - 'impeccable-live-insert-line', - 'impeccable-live-insert-placeholder', - 'impeccable-live-placeholder-resize', - 'impeccable-live-insert-input', - 'impeccable-live-insert-voice', - 'impeccable-live-insert-create', - 'impeccable-live-insert-create-tooltip', - ], - states: ['toggle-active', 'line', 'placeholder', 'resize', 'enabled', 'disabled', 'tooltip'], - }, - { - key: 'annotation-chrome', - ids: [ - 'impeccable-live-annot', - 'impeccable-live-annot-svg', - 'impeccable-live-annot-pins', - 'impeccable-live-annot-clear', - ], - states: ['overlay', 'drawing', 'pin', 'pin-edit', 'clear'], - }, - { - key: 'design-system-panel', - ids: ['impeccable-live-design-host'], - states: ['closed', 'open', 'tabs', 'token-tiles', 'copy'], - }, - { - key: 'toasts-and-errors', - ids: ['impeccable-live-toast'], - states: ['normal', 'error', 'no-variants-mounted'], - }, - { - key: 'css-isolation-boundary', - ids: ['impeccable-live-root'], - states: ['shadow-root', 'style-tags', 'hostile-css'], - }, -]); - -export const LIVE_UI_COMPONENT_IDS = Object.freeze([ - ...new Set(LIVE_UI_SURFACES.flatMap((surface) => surface.ids)), -]); - -export function resolveLiveUiRoot(env = globalThis) { - const doc = env?.document; - const explicit = env?.__IMPECCABLE_LIVE_UI_ROOT__ - || env?.window?.__IMPECCABLE_LIVE_UI_ROOT__; - if (explicit && typeof explicit.appendChild === 'function') return explicit; - return doc?.body || null; -} - -export function getLiveUiElementById(id, env = globalThis) { - const doc = env?.document; - const root = resolveLiveUiRoot(env); - if (!id) return null; - if (root?.getElementById) { - const found = root.getElementById(id); - if (found) return found; - } - if (root?.querySelector) { - const found = root.querySelector('#' + escapeCssIdent(id)); - if (found) return found; - } - return doc?.getElementById?.(id) || null; -} - -export function appendToLiveUiRoot(el, env = globalThis) { - const root = resolveLiveUiRoot(env); - if (!root) throw new Error('Impeccable live UI root is not available'); - root.appendChild(el); - return el; -} - -export function appendStyleToLiveUiRoot(styleEl, env = globalThis) { - const doc = env?.document; - const root = resolveLiveUiRoot(env); - if (root && root !== doc?.body) { - root.appendChild(styleEl); - } else { - (doc?.head || doc?.body || root).appendChild(styleEl); - } - return styleEl; -} - -export function activeElementDeep(doc = globalThis.document) { - let active = doc?.activeElement || null; - while (active?.shadowRoot?.activeElement) { - active = active.shadowRoot.activeElement; - } - return active; -} - -function escapeCssIdent(value) { - if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') { - return CSS.escape(String(value)); - } - return String(value).replace(/([ !"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, '\\$1'); -} diff --git a/tests/release.test.mjs b/tests/release.test.mjs new file mode 100644 index 000000000..4cbc3f999 --- /dev/null +++ b/tests/release.test.mjs @@ -0,0 +1,223 @@ +/** + * Guard tests for scripts/release.mjs, the tagging/publishing script for the + * three independently versioned components. Until now it had zero coverage + * while owning every refusal that protects a public release: dirty tree, + * unpushed HEAD, existing tag, disagreeing manifests, missing changelog + * entry, missing artifacts. + * + * The script resolves repoRoot from its own file location and runs top-level + * code on import, so these tests copy it into a disposable git repo (with a + * local bare `origin`) and spawn it exactly as a maintainer would. Every run + * uses --dry-run, which skips all mutating steps (tag, push, gh release, + * builds) but exercises every guard on the way there. + */ +import { describe, it, before, after, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const RELEASE_SCRIPT = path.join(REPO_ROOT, 'scripts', 'release.mjs'); + +const CHANGELOG = `--- +--- +
+
v1.2.3
+
    +
  • Loader contract pinned. Uses plugin.json checks & a guide.
    • +
  • Faster runner. Batched invocations cut wall time.
    • +
+
+
+
CLI v9.9.9
+
    +
  • New detect flags. Adds --fast.
    • +
+
+`; + +function git(cwd, ...args) { + return execFileSync('git', args, { cwd, encoding: 'utf-8' }).trim(); +} + +function runRelease(cwd, ...args) { + try { + const stdout = execFileSync(process.execPath, ['scripts/release.mjs', ...args, '--dry-run'], { + cwd, + encoding: 'utf-8', + timeout: 60000, + }); + return { code: 0, stdout, stderr: '' }; + } catch (err) { + return { code: err.status ?? 1, stdout: err.stdout ?? '', stderr: err.stderr ?? '' }; + } +} + +describe('release.mjs guards', () => { + let root; + let workDir; + let bareDir; + let baselineSha; + + const write = (rel, contents) => { + const abs = path.join(workDir, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, contents); + }; + + before(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-release-')); + bareDir = path.join(root, 'origin.git'); + workDir = path.join(root, 'work'); + execFileSync('git', ['init', '--bare', bareDir]); + fs.mkdirSync(workDir); + git(workDir, 'init', '-b', 'main'); + git(workDir, 'config', 'user.email', 'test@example.com'); + git(workDir, 'config', 'user.name', 'Release Test'); + + fs.mkdirSync(path.join(workDir, 'scripts')); + fs.copyFileSync(RELEASE_SCRIPT, path.join(workDir, 'scripts', 'release.mjs')); + write('.claude-plugin/plugin.json', JSON.stringify({ name: 'impeccable', version: '1.2.3' })); + write('.claude-plugin/marketplace.json', JSON.stringify({ plugins: [{ name: 'impeccable', version: '1.2.3' }] })); + write('package.json', JSON.stringify({ name: 'impeccable', version: '9.9.9' })); + write('extension/manifest.json', JSON.stringify({ version: '2.0.0' })); + write('site/pages/changelog.astro', CHANGELOG); + write('dist/universal.zip', 'zip'); + write('dist/extension.zip', 'zip'); + write('dist/extension-firefox.zip', 'zip'); + + git(workDir, 'add', '-A'); + git(workDir, 'commit', '-m', 'fixture'); + git(workDir, 'remote', 'add', 'origin', bareDir); + git(workDir, 'push', '-u', 'origin', 'main'); + baselineSha = git(workDir, 'rev-parse', 'HEAD'); + }); + + after(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + beforeEach(() => { + // Undo whatever the previous scenario staged, on both ends: local tree + // and tags back to the baseline commit, and origin force-reset too, since + // several scenarios push commits or tags that would poison later ones. + git(workDir, 'checkout', '--', '.'); + git(workDir, 'clean', '-fd'); + git(workDir, 'reset', '--hard', baselineSha); + git(workDir, 'push', '--force', 'origin', 'main'); + for (const tag of git(workDir, 'tag').split('\n').filter(Boolean)) { + git(workDir, 'tag', '-d', tag); + } + // --refs excludes the peeled `^{}` lines annotated tags produce, which + // are not deletable refs and would abort the cleanup. + for (const line of git(workDir, 'ls-remote', '--refs', '--tags', 'origin').split('\n').filter(Boolean)) { + const ref = line.split('\t')[1]; + if (ref) git(workDir, 'push', 'origin', `:${ref}`); + } + }); + + it('dry-runs a clean skill release end to end', () => { + const { code, stdout } = runRelease(workDir, 'skill'); + assert.equal(code, 0, stdout); + assert.match(stdout, /Skill 1\.2\.3/); + assert.match(stdout, /tag is free/); + assert.match(stdout, /\[dry-run\] git tag -a skill-v1\.2\.3/); + assert.match(stdout, /\[dry-run\] gh release create skill-v1\.2\.3/); + }); + + it('converts the changelog entry to markdown release notes', () => { + const { code, stdout } = runRelease(workDir, 'skill'); + assert.equal(code, 0, stdout); + assert.match(stdout, /- \*\*Loader contract pinned\.\*\* Uses `plugin\.json` checks & a \[guide\]\(https:\/\/example\.com\/docs\)\./); + assert.match(stdout, /- \*\*Faster runner\.\*\*/); + }); + + it('renders a tweet within the 280-char limit with the release URL', () => { + const { code, stdout } = runRelease(workDir, 'skill'); + assert.equal(code, 0, stdout); + const tweetMatch = stdout.match(/--- Tweet \((\d+)\/280 chars\)[^\n]*---\n([\s\S]*?)\n--- end tweet ---/); + assert.ok(tweetMatch, `no tweet block in output:\n${stdout}`); + assert.ok(Number(tweetMatch[1]) <= 280); + assert.match(tweetMatch[2], /Impeccable v1\.2\.3 is out\./); + assert.match(tweetMatch[2], /releases\/tag\/skill-v1\.2\.3/); + assert.match(tweetMatch[2], /• Loader contract pinned/); + }); + + it('matches the prefixed changelog label for the CLI component', () => { + const { code, stdout } = runRelease(workDir, 'cli'); + assert.equal(code, 0, stdout); + assert.match(stdout, /CLI 9\.9\.9/); + assert.match(stdout, /- \*\*New detect flags\.\*\*/); + }); + + it('refuses an unknown component', () => { + const { code, stderr } = runRelease(workDir, 'website'); + assert.equal(code, 1); + assert.match(stderr, /usage: release\.mjs/); + }); + + it('refuses a dirty working tree', () => { + write('README.md', 'uncommitted'); + const { code, stderr } = runRelease(workDir, 'skill'); + assert.equal(code, 1); + assert.match(stderr, /Working tree is dirty/); + }); + + it('refuses when HEAD is ahead of origin', () => { + write('note.txt', 'ahead'); + git(workDir, 'add', '-A'); + git(workDir, 'commit', '-m', 'unpushed'); + const { code, stderr } = runRelease(workDir, 'skill'); + assert.equal(code, 1); + assert.match(stderr, /Push your commits first/); + }); + + it('refuses when the tag already exists locally', () => { + git(workDir, 'tag', 'skill-v1.2.3'); + const { code, stderr } = runRelease(workDir, 'skill'); + assert.equal(code, 1); + assert.match(stderr, /already exists locally/); + }); + + it('refuses when the tag already exists on origin', () => { + git(workDir, 'tag', 'skill-v1.2.3'); + git(workDir, 'push', 'origin', 'skill-v1.2.3'); + git(workDir, 'tag', '-d', 'skill-v1.2.3'); + const { code, stderr } = runRelease(workDir, 'skill'); + assert.equal(code, 1); + assert.match(stderr, /already exists on origin/); + }); + + it('refuses when plugin.json and marketplace.json disagree', () => { + write('.claude-plugin/marketplace.json', JSON.stringify({ plugins: [{ name: 'impeccable', version: '1.0.0' }] })); + git(workDir, 'add', '-A'); + git(workDir, 'commit', '-m', 'mismatch'); + git(workDir, 'push', 'origin', 'main'); + const { code, stderr } = runRelease(workDir, 'skill'); + assert.equal(code, 1); + assert.match(stderr, /disagree\. Bump both\./); + }); + + it('refuses when the changelog entry is missing', () => { + write('extension/manifest.json', JSON.stringify({ version: '3.0.0' })); + git(workDir, 'add', '-A'); + git(workDir, 'commit', '-m', 'bump without changelog'); + git(workDir, 'push', 'origin', 'main'); + const { code, stderr } = runRelease(workDir, 'extension'); + assert.equal(code, 1); + assert.match(stderr, /No changelog entry found for "Extension v3\.0\.0"/); + }); + + it('refuses when a release artifact is missing', () => { + fs.rmSync(path.join(workDir, 'dist/universal.zip')); + git(workDir, 'add', '-A'); + git(workDir, 'commit', '-m', 'drop artifact'); + git(workDir, 'push', 'origin', 'main'); + const { code, stderr } = runRelease(workDir, 'skill'); + assert.equal(code, 1); + assert.match(stderr, /Missing artifact: dist\/universal\.zip/); + }); +}); diff --git a/tests/test-suites.test.mjs b/tests/test-suites.test.mjs index 72f0436fb..d2621b52a 100644 --- a/tests/test-suites.test.mjs +++ b/tests/test-suites.test.mjs @@ -7,6 +7,7 @@ import { SUITES, expandSuites, findTestFiles, + matchesSuiteTriggers, suiteFiles, } from '../scripts/test-suites.mjs'; @@ -36,4 +37,33 @@ describe('test suite registry', () => { assert.equal(SUITES[suite].optIn, undefined, `${suite} should not be opt-in`); } }); + + it('selects every suite when one of its own test files changes', () => { + // Change-based CI (ci-test-plan.mjs) picks suites via matchesSuiteTriggers. + // A test file whose edits select no suite, or only a suite that does not + // run it, is a silent CI gap; the generated own-file triggers close it. + for (const [name, suite] of Object.entries(SUITES)) { + for (const file of suiteFiles([name])) { + assert.equal( + matchesSuiteTriggers(name, [file]), + true, + `editing ${file} must select suite "${name}"`, + ); + } + } + }); + + it('never lets a test file trigger only suites that do not run it', () => { + const allSuiteNames = Object.keys(SUITES); + for (const file of findTestFiles()) { + const triggered = allSuiteNames.filter((name) => matchesSuiteTriggers(name, [file])); + const runsIn = allSuiteNames.filter((name) => suiteFiles([name]).includes(file)); + const useful = triggered.filter((name) => runsIn.includes(name)); + assert.ok( + useful.length > 0, + `${file} triggers [${triggered.join(', ')}] but runs in [${runsIn.join(', ')}]; ` + + 'at least one triggered suite must actually run it', + ); + } + }); });