mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-12 06:06:37 +03:00
[codex] Improve CI test coverage (#212)
* Improve CI test coverage * Stabilize live E2E harness * Shard live E2E CI * Cache live E2E CI dependencies * Stabilize live E2E smoke CI * Update generated live browser bundles * Tighten live E2E smoke runtime * Prevent live E2E smoke hangs * Stabilize live E2E CI coverage * Fix stale accept DOM cleanup * Regenerate live browser outputs
This commit is contained in:
Paul Bakaus
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { DEFAULT_SUITES, matchesSuiteTriggers } from './test-suites.mjs';
|
||||
|
||||
const eventName = process.env.GITHUB_EVENT_NAME || '';
|
||||
const localNoChanges = !eventName && !process.env.CI_CHANGED_FILES;
|
||||
const changedFiles = localNoChanges ? [] : getChangedFiles();
|
||||
const forceDeterministic = localNoChanges || eventName === 'push' || eventName === 'workflow_dispatch';
|
||||
const forceOptIn = eventName === 'workflow_dispatch';
|
||||
|
||||
const plan = {
|
||||
core: true,
|
||||
detector: forceDeterministic || matchesSuiteTriggers('detector', changedFiles),
|
||||
live: forceDeterministic || matchesSuiteTriggers('live', changedFiles),
|
||||
framework: forceDeterministic || matchesSuiteTriggers('framework', changedFiles),
|
||||
cli_remote_e2e: forceOptIn,
|
||||
live_e2e: forceOptIn || matchesSuiteTriggers('live-e2e', changedFiles),
|
||||
live_e2e_accept_cleanup: forceOptIn || matchesSuiteTriggers('live-e2e-accept-cleanup', changedFiles),
|
||||
skill_behavior: forceOptIn || matchesSuiteTriggers('skill-behavior', changedFiles),
|
||||
live_svelte_adapter_deepseek: forceOptIn || matchesSuiteTriggers('live-svelte-adapter-deepseek', changedFiles),
|
||||
};
|
||||
|
||||
writeGithubOutputs(plan);
|
||||
printSummary(plan, changedFiles);
|
||||
|
||||
function getChangedFiles() {
|
||||
if (process.env.CI_CHANGED_FILES) {
|
||||
return process.env.CI_CHANGED_FILES
|
||||
.split(/\r?\n/)
|
||||
.map((file) => file.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
const event = process.env.GITHUB_EVENT_NAME || '';
|
||||
const sha = process.env.GITHUB_SHA || 'HEAD';
|
||||
|
||||
if (event === 'pull_request' && process.env.GITHUB_BASE_REF) {
|
||||
const base = `origin/${process.env.GITHUB_BASE_REF}`;
|
||||
return gitDiffNames(`${base}...${sha}`) || gitDiffNames(`${base}...HEAD`) || allChanged();
|
||||
}
|
||||
|
||||
const before = process.env.GITHUB_EVENT_BEFORE;
|
||||
if (before && !/^0+$/.test(before)) {
|
||||
return gitDiffNames(`${before}..${sha}`) || allChanged();
|
||||
}
|
||||
|
||||
return allChanged();
|
||||
}
|
||||
|
||||
function allChanged() {
|
||||
return git(['ls-files']).split(/\r?\n/).filter(Boolean);
|
||||
}
|
||||
|
||||
function gitDiffNames(range) {
|
||||
try {
|
||||
return git(['diff', '--name-only', range]).split(/\r?\n/).filter(Boolean);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function git(args) {
|
||||
return execFileSync('git', args, { encoding: 'utf-8' });
|
||||
}
|
||||
|
||||
function writeGithubOutputs(outputs) {
|
||||
const outputPath = process.env.GITHUB_OUTPUT;
|
||||
if (!outputPath) return;
|
||||
const lines = [];
|
||||
for (const [key, value] of Object.entries(outputs)) {
|
||||
lines.push(`${key}=${value ? 'true' : 'false'}`);
|
||||
}
|
||||
fs.appendFileSync(outputPath, lines.join('\n') + '\n');
|
||||
}
|
||||
|
||||
function printSummary(outputs, files) {
|
||||
const deterministic = DEFAULT_SUITES.map((name) => `${name}=${outputs[name]}`).join(' ');
|
||||
console.log(`Event: ${eventName || 'local'}`);
|
||||
console.log(`Changed files: ${files.length}`);
|
||||
console.log(`Deterministic suites: ${deterministic}`);
|
||||
console.log(
|
||||
[
|
||||
`cli_remote_e2e=${outputs.cli_remote_e2e}`,
|
||||
`live_e2e=${outputs.live_e2e}`,
|
||||
`live_e2e_accept_cleanup=${outputs.live_e2e_accept_cleanup}`,
|
||||
`skill_behavior=${outputs.skill_behavior}`,
|
||||
`deepseek=${outputs.live_svelte_adapter_deepseek}`,
|
||||
].join(' '),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { DEFAULT_SUITES, OPT_IN_SUITES, SUITES, expandSuites } from './test-suites.mjs';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (args.includes('--list')) {
|
||||
printSuites();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const requestedSuites = args.filter((arg) => !arg.startsWith('-'));
|
||||
let suites;
|
||||
try {
|
||||
suites = expandSuites(requestedSuites);
|
||||
} catch (err) {
|
||||
console.error(err.message);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
for (const suiteName of suites) {
|
||||
const suite = SUITES[suiteName];
|
||||
console.log(`\n## test:${suiteName}`);
|
||||
console.log(suite.description);
|
||||
for (const command of suite.commands) {
|
||||
runCommand(command);
|
||||
}
|
||||
}
|
||||
|
||||
function runCommand(command) {
|
||||
const env = { ...process.env, ...(command.env || {}) };
|
||||
if (command.runner === 'bun') {
|
||||
runProcess('bun', ['test', ...command.files], { env });
|
||||
return;
|
||||
}
|
||||
|
||||
if (command.runner === 'node') {
|
||||
for (const file of command.files) {
|
||||
const args = ['--test'];
|
||||
if (command.timeoutMs) args.push(`--test-timeout=${command.timeoutMs}`);
|
||||
if (command.forceExit) args.push('--test-force-exit');
|
||||
args.push(file);
|
||||
runProcess(process.execPath, args, { env });
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported test runner "${command.runner}"`);
|
||||
}
|
||||
|
||||
function runProcess(cmd, args, { env }) {
|
||||
console.log(`$ ${formatCommand(cmd, args)}`);
|
||||
const result = spawnSync(cmd, args, {
|
||||
stdio: 'inherit',
|
||||
env,
|
||||
});
|
||||
if (result.error) {
|
||||
console.error(result.error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
if (result.status !== 0) process.exit(result.status || 1);
|
||||
}
|
||||
|
||||
function formatCommand(cmd, args) {
|
||||
const bin = cmd === process.execPath ? 'node' : cmd;
|
||||
return [bin, ...args].join(' ');
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Usage: node scripts/run-tests.mjs [suite...]
|
||||
|
||||
Aliases:
|
||||
default ${DEFAULT_SUITES.join(', ')}
|
||||
all-local ${DEFAULT_SUITES.join(', ')}
|
||||
all ${[...DEFAULT_SUITES, ...OPT_IN_SUITES].join(', ')}
|
||||
|
||||
Run with --list to see suite contents.`);
|
||||
}
|
||||
|
||||
function printSuites() {
|
||||
for (const [name, suite] of Object.entries(SUITES)) {
|
||||
const marker = suite.optIn ? ' (opt-in)' : '';
|
||||
console.log(`\n${name}${marker}`);
|
||||
console.log(` ${suite.description}`);
|
||||
for (const command of suite.commands) {
|
||||
console.log(` ${command.runner}:`);
|
||||
for (const file of command.files) console.log(` ${file}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
export const DEFAULT_SUITES = ['core', 'detector', 'live', 'framework'];
|
||||
export const OPT_IN_SUITES = [
|
||||
'cli-remote-e2e',
|
||||
'live-e2e',
|
||||
'live-e2e-accept-cleanup',
|
||||
'skill-behavior',
|
||||
'live-svelte-adapter-deepseek',
|
||||
];
|
||||
|
||||
const COMMON_INFRA_PATTERNS = [
|
||||
/^package\.json$/,
|
||||
/^bun\.lock$/,
|
||||
/^scripts\/run-tests\.mjs$/,
|
||||
/^scripts\/test-suites\.mjs$/,
|
||||
/^scripts\/ci-test-plan\.mjs$/,
|
||||
/^\.github\/workflows\/ci\.yml$/,
|
||||
];
|
||||
|
||||
export const SUITES = {
|
||||
core: {
|
||||
description: 'Build, provider transforms, CLI helpers, context, and storage unit tests.',
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^scripts\/(?!benchmark-detector|build-browser-detector|build-extension)/,
|
||||
/^skill\/(SKILL\.src\.md|agents\/|reference\/|scripts\/(cleanup-deprecated|context|context-signals|critique-storage|design-parser|impeccable-paths|is-generated))/,
|
||||
/^cli\/bin\//,
|
||||
/^tests\/(build|cleanup-deprecated|context|context-signals|critique-storage|design-parser|impeccable-paths|skills-cli|test-suites|windows-path-fix)\.test\.(js|mjs)$/,
|
||||
/^tests\/lib\//,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
runner: 'bun',
|
||||
files: [
|
||||
'tests/build.test.js',
|
||||
'tests/windows-path-fix.test.js',
|
||||
'tests/lib/provider-blocks.test.js',
|
||||
'tests/lib/transformers/provider-blocks.test.js',
|
||||
'tests/lib/utils.test.js',
|
||||
'tests/lib/transformers/factory.test.js',
|
||||
'tests/lib/transformers/providers.test.js',
|
||||
'tests/skills-cli.test.js',
|
||||
],
|
||||
},
|
||||
{
|
||||
runner: 'node',
|
||||
files: [
|
||||
'tests/ci-test-plan.test.mjs',
|
||||
'tests/cleanup-deprecated.test.mjs',
|
||||
'tests/context.test.mjs',
|
||||
'tests/context-signals.test.mjs',
|
||||
'tests/critique-storage.test.mjs',
|
||||
'tests/design-parser.test.mjs',
|
||||
'tests/impeccable-paths.test.mjs',
|
||||
'tests/test-suites.test.mjs',
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
detector: {
|
||||
description: 'Anti-pattern detector tests across text, jsdom fixtures, and Puppeteer browser paths.',
|
||||
needsPuppeteer: true,
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^cli\/engine\//,
|
||||
/^extension\/(background|content|detector|devtools|popup|manifest\.json)/,
|
||||
/^scripts\/(benchmark-detector|build-browser-detector|build-extension)\.js$/,
|
||||
/^site\/(pages\/detector|public\/antipattern|data\/anti-patterns-catalog\.js)/,
|
||||
/^tests\/(detect-antipatterns|fixtures\/antipatterns)/,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
runner: 'bun',
|
||||
files: [
|
||||
'tests/detect-antipatterns.test.js',
|
||||
'tests/lib/detector-bundle.test.js',
|
||||
],
|
||||
},
|
||||
{
|
||||
runner: 'node',
|
||||
files: [
|
||||
'tests/detect-antipatterns-fixtures.test.mjs',
|
||||
'tests/detect-antipatterns-browser.test.mjs',
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
live: {
|
||||
description: 'Fast live-mode unit and local-server integration tests, excluding full browser fixture sweeps.',
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^skill\/(reference\/live\.md|scripts\/(detect-csp|is-generated|live|live-|modern-screenshot|pin|palette))/,
|
||||
/^tests\/live-/,
|
||||
/^tests\/live-e2e\/(agent|agents\/llm-agent|cli-options|preactions|session|steer|ui)\.mjs$/,
|
||||
/^tests\/live-e2e\/agent-insert\.test\.mjs$/,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
runner: 'node',
|
||||
files: [
|
||||
'tests/live-accept.test.mjs',
|
||||
'tests/live-accept-scrub.test.mjs',
|
||||
'tests/live-browser-regression.test.mjs',
|
||||
'tests/live-browser-session.test.mjs',
|
||||
'tests/live-browser-source.test.mjs',
|
||||
'tests/live-commit-manual-edits.test.mjs',
|
||||
'tests/live-completion.test.mjs',
|
||||
'tests/live-copy-edit-agent.test.mjs',
|
||||
'tests/live-discard-manual-edits.test.mjs',
|
||||
'tests/live-e2e-agent-output.test.mjs',
|
||||
'tests/live-e2e-cli-options.test.mjs',
|
||||
'tests/live-e2e-llm-agent.test.mjs',
|
||||
'tests/live-e2e-steer-agent.test.mjs',
|
||||
'tests/live-e2e/agent-insert.test.mjs',
|
||||
'tests/live-event-validation.test.mjs',
|
||||
'tests/live-inject.test.mjs',
|
||||
'tests/live-insert.test.mjs',
|
||||
'tests/live-insert-ui.test.mjs',
|
||||
'tests/live-manual-edits-buffer.test.mjs',
|
||||
'tests/live-poll.test.mjs',
|
||||
'tests/live-poll-stream.test.mjs',
|
||||
'tests/live-recovery-commands.test.mjs',
|
||||
'tests/live-reference.test.mjs',
|
||||
'tests/live-server.test.mjs',
|
||||
'tests/live-session-store.test.mjs',
|
||||
'tests/live-wrap.test.mjs',
|
||||
'tests/live-wrap-buffer-aware.test.mjs',
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
framework: {
|
||||
description: 'Framework fixture coverage for live injection, CSP, generated-file detection, and wrapping.',
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^tests\/framework-fixtures/,
|
||||
/^tests\/framework-fixtures\.test\.mjs$/,
|
||||
/^skill\/scripts\/(detect-csp|is-generated|live-inject|live-sveltekit-adapter|live-wrap)\.mjs$/,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
runner: 'node',
|
||||
files: ['tests/framework-fixtures.test.mjs'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'cli-e2e': {
|
||||
description: 'Deterministic CLI install/update tests against a local universal bundle.',
|
||||
commands: [
|
||||
{
|
||||
runner: 'bun',
|
||||
files: ['tests/skills-cli.test.js'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'cli-remote-e2e': {
|
||||
description: 'Remote CLI install/update smoke tests against impeccable.style.',
|
||||
optIn: true,
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^cli\/bin\/commands\/skills\.mjs$/,
|
||||
/^tests\/skills-cli\.test\.js$/,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
runner: 'bun',
|
||||
env: { IMPECCABLE_CLI_REMOTE_E2E: '1' },
|
||||
files: ['tests/skills-cli.test.js'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'live-e2e': {
|
||||
description: 'Full Playwright live-mode click-to-accept sweep across runtime framework fixtures.',
|
||||
optIn: true,
|
||||
needsPlaywright: true,
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^skill\/scripts\/live/,
|
||||
/^tests\/framework-fixtures/,
|
||||
/^tests\/live-e2e(\.test\.mjs|\/)/,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
runner: 'node',
|
||||
timeoutMs: 600000,
|
||||
forceExit: true,
|
||||
files: ['tests/live-e2e.test.mjs'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'live-e2e-accept-cleanup': {
|
||||
description: 'Provider-backed post-accept cleanup regression.',
|
||||
optIn: true,
|
||||
needsPlaywright: true,
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^skill\/scripts\/(live-accept|live-browser|live-server|live-sveltekit-adapter|live-wrap)\.mjs$/,
|
||||
/^tests\/live-e2e-accept-cleanup-regression\.test\.mjs$/,
|
||||
/^tests\/live-e2e\//,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
runner: 'node',
|
||||
timeoutMs: 600000,
|
||||
files: ['tests/live-e2e-accept-cleanup-regression.test.mjs'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'live-e2e-agent': {
|
||||
description: 'Focused insert-mode fake-agent helper tests.',
|
||||
commands: [
|
||||
{
|
||||
runner: 'node',
|
||||
files: ['tests/live-e2e/agent-insert.test.mjs'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'skill-behavior': {
|
||||
description: 'LLM-backed skill setup behavior scenarios.',
|
||||
optIn: true,
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^skill\/SKILL\.src\.md$/,
|
||||
/^skill\/reference\/(init|document|brand|product|shape|craft|audit|polish|live)\.md$/,
|
||||
/^skill\/scripts\/(context|context-signals|detect|detect-csp)\.mjs$/,
|
||||
/^tests\/skill-behavior\//,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
runner: 'node',
|
||||
timeoutMs: 300000,
|
||||
files: ['tests/skill-behavior/scenarios.test.mjs'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'live-svelte-adapter-deepseek': {
|
||||
description: 'DeepSeek-backed Svelte adapter browser sweep.',
|
||||
optIn: true,
|
||||
needsPlaywright: true,
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^skill\/scripts\/(live-sveltekit-adapter|live-svelte-component|live-server|live-wrap)\.mjs$/,
|
||||
/^tests\/framework-fixtures\/vite8-sveltekit-stateful\//,
|
||||
/^tests\/live-svelte-adapter-deepseek\.test\.mjs$/,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
runner: 'node',
|
||||
timeoutMs: 1200000,
|
||||
files: ['tests/live-svelte-adapter-deepseek.test.mjs'],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
export function expandSuites(requested) {
|
||||
const names = requested.length === 0 ? ['default'] : requested;
|
||||
const expanded = [];
|
||||
for (const name of names) {
|
||||
if (name === 'default' || name === 'all-local') {
|
||||
expanded.push(...DEFAULT_SUITES);
|
||||
} else if (name === 'all') {
|
||||
expanded.push(...DEFAULT_SUITES, ...OPT_IN_SUITES);
|
||||
} else if (SUITES[name]) {
|
||||
expanded.push(name);
|
||||
} else {
|
||||
throw new Error(`Unknown test suite "${name}". Run: node scripts/run-tests.mjs --list`);
|
||||
}
|
||||
}
|
||||
return [...new Set(expanded)];
|
||||
}
|
||||
|
||||
export function suiteFiles(suiteNames) {
|
||||
const files = [];
|
||||
for (const name of suiteNames) {
|
||||
const suite = SUITES[name];
|
||||
if (!suite) throw new Error(`Unknown test suite "${name}"`);
|
||||
for (const command of suite.commands) {
|
||||
files.push(...command.files);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
export function findTestFiles(root = process.cwd()) {
|
||||
const out = [];
|
||||
const stack = [path.join(root, 'tests')];
|
||||
while (stack.length) {
|
||||
const dir = stack.pop();
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const abs = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
stack.push(abs);
|
||||
} else if (/\.test\.(js|mjs)$/.test(entry.name)) {
|
||||
out.push(path.relative(root, abs).split(path.sep).join('/'));
|
||||
}
|
||||
}
|
||||
}
|
||||
return out.sort();
|
||||
}
|
||||
|
||||
export function matchesSuiteTriggers(suiteName, changedFiles) {
|
||||
const suite = SUITES[suiteName];
|
||||
if (!suite) throw new Error(`Unknown test suite "${suiteName}"`);
|
||||
return changedFiles.some((file) => suite.triggers?.some((pattern) => pattern.test(file)));
|
||||
}
|
||||
Reference in New Issue
Block a user