[codex] Improve CI test coverage (#212)

* Improve CI test coverage

* Stabilize live E2E harness

* Shard live E2E CI

* Cache live E2E CI dependencies

* Stabilize live E2E smoke CI

* Update generated live browser bundles

* Tighten live E2E smoke runtime

* Prevent live E2E smoke hangs

* Stabilize live E2E CI coverage

* Fix stale accept DOM cleanup

* Regenerate live browser outputs
This commit is contained in:
Paul Bakaus
2026-06-08 10:39:12 -07:00
committed by GitHub
parent 1aedbcf538
commit 82801a4894
30 changed files with 2585 additions and 262 deletions
+124
View File
@@ -0,0 +1,124 @@
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
const SCRIPT = 'scripts/ci-test-plan.mjs';
describe('ci-test-plan', () => {
it('keeps docs-only pull requests on the core suite', () => {
const outputs = runPlan({
GITHUB_EVENT_NAME: 'pull_request',
CI_CHANGED_FILES: 'README.md',
});
assert.equal(outputs.core, 'true');
assert.equal(outputs.detector, 'false');
assert.equal(outputs.live, 'false');
assert.equal(outputs.framework, 'false');
assert.equal(outputs.live_e2e, 'false');
assert.equal(outputs.live_e2e_accept_cleanup, 'false');
assert.equal(outputs.live_svelte_adapter_deepseek, 'false');
});
it('routes detector changes to detector tests only', () => {
const outputs = runPlan({
GITHUB_EVENT_NAME: 'pull_request',
CI_CHANGED_FILES: 'cli/engine/detect-antipatterns.mjs',
});
assert.equal(outputs.detector, 'true');
assert.equal(outputs.live, 'false');
assert.equal(outputs.framework, 'false');
});
it('routes live server changes to live unit and full live E2E lanes', () => {
const outputs = runPlan({
GITHUB_EVENT_NAME: 'pull_request',
CI_CHANGED_FILES: 'skill/scripts/live-server.mjs',
});
assert.equal(outputs.live, 'true');
assert.equal(outputs.live_e2e, 'true');
assert.equal(outputs.live_e2e_accept_cleanup, 'true');
assert.equal(outputs.live_svelte_adapter_deepseek, 'true');
assert.equal(outputs.detector, 'false');
});
it('routes skill setup changes to the skill behavior lane', () => {
const outputs = runPlan({
GITHUB_EVENT_NAME: 'pull_request',
CI_CHANGED_FILES: 'skill/SKILL.src.md',
});
assert.equal(outputs.skill_behavior, 'true');
assert.equal(outputs.detector, 'false');
assert.equal(outputs.live, 'false');
});
it('forces deterministic suites on push without forcing opt-in E2E suites', () => {
const outputs = runPlan({
GITHUB_EVENT_NAME: 'push',
CI_CHANGED_FILES: 'README.md',
});
assert.equal(outputs.core, 'true');
assert.equal(outputs.detector, 'true');
assert.equal(outputs.live, 'true');
assert.equal(outputs.framework, 'true');
assert.equal(outputs.cli_remote_e2e, 'false');
assert.equal(outputs.live_e2e, 'false');
assert.equal(outputs.live_e2e_accept_cleanup, 'false');
assert.equal(outputs.live_svelte_adapter_deepseek, 'false');
});
it('enables remote smoke suites on manual dispatch', () => {
const outputs = runPlan({
GITHUB_EVENT_NAME: 'workflow_dispatch',
CI_CHANGED_FILES: 'README.md',
});
assert.equal(outputs.cli_remote_e2e, 'true');
assert.equal(outputs.live_e2e, 'true');
assert.equal(outputs.live_e2e_accept_cleanup, 'true');
assert.equal(outputs.skill_behavior, 'true');
assert.equal(outputs.live_svelte_adapter_deepseek, 'true');
});
it('exposes planned opt-in suite outputs to workflow jobs', () => {
const workflow = readFileSync('.github/workflows/ci.yml', 'utf-8');
assert.match(workflow, /live_e2e_accept_cleanup:\s*\$\{\{\s*steps\.plan\.outputs\.live_e2e_accept_cleanup\s*\}\}/);
assert.match(workflow, /live_svelte_adapter_deepseek:\s*\$\{\{\s*steps\.plan\.outputs\.live_svelte_adapter_deepseek\s*\}\}/);
assert.match(workflow, /live-e2e-accept-cleanup:/);
assert.match(workflow, /live-svelte-adapter-deepseek:/);
});
});
function runPlan(env) {
const tmp = mkdtempSync(join(tmpdir(), 'impeccable-ci-plan-'));
const outputPath = join(tmp, 'github-output');
try {
const result = spawnSync(process.execPath, [SCRIPT], {
cwd: process.cwd(),
encoding: 'utf-8',
env: {
...process.env,
GITHUB_OUTPUT: outputPath,
...env,
},
});
assert.equal(result.status, 0, result.stderr || result.stdout);
return Object.fromEntries(
readFileSync(outputPath, 'utf-8')
.trim()
.split(/\r?\n/)
.filter(Boolean)
.map((line) => line.split('=')),
);
} finally {
rmSync(tmp, { recursive: true, force: true });
}
}
+6
View File
@@ -84,6 +84,12 @@ The `runtime` block is optional. Fixtures without it only run the static unit ch
6. Runs a **Steer smoke** step (unless `runtime.steer === false`): submit a message in the global Steer bar, wait for the fake agent to reply `steer_done`, assert the bar unlocks and a `data-impeccable-steer` marker lands in source + DOM. Then continues with pick → Go → cycle → accept.
7. Tears everything down (Playwright close, dev server SIGTERM, live-server stop, tmp rm).
Useful runtime E2E filters:
- `IMPECCABLE_E2E_ONLY=<fixture>[,<fixture>]` scopes the run to selected fixture names.
- `IMPECCABLE_E2E_SCENARIOS=core` runs only the main click → Go → cycle → accept path; omit it or use `all` to include manual edit, annotation, and exit probes.
- `IMPECCABLE_E2E_TEST_TIMEOUT_MS`, `IMPECCABLE_E2E_INSTALL_TIMEOUT_MS`, and `IMPECCABLE_E2E_DEV_READY_TIMEOUT_MS` tighten CI smoke timeouts without changing fixture metadata.
Optional `runtime.steer` fields:
```json
+40 -2
View File
@@ -267,7 +267,7 @@ describe('live-browser source contracts', () => {
it('keeps sendEvent fire-and-forget by default while accept/discard opt into rejection', () => {
assert.match(
SOURCE,
/function sendEvent\(msg, opts\)[\s\S]*if \(opts && opts\.throwOnError\) throw err;[\s\S]*return null;/,
/function sendEvent\(msg, opts\)[\s\S]*if \(opts && opts\.throwOnError\) \{[\s\S]*console\.error\('\[impeccable\] Failed to send event:', err\);[\s\S]*throw err;[\s\S]*\}[\s\S]*console\.debug\('\[impeccable\] Dropped optional live event:', err\);[\s\S]*return null;/,
'event=live_browser.send_event_contract actor=browser operation=send_event_failure risk=fire_and_forget_callers_get_unhandled_rejections expected=default swallow with opt-in throw actual=missing',
);
assert.match(SOURCE, /if \(res\.ok\) return res;[\s\S]*const body = await res\.json\(\)\.catch\(\(\) => \(\{\}\)\);[\s\S]*handleFailure\(new Error\(body\.error \|\| \('HTTP ' \+ res\.status \+ ' ' \+ res\.statusText\)\)\)/);
@@ -321,9 +321,14 @@ describe('live-browser source contracts', () => {
);
assert.match(
SOURCE,
/function ensureAcceptedDomClean\(pending\)[\s\S]*?parent\.insertBefore\(accepted\.firstChild, wrapper\);[\s\S]*?wrapper\.remove\(\);/,
/function ensureAcceptedDomClean\(pending\)[\s\S]*?findAcceptedRuntimeWrapper\(sessionId\)[\s\S]*?acceptedDomAlreadyClean\(pending\)[\s\S]*?wrapper\.remove\(\);[\s\S]*?parent\.insertBefore\(accepted\.firstChild, wrapper\);[\s\S]*?wrapper\.remove\(\);/,
'post-cleanup fallback should unwrap the accepted variant instead of preserving live runtime wrappers',
);
assert.match(
SOURCE,
/function findAcceptedRuntimeWrapper\(sessionId\)[\s\S]*?data-impeccable-variants[\s\S]*?data-impeccable-carbonize/,
'post-cleanup fallback should also remove stale carbonize wrappers left by React HMR after accept',
);
assert.match(
SOURCE,
/if \(!accepted\) \{[\s\S]{0,120}?wrapper\.remove\(\);[\s\S]{0,120}?restoreAcceptedDomFromSnapshot\(pending\);[\s\S]{0,80}?return;/,
@@ -340,4 +345,37 @@ describe('live-browser source contracts', () => {
'missing accepted DOM after clean source should recover by reloading the clean page',
);
});
it('normalizes generated JSX source before source-fallback DOM parsing', () => {
assert.match(
SOURCE,
/parser\.parseFromString\(normalizeSourceFallbackBlock\(block, filePath\), 'text\/html'\)/,
'source fallback should normalize JSX wrapper syntax before DOMParser sees it',
);
assert.match(
SOURCE,
/function normalizeSourceFallbackBlock\(block, filePath\)[\s\S]*?<style\\b\(\[\^>\]\*\)>\\s\*\\\{\\s\*`\(\[\\s\\S\]\*\?\)`\\s\*\\\}\\s\*<\\\/style>/,
'source fallback should unwrap JSX style template literals',
);
assert.match(
SOURCE,
/replace\(\/\\bclassName\\s\*=\/g, 'class='\)/,
'source fallback should translate className back to HTML class attributes',
);
assert.match(
SOURCE,
/value\.replace\(\/\\\$\\\{\[\^}\]\*\\\}\/g, ' '\)/,
'source fallback should reduce JSX template className values to literal class tokens',
);
assert.doesNotMatch(
SOURCE,
/querySelectorAll\(tag \+ '\\.' \+ cls\.split/,
'source fallback should not construct unsafe selectors from JSX-ish class strings',
);
assert.match(
SOURCE,
/function jsxStyleObjectToCss\(body\)/,
'source fallback should translate simple JSX style objects such as display:none',
);
});
});
+13 -1
View File
@@ -8,7 +8,7 @@ import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { createFakeAgent, findSteerTargetFile, runAgentLoop, STEER_MARKER_ATTR } from './live-e2e/agent.mjs';
import { addSteerMarkerToSource, createFakeAgent, findSteerTargetFile, runAgentLoop, STEER_MARKER_ATTR } from './live-e2e/agent.mjs';
import { stageFixture, startLiveServer, stopLiveServer, FIXTURES_DIR } from './live-e2e/session.mjs';
import { SCRIPTS_DIR } from './live-e2e/session.mjs';
@@ -49,6 +49,18 @@ describe('live-e2e steer agent handler', () => {
assert.match(body, /hero-title/);
});
it('marks JSX template-expression className attributes', () => {
const source = [
'export default function App() {',
' return <h1 className={`hero-title ${styles.heroTitle}`}>Fixture</h1>;',
'}',
].join('\n');
const updated = addSteerMarkerToSource(source);
assert.match(updated, new RegExp(STEER_MARKER_ATTR + '="e2e"'));
assert.match(updated, /className=\{`hero-title \$\{styles\.heroTitle\}`\}/);
});
it('agent loop handles steer POST and writes the marker', async () => {
const sourceFile = findSteerTargetFile(tmp);
const before = readFileSync(sourceFile, 'utf-8');
+229 -32
View File
@@ -22,8 +22,8 @@
import { describe, it, before, after } from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { existsSync, readdirSync, readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname, join, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createFakeAgent } from './live-e2e/agent.mjs';
@@ -76,17 +76,24 @@ function listRuntimeFixtures() {
const allFixtures = listRuntimeFixtures();
// During development of the full-cycle test, a single fixture is much faster
// to iterate on. Set IMPECCABLE_E2E_ONLY=<name> to scope the run.
const onlyName = process.env.IMPECCABLE_E2E_ONLY;
const fixtures = onlyName
? allFixtures.filter((f) => f.name === onlyName)
// During development of the full-cycle test, a fixture subset is much faster
// to iterate on. Set IMPECCABLE_E2E_ONLY=<name>[,<name>...] to scope the run.
const onlyNames = parseFixtureFilter(process.env.IMPECCABLE_E2E_ONLY);
const fixtures = onlyNames.size > 0
? allFixtures.filter((f) => onlyNames.has(f.name))
: allFixtures;
const missingOnlyNames = [...onlyNames].filter((name) => !allFixtures.some((f) => f.name === name));
if (missingOnlyNames.length > 0) {
throw new Error(`Unknown IMPECCABLE_E2E_ONLY fixture(s): ${missingOnlyNames.join(', ')}`);
}
const manualOnly = process.env.IMPECCABLE_E2E_MANUAL_ONLY === '1'
|| process.env.IMPECCABLE_E2E_MANUAL_ONLY === 'true';
const reloadVariants = process.env.IMPECCABLE_E2E_RELOAD_VARIANTS === '1'
|| process.env.IMPECCABLE_E2E_RELOAD_VARIANTS === 'true';
const scenarioNames = parseFixtureFilter(process.env.IMPECCABLE_E2E_SCENARIOS);
const liveE2eTestTimeoutMs = readPositiveIntEnv('IMPECCABLE_E2E_TEST_TIMEOUT_MS');
const liveE2eTestOptions = liveE2eTestTimeoutMs ? { timeout: liveE2eTestTimeoutMs } : {};
if (fixtures.length === 0) {
describe('live-e2e (no runtime fixtures registered)', () => {
@@ -97,6 +104,26 @@ if (fixtures.length === 0) {
let playwright;
let browser;
function parseFixtureFilter(value) {
return new Set(
String(value || '')
.split(/[,\s]+/)
.map((name) => name.trim())
.filter(Boolean),
);
}
function readPositiveIntEnv(name) {
const raw = process.env[name];
if (raw == null || raw === '') return null;
const parsed = Number(raw);
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
}
function shouldRunScenario(name) {
return scenarioNames.size === 0 || scenarioNames.has('all') || scenarioNames.has(name);
}
before(async () => {
if (fixtures.length === 0) return;
try {
@@ -107,7 +134,7 @@ before(async () => {
);
}
try {
browser = await playwright.chromium.launch({ headless: true });
browser = await launchLiveE2eBrowser();
} catch (err) {
throw new Error(`Failed to launch Chromium (${err.message}). Run: npx playwright install chromium`);
}
@@ -117,9 +144,26 @@ after(async () => {
if (browser) await browser.close();
});
async function launchLiveE2eBrowser() {
return playwright.chromium.launch({ headless: true });
}
async function teardownAndResetBrowser(teardown) {
try {
await teardown();
} finally {
if (browser) await browser.close().catch(() => {});
browser = await launchLiveE2eBrowser();
}
}
for (const { name, fixture } of fixtures) {
describe(`live-e2e · ${name} (${fixture.runtime.styling || 'unknown-styling'})`, () => {
it('drives the full click → Go → cycle → accept cycle', async (t) => {
it('drives the full click → Go → cycle → accept cycle', liveE2eTestOptions, async (t) => {
if (!shouldRunScenario('core')) {
t.skip('scenario filter excludes core');
return;
}
if (manualOnly || process.env.IMPECCABLE_E2E_MANUAL_SCENARIO) {
t.skip('manual scenario filter is active');
return;
@@ -183,6 +227,7 @@ for (const { name, fixture } of fixtures) {
? pickSelector
: '[data-impeccable-variant="2"] > :first-child';
let stateProbeBaseline = null;
let sourceFile = null;
try {
// 1. Handshake
@@ -265,7 +310,7 @@ for (const { name, fixture } of fixtures) {
}
// 5. Source-side check: wrapper + style + variants are present
const sourceFile = await locateSessionFile(tmp);
sourceFile = await locateSessionFile(tmp);
const after = readFileSync(sourceFile, 'utf-8');
const svelteComponentSession = svelteComponentTargetFor(sourceFile);
if (svelteComponentSession) {
@@ -338,17 +383,18 @@ for (const { name, fixture } of fixtures) {
const cycleSequence = Array.isArray(fixture.runtime.variantSequence) && fixture.runtime.variantSequence.length > 0
? fixture.runtime.variantSequence
: [2];
let visible = await getVisibleVariant(page);
let visible = await readVisibleVariantForCycle(page);
let checkedVariantTwoStyle = false;
for (const targetVariant of cycleSequence) {
t.diagnostic(`Cycling to variant ${targetVariant}`);
while (visible < targetVariant) {
await clickNext(page);
visible = await getVisibleVariant(page);
}
while (visible > targetVariant) {
await clickPrev(page);
visible = await getVisibleVariant(page);
let cycleAttempts = 0;
while (visible !== targetVariant) {
if (cycleAttempts++ > expectedCount + 6) {
throw new Error(`variant ${targetVariant} did not become visible; last visible=${visible}`);
}
if (visible == null || visible < targetVariant) await clickNext(page);
else await clickPrev(page);
visible = await readVisibleVariantForCycle(page);
}
assert.equal(visible, targetVariant, `variant ${targetVariant} visible`);
if (agentMode === 'fake' && targetVariant === 2 && !checkedVariantTwoStyle) {
@@ -357,11 +403,49 @@ for (const { name, fixture } of fixtures) {
const el = query(sel) || document.querySelector(sel);
return el && getComputedStyle(el).fontWeight === '900';
}, variantContentSelector, { timeout: 5_000 }).catch(() => {});
const variantWeight = await page.evaluate((sel) => {
const query = window.__impeccableLiveQuery || ((s) => document.querySelector(s));
const el = query(sel) || document.querySelector(sel);
return el ? getComputedStyle(el).fontWeight : null;
}, variantContentSelector);
const variantWeight = await evaluatePageWithTimeout(
page,
(sel) => {
const query = window.__impeccableLiveQuery || ((s) => document.querySelector(s));
const el = query(sel) || document.querySelector(sel);
return el ? getComputedStyle(el).fontWeight : null;
},
variantContentSelector,
5_000,
'variant font-weight read',
);
if (variantWeight !== '900') {
const styleSnapshot = await evaluatePageWithTimeout(
page,
(sel) => {
const query = window.__impeccableLiveQuery || ((s) => document.querySelector(s));
const el = query(sel) || document.querySelector(sel);
const styleEl = document.querySelector('style[data-impeccable-css]');
const rules = [];
for (const sheet of [...document.styleSheets]) {
if (sheet.ownerNode !== styleEl) continue;
try {
rules.push(...[...sheet.cssRules].map((rule) => rule.cssText));
} catch (err) {
rules.push(`cssRules error: ${err.message}`);
}
}
return {
selector: sel,
element: el?.outerHTML || null,
parent: el?.parentElement?.outerHTML?.slice(0, 800) || null,
computedWeight: el ? getComputedStyle(el).fontWeight : null,
styleText: styleEl?.textContent || null,
rules,
};
},
variantContentSelector,
5_000,
'variant style snapshot',
).catch((err) => ({ error: err.message }));
t.diagnostic('--- variant style snapshot ---');
t.diagnostic(JSON.stringify(styleSnapshot, null, 2));
}
assert.equal(
variantWeight,
'900',
@@ -464,6 +548,7 @@ for (const { name, fixture } of fixtures) {
assert.doesNotMatch(final, /impeccable-variants-start/, 'variants-start marker removed');
assert.doesNotMatch(final, /impeccable-carbonize-start/, 'carbonize-start marker removed');
assert.doesNotMatch(final, /impeccable-carbonize-end/, 'carbonize-end marker removed');
assert.doesNotMatch(final, /data-impeccable-carbonize=/, 'carbonize wrapper removed');
assert.doesNotMatch(final, /data-impeccable-variant="/, 'no leftover variant scaffolding');
if (isInsert) {
if (agentMode === 'fake') {
@@ -544,6 +629,14 @@ for (const { name, fixture } of fixtures) {
);
}
} catch (err) {
await captureLiveE2eFailure({
name,
fixture,
session,
sourceFile,
error: err,
log: (m) => t.diagnostic(m),
});
if (knownLimitation) {
t.diagnostic(`KNOWN LIMITATION: ${knownLimitation}`);
t.diagnostic(`Failure: ${err.message?.split('\n')[0] || err}`);
@@ -552,15 +645,15 @@ for (const { name, fixture } of fixtures) {
}
throw err;
} finally {
await teardown();
await teardownAndResetBrowser(teardown);
}
});
if (Array.isArray(fixture.runtime.manualEditScenarios) && fixture.runtime.manualEditScenarios.length > 0) {
if (shouldRunScenario('manual') && Array.isArray(fixture.runtime.manualEditScenarios) && fixture.runtime.manualEditScenarios.length > 0) {
const manualScenarioFilter = process.env.IMPECCABLE_E2E_MANUAL_SCENARIO || '';
for (const scenario of fixture.runtime.manualEditScenarios) {
if (manualScenarioFilter && !scenario.name.includes(manualScenarioFilter)) continue;
it(`Edit copy → Save → Apply/commit: ${scenario.name}`, async (t) => {
it(`Edit copy → Save → Apply/commit: ${scenario.name}`, liveE2eTestOptions, async (t) => {
const manualAgent = await createManualScenarioAgent(t, scenario);
if (!manualAgent) return;
const { agent, agentMode, probeState } = manualAgent;
@@ -591,14 +684,14 @@ for (const { name, fixture } of fixtures) {
assert.equal(probeState?.applyCalls, 1, 'manual_edit_apply event should not be redelivered after the correct ack');
}
} finally {
await teardown();
await teardownAndResetBrowser(teardown);
}
});
}
}
if (fixture.runtime.liveChrome?.annotations) {
it('uploads annotations with generate and still accepts the variant', async (t) => {
if (shouldRunScenario('annotations') && fixture.runtime.liveChrome?.annotations) {
it('uploads annotations with generate and still accepts the variant', liveE2eTestOptions, async (t) => {
if (manualOnly || process.env.IMPECCABLE_E2E_MANUAL_SCENARIO) {
t.skip('manual scenario filter is active');
return;
@@ -661,13 +754,13 @@ for (const { name, fixture } of fixtures) {
await waitForBarHidden(page);
await waitForSourceClean(sourceFile, 20_000, { svelteComponentTarget });
} finally {
await teardown();
await teardownAndResetBrowser(teardown);
}
});
}
if (fixture.runtime.liveChrome?.bottomBar) {
it('Exit removes live chrome cleanly', async (t) => {
if (shouldRunScenario('exit') && fixture.runtime.liveChrome?.bottomBar) {
it('Exit removes live chrome cleanly', liveE2eTestOptions, async (t) => {
if (manualOnly || process.env.IMPECCABLE_E2E_MANUAL_SCENARIO) {
t.skip('manual scenario filter is active');
return;
@@ -705,6 +798,89 @@ function recordGenerateEvents(agent, events) {
};
}
async function captureLiveE2eFailure({ name, fixture, session, sourceFile, error, log = () => {} }) {
const root = process.env.IMPECCABLE_E2E_ARTIFACT_DIR;
if (!root || !session?.tmp) return;
try {
const tmp = session.tmp;
const dir = join(root, `${safeArtifactName(name)}-${Date.now()}`);
mkdirSync(dir, { recursive: true });
writeFileSync(join(dir, 'error.txt'), String(error?.stack || error?.message || error || ''), 'utf-8');
writeFileSync(join(dir, 'fixture.json'), JSON.stringify(fixture, null, 2), 'utf-8');
writeFileSync(join(dir, 'console-errors.log'), (session.consoleErrors || []).join('\n'), 'utf-8');
writeFileSync(join(dir, 'dev-server.log'), session.dev?.log?.() || '', 'utf-8');
writeCommandOutput(dir, 'git-status.txt', tmp, ['status', '--short']);
writeCommandOutput(dir, 'git-diff.patch', tmp, ['diff', '--', '.']);
const locatedSource = sourceFile || await locateSessionFile(tmp).catch(() => null);
if (locatedSource && existsSync(locatedSource)) {
writeFileSync(join(dir, 'source-file.txt'), relative(tmp, locatedSource), 'utf-8');
copyFileFromTmp(tmp, locatedSource, join(dir, 'sources'));
const sourceShadow = sourceShadowTargetFor(locatedSource);
if (sourceShadow && existsSync(sourceShadow)) copyFileFromTmp(tmp, sourceShadow, join(dir, 'sources'));
const svelteTarget = svelteComponentTargetFor(locatedSource);
if (svelteTarget?.sourceFile && existsSync(svelteTarget.sourceFile)) {
copyFileFromTmp(tmp, svelteTarget.sourceFile, join(dir, 'sources'));
}
}
for (const file of walkSources(tmp)) copyFileFromTmp(tmp, file, join(dir, 'sources'));
copyDirIfExists(join(tmp, '.impeccable', 'live'), join(dir, 'impeccable-live'));
copyDirIfExists(join(tmp, 'node_modules', '.impeccable-live'), join(dir, 'impeccable-live-preview'));
if (session.page) {
const html = await withCaptureTimeout(session.page.content(), 5_000, 'page content').catch((err) => `capture failed: ${err.message}`);
writeFileSync(join(dir, 'page.html'), html, 'utf-8');
await withCaptureTimeout(
session.page.screenshot({ path: join(dir, 'page.png'), fullPage: true }),
5_000,
'page screenshot',
).catch((err) => writeFileSync(join(dir, 'screenshot-error.txt'), err.message, 'utf-8'));
}
log(`Failure artifacts written to ${dir}`);
} catch (captureErr) {
log(`Failure artifact capture failed: ${captureErr.message}`);
}
}
function writeCommandOutput(dir, fileName, cwd, args) {
try {
const output = execFileSync('git', args, { cwd, encoding: 'utf-8' });
writeFileSync(join(dir, fileName), output, 'utf-8');
} catch (err) {
writeFileSync(join(dir, fileName), [err.stdout, err.stderr, err.message].filter(Boolean).join('\n'), 'utf-8');
}
}
function copyFileFromTmp(tmp, file, destRoot) {
const rel = relative(tmp, file);
if (!rel || rel.startsWith('..')) return;
const dest = join(destRoot, rel);
mkdirSync(dirname(dest), { recursive: true });
cpSync(file, dest);
}
function copyDirIfExists(from, to) {
if (!existsSync(from)) return;
mkdirSync(dirname(to), { recursive: true });
cpSync(from, to, { recursive: true });
}
function safeArtifactName(name) {
return String(name || 'fixture').replace(/[^a-z0-9._-]+/gi, '-').replace(/^-+|-+$/g, '') || 'fixture';
}
function withCaptureTimeout(promise, timeoutMs, label) {
let timer;
const timeout = new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
});
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
}
async function createManualScenarioAgent(t, scenario = {}) {
const requested = (process.env.IMPECCABLE_E2E_MANUAL_AGENT || process.env.IMPECCABLE_E2E_AGENT || 'auto')
.trim()
@@ -1084,6 +1260,7 @@ async function waitForAcceptedDom(page, selector, { allowVariantRoot = false, ti
if (all.length < 1) return false;
for (const el of all) {
if (el.closest('[data-impeccable-variants]')) return false;
if (el.closest('[data-impeccable-carbonize]')) return false;
if (!allowVariantRoot && el.closest('[data-impeccable-variant]')) return false;
}
return true;
@@ -1140,6 +1317,25 @@ async function assertVisibleText(page, selector, text, { timeout = 20_000 } = {}
}
}
async function readVisibleVariantForCycle(page, { timeout = 5_000 } = {}) {
const start = Date.now();
let last = null;
while (Date.now() - start < timeout) {
last = await getVisibleVariant(page);
if (Number.isInteger(last) && last > 0) return last;
await page.waitForTimeout(250);
}
return last;
}
async function evaluatePageWithTimeout(page, fn, arg, timeoutMs, label) {
let timer;
const timeout = new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
});
return Promise.race([page.evaluate(fn, arg), timeout]).finally(() => clearTimeout(timer));
}
async function getServerManualEditStashCount(live, pageUrl = '/') {
const res = await fetch(
`http://localhost:${live.port}/manual-edit-stash?token=${encodeURIComponent(live.token)}&pageUrl=${encodeURIComponent(pageUrl)}`,
@@ -1233,6 +1429,7 @@ async function waitForSourceClean(filePath, timeoutMs, { svelteComponentTarget:
last.includes('data-impeccable-variants=') ||
last.includes('impeccable-variants-start') ||
last.includes('impeccable-carbonize-start') ||
last.includes('data-impeccable-carbonize=') ||
last.includes('data-impeccable-variant=');
if (!dirty) return last;
await new Promise((r) => setTimeout(r, 100));
+76 -27
View File
@@ -1747,6 +1747,10 @@ export async function runAgentLoop({
}),
signal,
});
if (completionType === 'agent_done' && acceptResult.handled === true && acceptResult.carbonize === true) {
await runLiveComplete({ tmp, scriptsDir, id: event.id });
log(`completed carbonize session ${event.id}`);
}
} catch (err) {
if (signal.aborted) return;
log('accept failed: ' + err.message);
@@ -1876,20 +1880,32 @@ export async function applySteerEdits(tmp, { file, edits }) {
async function handleSteerDeterministic(context) {
const { targetFileAbs, target } = context;
let body = await fs.readFile(targetFileAbs, 'utf-8');
const next = addSteerMarkerToSource(body, target);
if (next === body) return;
if (!next) {
const { classes = 'hero-title', tag = 'h1' } = target;
const classToken = classes.split(/\s+/)[0];
throw new Error(`steer target <${tag}.${classToken}> not found in ${targetFileAbs}`);
}
body = next;
await fs.writeFile(targetFileAbs, body, 'utf-8');
}
export function addSteerMarkerToSource(body, target = { classes: 'hero-title', tag: 'h1' }) {
const attr = `${STEER_MARKER_ATTR}="${STEER_MARKER_VALUE}"`;
if (body.includes(attr)) return;
if (body.includes(attr)) return body;
const { classes = 'hero-title', tag = 'h1' } = target;
const classToken = classes.split(/\s+/)[0];
const escapedTag = escapeRegExp(tag);
const escapedClass = escapeRegExp(classToken);
const classValue = `(?:["'][^"']*\\b${escapedClass}\\b[^"']*["']|\\{[^}]*\\b${escapedClass}\\b[^}]*\\})`;
const openTagRe = new RegExp(
`(<${tag}\\b(?=[^>]*\\b(?:className|class)=["'][^"']*\\b${classToken}\\b)[^>]*)(>)`,
`(<${escapedTag}\\b(?=[^>]*\\b(?:className|class)\\s*=\\s*${classValue})[^>]*)(>)`,
'i',
);
if (!openTagRe.test(body)) {
throw new Error(`steer target <${tag}.${classToken}> not found in ${targetFileAbs}`);
}
body = body.replace(openTagRe, `$1 ${attr}$2`);
await fs.writeFile(targetFileAbs, body, 'utf-8');
if (!openTagRe.test(body)) return null;
return body.replace(openTagRe, `$1 ${attr}$2`);
}
function findSteerTargetFileSync(tmp, target) {
@@ -1976,26 +1992,12 @@ async function runCarbonizeCleanup({ tmp, file, sessionId /* , variant */ }) {
}
// 2. Unwrap the temporary `<div data-impeccable-variant="N" ...>` placed
// around the accepted content. live-accept emits this wrapper with
// `style="display: contents"` so it doesn't affect layout. We strip the
// wrapper open/close lines and keep what's between.
// Match the opening div (any single line) followed by inner content
// followed by `</div>`, where the open carries data-impeccable-variant
// and is NOT inside a data-impeccable-variants wrapper (the variants
// wrapper has the trailing `s`).
body = body.replace(
/^([ \t]*)<div\b[^>]*\bdata-impeccable-variant="[^"]+"[^>]*>\n([\s\S]*?)\n[ \t]*<\/div>\n/m,
(match, indent, inner) => {
// Re-indent inner content to the wrapper's indent level.
const innerLines = inner.split('\n');
const innerIndent = (innerLines[0].match(/^\s*/) || [''])[0];
const dedented = innerLines.map((l) => {
if (l.startsWith(innerIndent)) return indent + l.slice(innerIndent.length);
return l;
}).join('\n');
return expandAcceptedVariantMarkup(dedented, indent) + '\n';
},
);
// around the accepted content. For JSX targets, live-accept also adds an
// outer `<div data-impeccable-carbonize>` so the carbonize block and accepted
// node occupy one child slot; strip that shell after the accepted node is
// clean.
body = unwrapDivAttributeWrapper(body, 'data-impeccable-variant', { expandSingleLineContainer: true });
body = unwrapDivAttributeWrapper(body, 'data-impeccable-carbonize');
// 3. Strip any `data-impeccable-hoist-id` attributes the normalize step
// may have injected when the model emitted inline styles. The hoisted
@@ -2007,6 +2009,49 @@ async function runCarbonizeCleanup({ tmp, file, sessionId /* , variant */ }) {
await fs.writeFile(filePath, body, 'utf-8');
}
function unwrapDivAttributeWrapper(body, attrName, { expandSingleLineContainer = false } = {}) {
const lines = String(body).split('\n');
const attrRe = new RegExp(`\\b${escapeRegExp(attrName)}=`);
for (let i = 0; i < lines.length; i++) {
if (!/<div\b/.test(lines[i]) || !attrRe.test(lines[i])) continue;
const indent = (lines[i].match(/^(\s*)/) || [''])[1];
let depth = countDivDepthDelta(lines[i]);
for (let j = i + 1; j < lines.length; j++) {
depth += countDivDepthDelta(lines[j]);
if (depth !== 0) continue;
let replacement = reindentWrapperBody(lines.slice(i + 1, j), indent).join('\n');
if (expandSingleLineContainer) {
replacement = expandAcceptedVariantMarkup(replacement, indent);
}
lines.splice(i, j - i + 1, ...replacement.split('\n'));
return lines.join('\n');
}
}
return body;
}
function countDivDepthDelta(line) {
return countMatches(line, /<div\b/g) - countMatches(line, /<\/div>/g);
}
function countMatches(value, re) {
return [...String(value || '').matchAll(re)].length;
}
function reindentWrapperBody(lines, indent) {
const firstContentLine = lines.find((line) => line.trim() !== '');
const innerIndent = (firstContentLine?.match(/^(\s*)/) || [''])[1] || '';
return lines.map((line) => {
if (line.trim() === '') return '';
if (innerIndent && line.startsWith(innerIndent)) return indent + line.slice(innerIndent.length);
return indent + line.trimStart();
});
}
function expandAcceptedVariantMarkup(source, indent) {
const lines = source.split('\n');
if (lines.length !== 1) return source;
@@ -2083,3 +2128,7 @@ async function runAccept({ tmp, scriptsDir, id, variant, discard, paramValues, p
const last = stdout.trim().split('\n').filter(Boolean).pop();
return JSON.parse(last);
}
async function runLiveComplete({ tmp, scriptsDir, id }) {
await execFileP(process.execPath, [path.join(scriptsDir, 'live-complete.mjs'), '--id', id], { cwd: tmp });
}
+58 -8
View File
@@ -51,9 +51,27 @@ export function stageFixture(name, fixture) {
return tmp;
}
export function runInstall(tmp, command) {
export function runInstall(tmp, command, { timeoutMs = readTimeoutEnv('IMPECCABLE_E2E_INSTALL_TIMEOUT_MS', 180_000) } = {}) {
const [cmd, ...args] = command;
execFileSync(cmd, args, { cwd: tmp, stdio: 'inherit' });
const installArgs = addNpmInstallDefaults(cmd, args);
try {
execFileSync(cmd, installArgs, { cwd: tmp, stdio: 'inherit', timeout: timeoutMs });
} catch (err) {
if (err.signal === 'SIGTERM' || err.signal === 'SIGKILL' || err.killed) {
err.message = `fixture dependency install timed out after ${timeoutMs}ms: ${cmd} ${installArgs.join(' ')}`;
}
throw err;
}
}
function addNpmInstallDefaults(cmd, args) {
if (cmd !== 'npm') return args;
if (!['install', 'ci'].includes(args[0])) return args;
const out = [...args];
for (const flag of ['--prefer-offline', '--no-progress']) {
if (!out.some((arg) => arg === flag || arg.startsWith(flag + '='))) out.push(flag);
}
return out;
}
// ---------------------------------------------------------------------------
@@ -121,11 +139,15 @@ export function startDevServer(tmp, runtime) {
child.stderr.on('data', capture);
const ready = new Promise((resolve, reject) => {
const readyTimeoutMs = readTimeoutEnv(
'IMPECCABLE_E2E_DEV_READY_TIMEOUT_MS',
runtime.readyTimeoutMs ?? 120_000,
);
const timeout = setTimeout(() => {
reject(new Error(
`dev server ready timeout (${runtime.readyTimeoutMs}ms). Tail:\n${bufLog.join('')}`,
`dev server ready timeout (${readyTimeoutMs}ms). Tail:\n${bufLog.join('')}`,
));
}, runtime.readyTimeoutMs ?? 120_000);
}, readyTimeoutMs);
const checkMatch = (buf) => {
const m = buf.toString().match(readyRe);
@@ -145,13 +167,27 @@ export function startDevServer(tmp, runtime) {
return { child, ready, log: () => bufLog.join('') };
}
function readTimeoutEnv(name, fallback) {
const raw = process.env[name];
if (raw == null || raw === '') return fallback;
const parsed = Number(raw);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
export async function stopDevServer(child) {
if (!child || child.killed) return;
const exited = new Promise((resolve) => child.once('exit', resolve));
if (!child || child.exitCode != null || child.signalCode != null) return;
let didExit = false;
const exited = new Promise((resolve) => child.once('exit', () => {
didExit = true;
resolve();
}));
child.kill('SIGTERM');
const timeoutPromise = new Promise((resolve) => setTimeout(resolve, 5_000));
await Promise.race([exited, timeoutPromise]);
if (!child.killed) child.kill('SIGKILL');
if (!didExit && child.exitCode == null && child.signalCode == null) {
child.kill('SIGKILL');
await Promise.race([exited, new Promise((resolve) => setTimeout(resolve, 1_000))]);
}
}
// ---------------------------------------------------------------------------
@@ -196,20 +232,27 @@ export async function bootFixtureSession({ name, fixture, browser, agent, wrapTa
};
try {
const startedAt = Date.now();
log(`installing deps`);
runInstall(tmp, runtime.install);
log(`deps installed in ${formatDuration(Date.now() - startedAt)}`);
const liveStartedAt = Date.now();
log(`starting live-server`);
live = startLiveServer(tmp);
log(`live-server ready in ${formatDuration(Date.now() - liveStartedAt)}`);
const injectStartedAt = Date.now();
log(`live-inject --port ${live.port}`);
const injectResult = runInject(tmp, live.port);
if (!injectResult.ok) throw new Error('live-inject failed: ' + JSON.stringify(injectResult));
log(`live-inject complete in ${formatDuration(Date.now() - injectStartedAt)}`);
const devStartedAt = Date.now();
log(`spawning dev server: ${runtime.devCommand.join(' ')}`);
dev = startDevServer(tmp, runtime);
const { port: devPort } = await dev.ready;
log(`dev server ready on ${devPort}`);
log(`dev server ready on ${devPort} in ${formatDuration(Date.now() - devStartedAt)}`);
// Agent loop runs concurrently — abort on teardown.
agentAbort = new AbortController();
@@ -239,10 +282,12 @@ export async function bootFixtureSession({ name, fixture, browser, agent, wrapTa
if (msg.type() === 'error') consoleErrors.push(`console.error: ${msg.text()}`);
});
const pageStartedAt = Date.now();
await page.goto(`${scheme}://127.0.0.1:${devPort}`, {
waitUntil: 'domcontentloaded',
timeout: 30_000,
});
log(`page loaded in ${formatDuration(Date.now() - pageStartedAt)}`);
return {
tmp,
@@ -260,3 +305,8 @@ export async function bootFixtureSession({ name, fixture, browser, agent, wrapTa
throw err;
}
}
function formatDuration(ms) {
if (ms < 1_000) return `${ms}ms`;
return `${(ms / 1_000).toFixed(1)}s`;
}
+51 -23
View File
@@ -139,9 +139,21 @@ function installLiveQueryHelpersInPage() {
};
}
export async function installLiveQueryHelpers(page) {
export async function installLiveQueryHelpers(page, { timeout = 5_000 } = {}) {
await page.addInitScript(installLiveQueryHelpersInPage).catch(() => {});
await page.evaluate(installLiveQueryHelpersInPage);
await withTimeout(
page.evaluate(installLiveQueryHelpersInPage),
timeout,
'install live query helpers',
);
}
function withTimeout(promise, timeout, label) {
let timer;
const timeoutPromise = new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeout}ms`)), timeout);
});
return Promise.race([promise, timeoutPromise]).finally(() => clearTimeout(timer));
}
async function clickLiveControl(page, selector) {
@@ -605,14 +617,14 @@ export async function clickPrev(page) {
}
async function clickBarButton(page, label) {
await installLiveQueryHelpers(page);
const button = page.locator(`${BAR_ID} button`, { hasText: label });
const textMatch = label instanceof RegExp
? { kind: 'regex', source: label.source, flags: label.flags }
: { kind: 'text', value: String(label) };
let lastErr;
for (let attempt = 0; attempt < 3; attempt++) {
try {
await installLiveQueryHelpers(page);
const button = page.locator(`${BAR_ID} button`, { hasText: label });
await button.click({ timeout: 5_000 });
return;
} catch (err) {
@@ -637,11 +649,19 @@ async function clickBarButton(page, label) {
}
async function dispatchBarButton(page, label) {
await installLiveQueryHelpers(page);
const textMatch = label instanceof RegExp
? { kind: 'regex', source: label.source, flags: label.flags }
: { kind: 'text', value: String(label) };
return page.evaluate(findAndClickBarButton, { barSel: BAR_ID, textMatch });
try {
await installLiveQueryHelpers(page);
const textMatch = label instanceof RegExp
? { kind: 'regex', source: label.source, flags: label.flags }
: { kind: 'text', value: String(label) };
return await withTimeout(
page.evaluate(findAndClickBarButton, { barSel: BAR_ID, textMatch }),
5_000,
'dispatch bar button',
);
} catch {
return false;
}
}
function findAndClickBarButton({ barSel, textMatch }) {
@@ -662,20 +682,28 @@ function findAndClickBarButton({ barSel, textMatch }) {
* Read the currently visible variant index (the "i" in "i/N").
*/
export async function getVisibleVariant(page) {
await installLiveQueryHelpers(page);
return page.evaluate((barSel) => {
const wrapper = window.__impeccableLiveQuery('[data-impeccable-variants]');
if (wrapper) {
const variants = [...wrapper.querySelectorAll('[data-impeccable-variant]:not([data-impeccable-variant="original"])')];
const visible = variants.find((variant) => variant.style.display !== 'none');
const idx = visible ? parseInt(visible.dataset.impeccableVariant || '0', 10) : 0;
if (idx > 0) return idx;
}
const bar = window.__impeccableLiveQuery(barSel);
if (!bar) return null;
const m = (bar.textContent || '').match(/(\d+)\s*\/\s*(\d+)/);
return m ? parseInt(m[1], 10) : null;
}, BAR_ID);
try {
await installLiveQueryHelpers(page);
return await withTimeout(
page.evaluate((barSel) => {
const wrapper = window.__impeccableLiveQuery('[data-impeccable-variants]');
if (wrapper) {
const variants = [...wrapper.querySelectorAll('[data-impeccable-variant]:not([data-impeccable-variant="original"])')];
const visible = variants.find((variant) => variant.style.display !== 'none');
const idx = visible ? parseInt(visible.dataset.impeccableVariant || '0', 10) : 0;
if (idx > 0) return idx;
}
const bar = window.__impeccableLiveQuery(barSel);
if (!bar) return null;
const m = (bar.textContent || '').match(/(\d+)\s*\/\s*(\d+)/);
return m ? parseInt(m[1], 10) : null;
}, BAR_ID),
5_000,
'read visible variant',
);
} catch {
return null;
}
}
/**
+102 -15
View File
@@ -3,10 +3,11 @@
*
* Creates real temp directories, runs the CLI, and verifies results.
*
* Pure blocks (already-installed detection, unprefix migration) run in the
* default `bun run test`. Network blocks that download the universal bundle use
* `describeNet` and run only under `bun run test:cli-e2e` (IMPECCABLE_CLI_E2E=1),
* skipping gracefully when impeccable.style is unreachable.
* Deterministic install/update coverage uses a local universal bundle override
* and runs in the default suite. Remote smoke blocks that download the
* production universal bundle use `describeRemote` and run only under
* `bun run test:cli-remote-e2e` (IMPECCABLE_CLI_REMOTE_E2E=1), skipping
* gracefully when impeccable.style is unreachable.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { execSync } from 'child_process';
@@ -57,6 +58,24 @@ function createFakeLinkSource(root, providers = ['.claude']) {
}
}
function createFakeUniversalBundle(root, providers = ['.claude', '.agents', '.cursor']) {
const bundleRoot = join(root, 'universal-bundle');
for (const provider of providers) {
const skillDir = join(bundleRoot, provider, 'skills', 'impeccable');
mkdirSync(join(skillDir, 'scripts'), { recursive: true });
writeFileSync(join(skillDir, 'SKILL.md'), [
'---',
'name: impeccable',
'version: 9.9.9-local',
'---',
'',
`Local deterministic bundle for ${provider}.`,
].join('\n'));
writeFileSync(join(skillDir, 'scripts', 'context.mjs'), 'console.log("local bundle context");\n');
}
return bundleRoot;
}
/**
* Simulate an install from the era when the CLI offered a command prefix: the
* skill lives at `<prefix>impeccable`. Optionally drop in a third-party skill
@@ -71,19 +90,19 @@ function createPrefixedInstall(root, { prefix = 'i-', providers = ['.claude'], f
// ─── Already-installed detection ─────────────────────────────────────────────
// Network e2e blocks (real bundle downloads from impeccable.style) run only
// under `bun run test:cli-e2e` (IMPECCABLE_CLI_E2E=1). The default `bun run test`
// skips them so it stays fast and works offline; when opted in they still skip
// gracefully if the bundle endpoint is unreachable.
const WANT_CLI_E2E = process.env.IMPECCABLE_CLI_E2E === '1';
// Remote e2e blocks (real bundle downloads from impeccable.style) run only
// under `bun run test:cli-remote-e2e` (IMPECCABLE_CLI_REMOTE_E2E=1). The default
// suite skips them so it stays offline and stable; when opted in they still
// skip gracefully if the bundle endpoint is unreachable.
const WANT_CLI_REMOTE_E2E = process.env.IMPECCABLE_CLI_REMOTE_E2E === '1';
let bundleReachable = false;
if (WANT_CLI_E2E) {
if (WANT_CLI_REMOTE_E2E) {
try {
execSync('curl -sfIL --max-time 10 https://impeccable.style/api/download/bundle/universal -o /dev/null', { stdio: 'pipe' });
bundleReachable = true;
} catch {}
}
const describeNet = (WANT_CLI_E2E && bundleReachable) ? describe : describe.skip;
const describeRemote = (WANT_CLI_REMOTE_E2E && bundleReachable) ? describe : describe.skip;
describe('skills install: already-installed detection', () => {
test('detects impeccable sentinel and bails', () => {
@@ -291,9 +310,77 @@ describe('skills: unprefix migration', () => {
});
});
// ─── Update fallback (direct download) ───────────────────────────────────────
// ─── Install/update from local universal bundle ──────────────────────────────
describeNet('skills update: refreshes from the universal bundle', () => {
describe('skills install/update: local universal bundle e2e', () => {
test('installs provider-specific skills into a fresh project', () => {
const tmp = mkdtempSync(join(tmpdir(), 'imp-test-local-install-'));
execSync('git init', { cwd: tmp });
const bundleRoot = createFakeUniversalBundle(tmp);
const output = run('skills install -y --providers=claude,codex,cursor', {
cwd: tmp,
env: { ...process.env, IMPECCABLE_BUNDLE_PATH: bundleRoot },
});
expect(output).toContain('Done!');
for (const provider of ['.claude', '.agents', '.cursor']) {
const skillDir = join(tmp, provider, 'skills', 'impeccable');
expect(existsSync(join(skillDir, 'SKILL.md'))).toBe(true);
expect(readFileSync(join(skillDir, 'SKILL.md'), 'utf8')).toContain(`Local deterministic bundle for ${provider}.`);
expect(existsSync(join(skillDir, 'scripts', 'context.mjs'))).toBe(true);
}
rmSync(tmp, { recursive: true, force: true });
}, 15000);
test('updates stale copied skills from the local bundle', () => {
const tmp = mkdtempSync(join(tmpdir(), 'imp-test-local-update-'));
execSync('git init', { cwd: tmp });
const bundleRoot = createFakeUniversalBundle(tmp, ['.claude']);
const skillDir = join(tmp, '.claude', 'skills', 'impeccable');
mkdirSync(skillDir, { recursive: true });
writeFileSync(join(skillDir, 'SKILL.md'), '---\nname: impeccable\nstale: true\n---\nOld content.\n');
const output = run('skills update -y', {
cwd: tmp,
env: { ...process.env, IMPECCABLE_BUNDLE_PATH: bundleRoot },
});
expect(output).toContain('Updated');
const content = readFileSync(join(skillDir, 'SKILL.md'), 'utf8');
expect(content).not.toContain('stale: true');
expect(content).toContain('version: 9.9.9-local');
expect(existsSync(join(skillDir, 'scripts', 'context.mjs'))).toBe(true);
rmSync(tmp, { recursive: true, force: true });
}, 15000);
test('--force reinstall over an old prefixed install lands on canonical impeccable', () => {
const tmp = mkdtempSync(join(tmpdir(), 'imp-test-local-force-'));
execSync('git init', { cwd: tmp });
const bundleRoot = createFakeUniversalBundle(tmp, ['.claude']);
const prefixed = join(tmp, '.claude', 'skills', 'i-impeccable');
mkdirSync(prefixed, { recursive: true });
writeFileSync(join(prefixed, 'SKILL.md'), '---\nname: i-impeccable\n---\n');
run('skills install -y --force --providers=claude', {
cwd: tmp,
env: { ...process.env, IMPECCABLE_BUNDLE_PATH: bundleRoot },
});
const skills = readdirSync(join(tmp, '.claude', 'skills'));
expect(skills).toContain('impeccable');
expect(skills).not.toContain('i-impeccable');
rmSync(tmp, { recursive: true, force: true });
}, 15000);
});
// ─── Update fallback (remote direct download smoke) ──────────────────────────
describeRemote('skills update: refreshes from the production universal bundle', () => {
let tmp;
beforeAll(() => {
@@ -328,9 +415,9 @@ describeNet('skills update: refreshes from the universal bundle', () => {
});
});
// ─── Full install e2e (downloads the universal bundle) ───────────────────────
// ─── Full install remote smoke (downloads the production universal bundle) ───
describeNet('skills install: full e2e (universal bundle download)', () => {
describeRemote('skills install: production universal bundle download', () => {
let tmp;
beforeAll(() => {
+39
View File
@@ -0,0 +1,39 @@
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import {
DEFAULT_SUITES,
OPT_IN_SUITES,
SUITES,
expandSuites,
findTestFiles,
suiteFiles,
} from '../scripts/test-suites.mjs';
describe('test suite registry', () => {
it('assigns every test file to a default or opt-in suite', () => {
const allDiscovered = findTestFiles();
const allRegistered = new Set(suiteFiles([...DEFAULT_SUITES, ...OPT_IN_SUITES]));
const missing = allDiscovered.filter((file) => !allRegistered.has(file));
assert.deepEqual(
missing,
[],
'new test files must be added to scripts/test-suites.mjs, either in a default suite or an opt-in suite',
);
});
it('keeps default local suites free of duplicate test files', () => {
const files = suiteFiles(DEFAULT_SUITES);
const duplicates = files.filter((file, index) => files.indexOf(file) !== index);
assert.deepEqual(duplicates, []);
});
it('keeps opt-in suites out of the default alias', () => {
const expanded = expandSuites(['default']);
for (const suite of expanded) {
assert.equal(SUITES[suite].optIn, undefined, `${suite} should not be opt-in`);
}
});
});