mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-19 17:46:36 +03:00
[codex] Improve CI test coverage (#212)
* Improve CI test coverage * Stabilize live E2E harness * Shard live E2E CI * Cache live E2E CI dependencies * Stabilize live E2E smoke CI * Update generated live browser bundles * Tighten live E2E smoke runtime * Prevent live E2E smoke hangs * Stabilize live E2E CI coverage * Fix stale accept DOM cleanup * Regenerate live browser outputs
This commit is contained in:
@@ -0,0 +1,124 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
const SCRIPT = 'scripts/ci-test-plan.mjs';
|
||||
|
||||
describe('ci-test-plan', () => {
|
||||
it('keeps docs-only pull requests on the core suite', () => {
|
||||
const outputs = runPlan({
|
||||
GITHUB_EVENT_NAME: 'pull_request',
|
||||
CI_CHANGED_FILES: 'README.md',
|
||||
});
|
||||
|
||||
assert.equal(outputs.core, 'true');
|
||||
assert.equal(outputs.detector, 'false');
|
||||
assert.equal(outputs.live, 'false');
|
||||
assert.equal(outputs.framework, 'false');
|
||||
assert.equal(outputs.live_e2e, 'false');
|
||||
assert.equal(outputs.live_e2e_accept_cleanup, 'false');
|
||||
assert.equal(outputs.live_svelte_adapter_deepseek, 'false');
|
||||
});
|
||||
|
||||
it('routes detector changes to detector tests only', () => {
|
||||
const outputs = runPlan({
|
||||
GITHUB_EVENT_NAME: 'pull_request',
|
||||
CI_CHANGED_FILES: 'cli/engine/detect-antipatterns.mjs',
|
||||
});
|
||||
|
||||
assert.equal(outputs.detector, 'true');
|
||||
assert.equal(outputs.live, 'false');
|
||||
assert.equal(outputs.framework, 'false');
|
||||
});
|
||||
|
||||
it('routes live server changes to live unit and full live E2E lanes', () => {
|
||||
const outputs = runPlan({
|
||||
GITHUB_EVENT_NAME: 'pull_request',
|
||||
CI_CHANGED_FILES: 'skill/scripts/live-server.mjs',
|
||||
});
|
||||
|
||||
assert.equal(outputs.live, 'true');
|
||||
assert.equal(outputs.live_e2e, 'true');
|
||||
assert.equal(outputs.live_e2e_accept_cleanup, 'true');
|
||||
assert.equal(outputs.live_svelte_adapter_deepseek, 'true');
|
||||
assert.equal(outputs.detector, 'false');
|
||||
});
|
||||
|
||||
it('routes skill setup changes to the skill behavior lane', () => {
|
||||
const outputs = runPlan({
|
||||
GITHUB_EVENT_NAME: 'pull_request',
|
||||
CI_CHANGED_FILES: 'skill/SKILL.src.md',
|
||||
});
|
||||
|
||||
assert.equal(outputs.skill_behavior, 'true');
|
||||
assert.equal(outputs.detector, 'false');
|
||||
assert.equal(outputs.live, 'false');
|
||||
});
|
||||
|
||||
it('forces deterministic suites on push without forcing opt-in E2E suites', () => {
|
||||
const outputs = runPlan({
|
||||
GITHUB_EVENT_NAME: 'push',
|
||||
CI_CHANGED_FILES: 'README.md',
|
||||
});
|
||||
|
||||
assert.equal(outputs.core, 'true');
|
||||
assert.equal(outputs.detector, 'true');
|
||||
assert.equal(outputs.live, 'true');
|
||||
assert.equal(outputs.framework, 'true');
|
||||
assert.equal(outputs.cli_remote_e2e, 'false');
|
||||
assert.equal(outputs.live_e2e, 'false');
|
||||
assert.equal(outputs.live_e2e_accept_cleanup, 'false');
|
||||
assert.equal(outputs.live_svelte_adapter_deepseek, 'false');
|
||||
});
|
||||
|
||||
it('enables remote smoke suites on manual dispatch', () => {
|
||||
const outputs = runPlan({
|
||||
GITHUB_EVENT_NAME: 'workflow_dispatch',
|
||||
CI_CHANGED_FILES: 'README.md',
|
||||
});
|
||||
|
||||
assert.equal(outputs.cli_remote_e2e, 'true');
|
||||
assert.equal(outputs.live_e2e, 'true');
|
||||
assert.equal(outputs.live_e2e_accept_cleanup, 'true');
|
||||
assert.equal(outputs.skill_behavior, 'true');
|
||||
assert.equal(outputs.live_svelte_adapter_deepseek, 'true');
|
||||
});
|
||||
|
||||
it('exposes planned opt-in suite outputs to workflow jobs', () => {
|
||||
const workflow = readFileSync('.github/workflows/ci.yml', 'utf-8');
|
||||
|
||||
assert.match(workflow, /live_e2e_accept_cleanup:\s*\$\{\{\s*steps\.plan\.outputs\.live_e2e_accept_cleanup\s*\}\}/);
|
||||
assert.match(workflow, /live_svelte_adapter_deepseek:\s*\$\{\{\s*steps\.plan\.outputs\.live_svelte_adapter_deepseek\s*\}\}/);
|
||||
assert.match(workflow, /live-e2e-accept-cleanup:/);
|
||||
assert.match(workflow, /live-svelte-adapter-deepseek:/);
|
||||
});
|
||||
});
|
||||
|
||||
function runPlan(env) {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'impeccable-ci-plan-'));
|
||||
const outputPath = join(tmp, 'github-output');
|
||||
try {
|
||||
const result = spawnSync(process.execPath, [SCRIPT], {
|
||||
cwd: process.cwd(),
|
||||
encoding: 'utf-8',
|
||||
env: {
|
||||
...process.env,
|
||||
GITHUB_OUTPUT: outputPath,
|
||||
...env,
|
||||
},
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr || result.stdout);
|
||||
return Object.fromEntries(
|
||||
readFileSync(outputPath, 'utf-8')
|
||||
.trim()
|
||||
.split(/\r?\n/)
|
||||
.filter(Boolean)
|
||||
.map((line) => line.split('=')),
|
||||
);
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -84,6 +84,12 @@ The `runtime` block is optional. Fixtures without it only run the static unit ch
|
||||
6. Runs a **Steer smoke** step (unless `runtime.steer === false`): submit a message in the global Steer bar, wait for the fake agent to reply `steer_done`, assert the bar unlocks and a `data-impeccable-steer` marker lands in source + DOM. Then continues with pick → Go → cycle → accept.
|
||||
7. Tears everything down (Playwright close, dev server SIGTERM, live-server stop, tmp rm).
|
||||
|
||||
Useful runtime E2E filters:
|
||||
|
||||
- `IMPECCABLE_E2E_ONLY=<fixture>[,<fixture>]` scopes the run to selected fixture names.
|
||||
- `IMPECCABLE_E2E_SCENARIOS=core` runs only the main click → Go → cycle → accept path; omit it or use `all` to include manual edit, annotation, and exit probes.
|
||||
- `IMPECCABLE_E2E_TEST_TIMEOUT_MS`, `IMPECCABLE_E2E_INSTALL_TIMEOUT_MS`, and `IMPECCABLE_E2E_DEV_READY_TIMEOUT_MS` tighten CI smoke timeouts without changing fixture metadata.
|
||||
|
||||
Optional `runtime.steer` fields:
|
||||
|
||||
```json
|
||||
|
||||
@@ -267,7 +267,7 @@ describe('live-browser source contracts', () => {
|
||||
it('keeps sendEvent fire-and-forget by default while accept/discard opt into rejection', () => {
|
||||
assert.match(
|
||||
SOURCE,
|
||||
/function sendEvent\(msg, opts\)[\s\S]*if \(opts && opts\.throwOnError\) throw err;[\s\S]*return null;/,
|
||||
/function sendEvent\(msg, opts\)[\s\S]*if \(opts && opts\.throwOnError\) \{[\s\S]*console\.error\('\[impeccable\] Failed to send event:', err\);[\s\S]*throw err;[\s\S]*\}[\s\S]*console\.debug\('\[impeccable\] Dropped optional live event:', err\);[\s\S]*return null;/,
|
||||
'event=live_browser.send_event_contract actor=browser operation=send_event_failure risk=fire_and_forget_callers_get_unhandled_rejections expected=default swallow with opt-in throw actual=missing',
|
||||
);
|
||||
assert.match(SOURCE, /if \(res\.ok\) return res;[\s\S]*const body = await res\.json\(\)\.catch\(\(\) => \(\{\}\)\);[\s\S]*handleFailure\(new Error\(body\.error \|\| \('HTTP ' \+ res\.status \+ ' ' \+ res\.statusText\)\)\)/);
|
||||
@@ -321,9 +321,14 @@ describe('live-browser source contracts', () => {
|
||||
);
|
||||
assert.match(
|
||||
SOURCE,
|
||||
/function ensureAcceptedDomClean\(pending\)[\s\S]*?parent\.insertBefore\(accepted\.firstChild, wrapper\);[\s\S]*?wrapper\.remove\(\);/,
|
||||
/function ensureAcceptedDomClean\(pending\)[\s\S]*?findAcceptedRuntimeWrapper\(sessionId\)[\s\S]*?acceptedDomAlreadyClean\(pending\)[\s\S]*?wrapper\.remove\(\);[\s\S]*?parent\.insertBefore\(accepted\.firstChild, wrapper\);[\s\S]*?wrapper\.remove\(\);/,
|
||||
'post-cleanup fallback should unwrap the accepted variant instead of preserving live runtime wrappers',
|
||||
);
|
||||
assert.match(
|
||||
SOURCE,
|
||||
/function findAcceptedRuntimeWrapper\(sessionId\)[\s\S]*?data-impeccable-variants[\s\S]*?data-impeccable-carbonize/,
|
||||
'post-cleanup fallback should also remove stale carbonize wrappers left by React HMR after accept',
|
||||
);
|
||||
assert.match(
|
||||
SOURCE,
|
||||
/if \(!accepted\) \{[\s\S]{0,120}?wrapper\.remove\(\);[\s\S]{0,120}?restoreAcceptedDomFromSnapshot\(pending\);[\s\S]{0,80}?return;/,
|
||||
@@ -340,4 +345,37 @@ describe('live-browser source contracts', () => {
|
||||
'missing accepted DOM after clean source should recover by reloading the clean page',
|
||||
);
|
||||
});
|
||||
|
||||
it('normalizes generated JSX source before source-fallback DOM parsing', () => {
|
||||
assert.match(
|
||||
SOURCE,
|
||||
/parser\.parseFromString\(normalizeSourceFallbackBlock\(block, filePath\), 'text\/html'\)/,
|
||||
'source fallback should normalize JSX wrapper syntax before DOMParser sees it',
|
||||
);
|
||||
assert.match(
|
||||
SOURCE,
|
||||
/function normalizeSourceFallbackBlock\(block, filePath\)[\s\S]*?<style\\b\(\[\^>\]\*\)>\\s\*\\\{\\s\*`\(\[\\s\\S\]\*\?\)`\\s\*\\\}\\s\*<\\\/style>/,
|
||||
'source fallback should unwrap JSX style template literals',
|
||||
);
|
||||
assert.match(
|
||||
SOURCE,
|
||||
/replace\(\/\\bclassName\\s\*=\/g, 'class='\)/,
|
||||
'source fallback should translate className back to HTML class attributes',
|
||||
);
|
||||
assert.match(
|
||||
SOURCE,
|
||||
/value\.replace\(\/\\\$\\\{\[\^}\]\*\\\}\/g, ' '\)/,
|
||||
'source fallback should reduce JSX template className values to literal class tokens',
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
SOURCE,
|
||||
/querySelectorAll\(tag \+ '\\.' \+ cls\.split/,
|
||||
'source fallback should not construct unsafe selectors from JSX-ish class strings',
|
||||
);
|
||||
assert.match(
|
||||
SOURCE,
|
||||
/function jsxStyleObjectToCss\(body\)/,
|
||||
'source fallback should translate simple JSX style objects such as display:none',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,7 +8,7 @@ import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { createFakeAgent, findSteerTargetFile, runAgentLoop, STEER_MARKER_ATTR } from './live-e2e/agent.mjs';
|
||||
import { addSteerMarkerToSource, createFakeAgent, findSteerTargetFile, runAgentLoop, STEER_MARKER_ATTR } from './live-e2e/agent.mjs';
|
||||
import { stageFixture, startLiveServer, stopLiveServer, FIXTURES_DIR } from './live-e2e/session.mjs';
|
||||
import { SCRIPTS_DIR } from './live-e2e/session.mjs';
|
||||
|
||||
@@ -49,6 +49,18 @@ describe('live-e2e steer agent handler', () => {
|
||||
assert.match(body, /hero-title/);
|
||||
});
|
||||
|
||||
it('marks JSX template-expression className attributes', () => {
|
||||
const source = [
|
||||
'export default function App() {',
|
||||
' return <h1 className={`hero-title ${styles.heroTitle}`}>Fixture</h1>;',
|
||||
'}',
|
||||
].join('\n');
|
||||
const updated = addSteerMarkerToSource(source);
|
||||
|
||||
assert.match(updated, new RegExp(STEER_MARKER_ATTR + '="e2e"'));
|
||||
assert.match(updated, /className=\{`hero-title \$\{styles\.heroTitle\}`\}/);
|
||||
});
|
||||
|
||||
it('agent loop handles steer POST and writes the marker', async () => {
|
||||
const sourceFile = findSteerTargetFile(tmp);
|
||||
const before = readFileSync(sourceFile, 'utf-8');
|
||||
|
||||
+229
-32
@@ -22,8 +22,8 @@
|
||||
import { describe, it, before, after } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join, relative } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { createFakeAgent } from './live-e2e/agent.mjs';
|
||||
@@ -76,17 +76,24 @@ function listRuntimeFixtures() {
|
||||
|
||||
const allFixtures = listRuntimeFixtures();
|
||||
|
||||
// During development of the full-cycle test, a single fixture is much faster
|
||||
// to iterate on. Set IMPECCABLE_E2E_ONLY=<name> to scope the run.
|
||||
const onlyName = process.env.IMPECCABLE_E2E_ONLY;
|
||||
const fixtures = onlyName
|
||||
? allFixtures.filter((f) => f.name === onlyName)
|
||||
// During development of the full-cycle test, a fixture subset is much faster
|
||||
// to iterate on. Set IMPECCABLE_E2E_ONLY=<name>[,<name>...] to scope the run.
|
||||
const onlyNames = parseFixtureFilter(process.env.IMPECCABLE_E2E_ONLY);
|
||||
const fixtures = onlyNames.size > 0
|
||||
? allFixtures.filter((f) => onlyNames.has(f.name))
|
||||
: allFixtures;
|
||||
const missingOnlyNames = [...onlyNames].filter((name) => !allFixtures.some((f) => f.name === name));
|
||||
if (missingOnlyNames.length > 0) {
|
||||
throw new Error(`Unknown IMPECCABLE_E2E_ONLY fixture(s): ${missingOnlyNames.join(', ')}`);
|
||||
}
|
||||
|
||||
const manualOnly = process.env.IMPECCABLE_E2E_MANUAL_ONLY === '1'
|
||||
|| process.env.IMPECCABLE_E2E_MANUAL_ONLY === 'true';
|
||||
const reloadVariants = process.env.IMPECCABLE_E2E_RELOAD_VARIANTS === '1'
|
||||
|| process.env.IMPECCABLE_E2E_RELOAD_VARIANTS === 'true';
|
||||
const scenarioNames = parseFixtureFilter(process.env.IMPECCABLE_E2E_SCENARIOS);
|
||||
const liveE2eTestTimeoutMs = readPositiveIntEnv('IMPECCABLE_E2E_TEST_TIMEOUT_MS');
|
||||
const liveE2eTestOptions = liveE2eTestTimeoutMs ? { timeout: liveE2eTestTimeoutMs } : {};
|
||||
|
||||
if (fixtures.length === 0) {
|
||||
describe('live-e2e (no runtime fixtures registered)', () => {
|
||||
@@ -97,6 +104,26 @@ if (fixtures.length === 0) {
|
||||
let playwright;
|
||||
let browser;
|
||||
|
||||
function parseFixtureFilter(value) {
|
||||
return new Set(
|
||||
String(value || '')
|
||||
.split(/[,\s]+/)
|
||||
.map((name) => name.trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
}
|
||||
|
||||
function readPositiveIntEnv(name) {
|
||||
const raw = process.env[name];
|
||||
if (raw == null || raw === '') return null;
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
|
||||
}
|
||||
|
||||
function shouldRunScenario(name) {
|
||||
return scenarioNames.size === 0 || scenarioNames.has('all') || scenarioNames.has(name);
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (fixtures.length === 0) return;
|
||||
try {
|
||||
@@ -107,7 +134,7 @@ before(async () => {
|
||||
);
|
||||
}
|
||||
try {
|
||||
browser = await playwright.chromium.launch({ headless: true });
|
||||
browser = await launchLiveE2eBrowser();
|
||||
} catch (err) {
|
||||
throw new Error(`Failed to launch Chromium (${err.message}). Run: npx playwright install chromium`);
|
||||
}
|
||||
@@ -117,9 +144,26 @@ after(async () => {
|
||||
if (browser) await browser.close();
|
||||
});
|
||||
|
||||
async function launchLiveE2eBrowser() {
|
||||
return playwright.chromium.launch({ headless: true });
|
||||
}
|
||||
|
||||
async function teardownAndResetBrowser(teardown) {
|
||||
try {
|
||||
await teardown();
|
||||
} finally {
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
browser = await launchLiveE2eBrowser();
|
||||
}
|
||||
}
|
||||
|
||||
for (const { name, fixture } of fixtures) {
|
||||
describe(`live-e2e · ${name} (${fixture.runtime.styling || 'unknown-styling'})`, () => {
|
||||
it('drives the full click → Go → cycle → accept cycle', async (t) => {
|
||||
it('drives the full click → Go → cycle → accept cycle', liveE2eTestOptions, async (t) => {
|
||||
if (!shouldRunScenario('core')) {
|
||||
t.skip('scenario filter excludes core');
|
||||
return;
|
||||
}
|
||||
if (manualOnly || process.env.IMPECCABLE_E2E_MANUAL_SCENARIO) {
|
||||
t.skip('manual scenario filter is active');
|
||||
return;
|
||||
@@ -183,6 +227,7 @@ for (const { name, fixture } of fixtures) {
|
||||
? pickSelector
|
||||
: '[data-impeccable-variant="2"] > :first-child';
|
||||
let stateProbeBaseline = null;
|
||||
let sourceFile = null;
|
||||
|
||||
try {
|
||||
// 1. Handshake
|
||||
@@ -265,7 +310,7 @@ for (const { name, fixture } of fixtures) {
|
||||
}
|
||||
|
||||
// 5. Source-side check: wrapper + style + variants are present
|
||||
const sourceFile = await locateSessionFile(tmp);
|
||||
sourceFile = await locateSessionFile(tmp);
|
||||
const after = readFileSync(sourceFile, 'utf-8');
|
||||
const svelteComponentSession = svelteComponentTargetFor(sourceFile);
|
||||
if (svelteComponentSession) {
|
||||
@@ -338,17 +383,18 @@ for (const { name, fixture } of fixtures) {
|
||||
const cycleSequence = Array.isArray(fixture.runtime.variantSequence) && fixture.runtime.variantSequence.length > 0
|
||||
? fixture.runtime.variantSequence
|
||||
: [2];
|
||||
let visible = await getVisibleVariant(page);
|
||||
let visible = await readVisibleVariantForCycle(page);
|
||||
let checkedVariantTwoStyle = false;
|
||||
for (const targetVariant of cycleSequence) {
|
||||
t.diagnostic(`Cycling to variant ${targetVariant}`);
|
||||
while (visible < targetVariant) {
|
||||
await clickNext(page);
|
||||
visible = await getVisibleVariant(page);
|
||||
}
|
||||
while (visible > targetVariant) {
|
||||
await clickPrev(page);
|
||||
visible = await getVisibleVariant(page);
|
||||
let cycleAttempts = 0;
|
||||
while (visible !== targetVariant) {
|
||||
if (cycleAttempts++ > expectedCount + 6) {
|
||||
throw new Error(`variant ${targetVariant} did not become visible; last visible=${visible}`);
|
||||
}
|
||||
if (visible == null || visible < targetVariant) await clickNext(page);
|
||||
else await clickPrev(page);
|
||||
visible = await readVisibleVariantForCycle(page);
|
||||
}
|
||||
assert.equal(visible, targetVariant, `variant ${targetVariant} visible`);
|
||||
if (agentMode === 'fake' && targetVariant === 2 && !checkedVariantTwoStyle) {
|
||||
@@ -357,11 +403,49 @@ for (const { name, fixture } of fixtures) {
|
||||
const el = query(sel) || document.querySelector(sel);
|
||||
return el && getComputedStyle(el).fontWeight === '900';
|
||||
}, variantContentSelector, { timeout: 5_000 }).catch(() => {});
|
||||
const variantWeight = await page.evaluate((sel) => {
|
||||
const query = window.__impeccableLiveQuery || ((s) => document.querySelector(s));
|
||||
const el = query(sel) || document.querySelector(sel);
|
||||
return el ? getComputedStyle(el).fontWeight : null;
|
||||
}, variantContentSelector);
|
||||
const variantWeight = await evaluatePageWithTimeout(
|
||||
page,
|
||||
(sel) => {
|
||||
const query = window.__impeccableLiveQuery || ((s) => document.querySelector(s));
|
||||
const el = query(sel) || document.querySelector(sel);
|
||||
return el ? getComputedStyle(el).fontWeight : null;
|
||||
},
|
||||
variantContentSelector,
|
||||
5_000,
|
||||
'variant font-weight read',
|
||||
);
|
||||
if (variantWeight !== '900') {
|
||||
const styleSnapshot = await evaluatePageWithTimeout(
|
||||
page,
|
||||
(sel) => {
|
||||
const query = window.__impeccableLiveQuery || ((s) => document.querySelector(s));
|
||||
const el = query(sel) || document.querySelector(sel);
|
||||
const styleEl = document.querySelector('style[data-impeccable-css]');
|
||||
const rules = [];
|
||||
for (const sheet of [...document.styleSheets]) {
|
||||
if (sheet.ownerNode !== styleEl) continue;
|
||||
try {
|
||||
rules.push(...[...sheet.cssRules].map((rule) => rule.cssText));
|
||||
} catch (err) {
|
||||
rules.push(`cssRules error: ${err.message}`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
selector: sel,
|
||||
element: el?.outerHTML || null,
|
||||
parent: el?.parentElement?.outerHTML?.slice(0, 800) || null,
|
||||
computedWeight: el ? getComputedStyle(el).fontWeight : null,
|
||||
styleText: styleEl?.textContent || null,
|
||||
rules,
|
||||
};
|
||||
},
|
||||
variantContentSelector,
|
||||
5_000,
|
||||
'variant style snapshot',
|
||||
).catch((err) => ({ error: err.message }));
|
||||
t.diagnostic('--- variant style snapshot ---');
|
||||
t.diagnostic(JSON.stringify(styleSnapshot, null, 2));
|
||||
}
|
||||
assert.equal(
|
||||
variantWeight,
|
||||
'900',
|
||||
@@ -464,6 +548,7 @@ for (const { name, fixture } of fixtures) {
|
||||
assert.doesNotMatch(final, /impeccable-variants-start/, 'variants-start marker removed');
|
||||
assert.doesNotMatch(final, /impeccable-carbonize-start/, 'carbonize-start marker removed');
|
||||
assert.doesNotMatch(final, /impeccable-carbonize-end/, 'carbonize-end marker removed');
|
||||
assert.doesNotMatch(final, /data-impeccable-carbonize=/, 'carbonize wrapper removed');
|
||||
assert.doesNotMatch(final, /data-impeccable-variant="/, 'no leftover variant scaffolding');
|
||||
if (isInsert) {
|
||||
if (agentMode === 'fake') {
|
||||
@@ -544,6 +629,14 @@ for (const { name, fixture } of fixtures) {
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
await captureLiveE2eFailure({
|
||||
name,
|
||||
fixture,
|
||||
session,
|
||||
sourceFile,
|
||||
error: err,
|
||||
log: (m) => t.diagnostic(m),
|
||||
});
|
||||
if (knownLimitation) {
|
||||
t.diagnostic(`KNOWN LIMITATION: ${knownLimitation}`);
|
||||
t.diagnostic(`Failure: ${err.message?.split('\n')[0] || err}`);
|
||||
@@ -552,15 +645,15 @@ for (const { name, fixture } of fixtures) {
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
await teardown();
|
||||
await teardownAndResetBrowser(teardown);
|
||||
}
|
||||
});
|
||||
|
||||
if (Array.isArray(fixture.runtime.manualEditScenarios) && fixture.runtime.manualEditScenarios.length > 0) {
|
||||
if (shouldRunScenario('manual') && Array.isArray(fixture.runtime.manualEditScenarios) && fixture.runtime.manualEditScenarios.length > 0) {
|
||||
const manualScenarioFilter = process.env.IMPECCABLE_E2E_MANUAL_SCENARIO || '';
|
||||
for (const scenario of fixture.runtime.manualEditScenarios) {
|
||||
if (manualScenarioFilter && !scenario.name.includes(manualScenarioFilter)) continue;
|
||||
it(`Edit copy → Save → Apply/commit: ${scenario.name}`, async (t) => {
|
||||
it(`Edit copy → Save → Apply/commit: ${scenario.name}`, liveE2eTestOptions, async (t) => {
|
||||
const manualAgent = await createManualScenarioAgent(t, scenario);
|
||||
if (!manualAgent) return;
|
||||
const { agent, agentMode, probeState } = manualAgent;
|
||||
@@ -591,14 +684,14 @@ for (const { name, fixture } of fixtures) {
|
||||
assert.equal(probeState?.applyCalls, 1, 'manual_edit_apply event should not be redelivered after the correct ack');
|
||||
}
|
||||
} finally {
|
||||
await teardown();
|
||||
await teardownAndResetBrowser(teardown);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (fixture.runtime.liveChrome?.annotations) {
|
||||
it('uploads annotations with generate and still accepts the variant', async (t) => {
|
||||
if (shouldRunScenario('annotations') && fixture.runtime.liveChrome?.annotations) {
|
||||
it('uploads annotations with generate and still accepts the variant', liveE2eTestOptions, async (t) => {
|
||||
if (manualOnly || process.env.IMPECCABLE_E2E_MANUAL_SCENARIO) {
|
||||
t.skip('manual scenario filter is active');
|
||||
return;
|
||||
@@ -661,13 +754,13 @@ for (const { name, fixture } of fixtures) {
|
||||
await waitForBarHidden(page);
|
||||
await waitForSourceClean(sourceFile, 20_000, { svelteComponentTarget });
|
||||
} finally {
|
||||
await teardown();
|
||||
await teardownAndResetBrowser(teardown);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (fixture.runtime.liveChrome?.bottomBar) {
|
||||
it('Exit removes live chrome cleanly', async (t) => {
|
||||
if (shouldRunScenario('exit') && fixture.runtime.liveChrome?.bottomBar) {
|
||||
it('Exit removes live chrome cleanly', liveE2eTestOptions, async (t) => {
|
||||
if (manualOnly || process.env.IMPECCABLE_E2E_MANUAL_SCENARIO) {
|
||||
t.skip('manual scenario filter is active');
|
||||
return;
|
||||
@@ -705,6 +798,89 @@ function recordGenerateEvents(agent, events) {
|
||||
};
|
||||
}
|
||||
|
||||
async function captureLiveE2eFailure({ name, fixture, session, sourceFile, error, log = () => {} }) {
|
||||
const root = process.env.IMPECCABLE_E2E_ARTIFACT_DIR;
|
||||
if (!root || !session?.tmp) return;
|
||||
|
||||
try {
|
||||
const tmp = session.tmp;
|
||||
const dir = join(root, `${safeArtifactName(name)}-${Date.now()}`);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
|
||||
writeFileSync(join(dir, 'error.txt'), String(error?.stack || error?.message || error || ''), 'utf-8');
|
||||
writeFileSync(join(dir, 'fixture.json'), JSON.stringify(fixture, null, 2), 'utf-8');
|
||||
writeFileSync(join(dir, 'console-errors.log'), (session.consoleErrors || []).join('\n'), 'utf-8');
|
||||
writeFileSync(join(dir, 'dev-server.log'), session.dev?.log?.() || '', 'utf-8');
|
||||
writeCommandOutput(dir, 'git-status.txt', tmp, ['status', '--short']);
|
||||
writeCommandOutput(dir, 'git-diff.patch', tmp, ['diff', '--', '.']);
|
||||
|
||||
const locatedSource = sourceFile || await locateSessionFile(tmp).catch(() => null);
|
||||
if (locatedSource && existsSync(locatedSource)) {
|
||||
writeFileSync(join(dir, 'source-file.txt'), relative(tmp, locatedSource), 'utf-8');
|
||||
copyFileFromTmp(tmp, locatedSource, join(dir, 'sources'));
|
||||
const sourceShadow = sourceShadowTargetFor(locatedSource);
|
||||
if (sourceShadow && existsSync(sourceShadow)) copyFileFromTmp(tmp, sourceShadow, join(dir, 'sources'));
|
||||
const svelteTarget = svelteComponentTargetFor(locatedSource);
|
||||
if (svelteTarget?.sourceFile && existsSync(svelteTarget.sourceFile)) {
|
||||
copyFileFromTmp(tmp, svelteTarget.sourceFile, join(dir, 'sources'));
|
||||
}
|
||||
}
|
||||
|
||||
for (const file of walkSources(tmp)) copyFileFromTmp(tmp, file, join(dir, 'sources'));
|
||||
copyDirIfExists(join(tmp, '.impeccable', 'live'), join(dir, 'impeccable-live'));
|
||||
copyDirIfExists(join(tmp, 'node_modules', '.impeccable-live'), join(dir, 'impeccable-live-preview'));
|
||||
|
||||
if (session.page) {
|
||||
const html = await withCaptureTimeout(session.page.content(), 5_000, 'page content').catch((err) => `capture failed: ${err.message}`);
|
||||
writeFileSync(join(dir, 'page.html'), html, 'utf-8');
|
||||
await withCaptureTimeout(
|
||||
session.page.screenshot({ path: join(dir, 'page.png'), fullPage: true }),
|
||||
5_000,
|
||||
'page screenshot',
|
||||
).catch((err) => writeFileSync(join(dir, 'screenshot-error.txt'), err.message, 'utf-8'));
|
||||
}
|
||||
|
||||
log(`Failure artifacts written to ${dir}`);
|
||||
} catch (captureErr) {
|
||||
log(`Failure artifact capture failed: ${captureErr.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function writeCommandOutput(dir, fileName, cwd, args) {
|
||||
try {
|
||||
const output = execFileSync('git', args, { cwd, encoding: 'utf-8' });
|
||||
writeFileSync(join(dir, fileName), output, 'utf-8');
|
||||
} catch (err) {
|
||||
writeFileSync(join(dir, fileName), [err.stdout, err.stderr, err.message].filter(Boolean).join('\n'), 'utf-8');
|
||||
}
|
||||
}
|
||||
|
||||
function copyFileFromTmp(tmp, file, destRoot) {
|
||||
const rel = relative(tmp, file);
|
||||
if (!rel || rel.startsWith('..')) return;
|
||||
const dest = join(destRoot, rel);
|
||||
mkdirSync(dirname(dest), { recursive: true });
|
||||
cpSync(file, dest);
|
||||
}
|
||||
|
||||
function copyDirIfExists(from, to) {
|
||||
if (!existsSync(from)) return;
|
||||
mkdirSync(dirname(to), { recursive: true });
|
||||
cpSync(from, to, { recursive: true });
|
||||
}
|
||||
|
||||
function safeArtifactName(name) {
|
||||
return String(name || 'fixture').replace(/[^a-z0-9._-]+/gi, '-').replace(/^-+|-+$/g, '') || 'fixture';
|
||||
}
|
||||
|
||||
function withCaptureTimeout(promise, timeoutMs, label) {
|
||||
let timer;
|
||||
const timeout = new Promise((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
|
||||
});
|
||||
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
||||
}
|
||||
|
||||
async function createManualScenarioAgent(t, scenario = {}) {
|
||||
const requested = (process.env.IMPECCABLE_E2E_MANUAL_AGENT || process.env.IMPECCABLE_E2E_AGENT || 'auto')
|
||||
.trim()
|
||||
@@ -1084,6 +1260,7 @@ async function waitForAcceptedDom(page, selector, { allowVariantRoot = false, ti
|
||||
if (all.length < 1) return false;
|
||||
for (const el of all) {
|
||||
if (el.closest('[data-impeccable-variants]')) return false;
|
||||
if (el.closest('[data-impeccable-carbonize]')) return false;
|
||||
if (!allowVariantRoot && el.closest('[data-impeccable-variant]')) return false;
|
||||
}
|
||||
return true;
|
||||
@@ -1140,6 +1317,25 @@ async function assertVisibleText(page, selector, text, { timeout = 20_000 } = {}
|
||||
}
|
||||
}
|
||||
|
||||
async function readVisibleVariantForCycle(page, { timeout = 5_000 } = {}) {
|
||||
const start = Date.now();
|
||||
let last = null;
|
||||
while (Date.now() - start < timeout) {
|
||||
last = await getVisibleVariant(page);
|
||||
if (Number.isInteger(last) && last > 0) return last;
|
||||
await page.waitForTimeout(250);
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
async function evaluatePageWithTimeout(page, fn, arg, timeoutMs, label) {
|
||||
let timer;
|
||||
const timeout = new Promise((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
|
||||
});
|
||||
return Promise.race([page.evaluate(fn, arg), timeout]).finally(() => clearTimeout(timer));
|
||||
}
|
||||
|
||||
async function getServerManualEditStashCount(live, pageUrl = '/') {
|
||||
const res = await fetch(
|
||||
`http://localhost:${live.port}/manual-edit-stash?token=${encodeURIComponent(live.token)}&pageUrl=${encodeURIComponent(pageUrl)}`,
|
||||
@@ -1233,6 +1429,7 @@ async function waitForSourceClean(filePath, timeoutMs, { svelteComponentTarget:
|
||||
last.includes('data-impeccable-variants=') ||
|
||||
last.includes('impeccable-variants-start') ||
|
||||
last.includes('impeccable-carbonize-start') ||
|
||||
last.includes('data-impeccable-carbonize=') ||
|
||||
last.includes('data-impeccable-variant=');
|
||||
if (!dirty) return last;
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
|
||||
+76
-27
@@ -1747,6 +1747,10 @@ export async function runAgentLoop({
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
if (completionType === 'agent_done' && acceptResult.handled === true && acceptResult.carbonize === true) {
|
||||
await runLiveComplete({ tmp, scriptsDir, id: event.id });
|
||||
log(`completed carbonize session ${event.id}`);
|
||||
}
|
||||
} catch (err) {
|
||||
if (signal.aborted) return;
|
||||
log('accept failed: ' + err.message);
|
||||
@@ -1876,20 +1880,32 @@ export async function applySteerEdits(tmp, { file, edits }) {
|
||||
async function handleSteerDeterministic(context) {
|
||||
const { targetFileAbs, target } = context;
|
||||
let body = await fs.readFile(targetFileAbs, 'utf-8');
|
||||
const next = addSteerMarkerToSource(body, target);
|
||||
if (next === body) return;
|
||||
if (!next) {
|
||||
const { classes = 'hero-title', tag = 'h1' } = target;
|
||||
const classToken = classes.split(/\s+/)[0];
|
||||
throw new Error(`steer target <${tag}.${classToken}> not found in ${targetFileAbs}`);
|
||||
}
|
||||
body = next;
|
||||
await fs.writeFile(targetFileAbs, body, 'utf-8');
|
||||
}
|
||||
|
||||
export function addSteerMarkerToSource(body, target = { classes: 'hero-title', tag: 'h1' }) {
|
||||
const attr = `${STEER_MARKER_ATTR}="${STEER_MARKER_VALUE}"`;
|
||||
if (body.includes(attr)) return;
|
||||
if (body.includes(attr)) return body;
|
||||
|
||||
const { classes = 'hero-title', tag = 'h1' } = target;
|
||||
const classToken = classes.split(/\s+/)[0];
|
||||
const escapedTag = escapeRegExp(tag);
|
||||
const escapedClass = escapeRegExp(classToken);
|
||||
const classValue = `(?:["'][^"']*\\b${escapedClass}\\b[^"']*["']|\\{[^}]*\\b${escapedClass}\\b[^}]*\\})`;
|
||||
const openTagRe = new RegExp(
|
||||
`(<${tag}\\b(?=[^>]*\\b(?:className|class)=["'][^"']*\\b${classToken}\\b)[^>]*)(>)`,
|
||||
`(<${escapedTag}\\b(?=[^>]*\\b(?:className|class)\\s*=\\s*${classValue})[^>]*)(>)`,
|
||||
'i',
|
||||
);
|
||||
if (!openTagRe.test(body)) {
|
||||
throw new Error(`steer target <${tag}.${classToken}> not found in ${targetFileAbs}`);
|
||||
}
|
||||
body = body.replace(openTagRe, `$1 ${attr}$2`);
|
||||
await fs.writeFile(targetFileAbs, body, 'utf-8');
|
||||
if (!openTagRe.test(body)) return null;
|
||||
return body.replace(openTagRe, `$1 ${attr}$2`);
|
||||
}
|
||||
|
||||
function findSteerTargetFileSync(tmp, target) {
|
||||
@@ -1976,26 +1992,12 @@ async function runCarbonizeCleanup({ tmp, file, sessionId /* , variant */ }) {
|
||||
}
|
||||
|
||||
// 2. Unwrap the temporary `<div data-impeccable-variant="N" ...>` placed
|
||||
// around the accepted content. live-accept emits this wrapper with
|
||||
// `style="display: contents"` so it doesn't affect layout. We strip the
|
||||
// wrapper open/close lines and keep what's between.
|
||||
// Match the opening div (any single line) followed by inner content
|
||||
// followed by `</div>`, where the open carries data-impeccable-variant
|
||||
// and is NOT inside a data-impeccable-variants wrapper (the variants
|
||||
// wrapper has the trailing `s`).
|
||||
body = body.replace(
|
||||
/^([ \t]*)<div\b[^>]*\bdata-impeccable-variant="[^"]+"[^>]*>\n([\s\S]*?)\n[ \t]*<\/div>\n/m,
|
||||
(match, indent, inner) => {
|
||||
// Re-indent inner content to the wrapper's indent level.
|
||||
const innerLines = inner.split('\n');
|
||||
const innerIndent = (innerLines[0].match(/^\s*/) || [''])[0];
|
||||
const dedented = innerLines.map((l) => {
|
||||
if (l.startsWith(innerIndent)) return indent + l.slice(innerIndent.length);
|
||||
return l;
|
||||
}).join('\n');
|
||||
return expandAcceptedVariantMarkup(dedented, indent) + '\n';
|
||||
},
|
||||
);
|
||||
// around the accepted content. For JSX targets, live-accept also adds an
|
||||
// outer `<div data-impeccable-carbonize>` so the carbonize block and accepted
|
||||
// node occupy one child slot; strip that shell after the accepted node is
|
||||
// clean.
|
||||
body = unwrapDivAttributeWrapper(body, 'data-impeccable-variant', { expandSingleLineContainer: true });
|
||||
body = unwrapDivAttributeWrapper(body, 'data-impeccable-carbonize');
|
||||
|
||||
// 3. Strip any `data-impeccable-hoist-id` attributes the normalize step
|
||||
// may have injected when the model emitted inline styles. The hoisted
|
||||
@@ -2007,6 +2009,49 @@ async function runCarbonizeCleanup({ tmp, file, sessionId /* , variant */ }) {
|
||||
await fs.writeFile(filePath, body, 'utf-8');
|
||||
}
|
||||
|
||||
function unwrapDivAttributeWrapper(body, attrName, { expandSingleLineContainer = false } = {}) {
|
||||
const lines = String(body).split('\n');
|
||||
const attrRe = new RegExp(`\\b${escapeRegExp(attrName)}=`);
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (!/<div\b/.test(lines[i]) || !attrRe.test(lines[i])) continue;
|
||||
|
||||
const indent = (lines[i].match(/^(\s*)/) || [''])[1];
|
||||
let depth = countDivDepthDelta(lines[i]);
|
||||
for (let j = i + 1; j < lines.length; j++) {
|
||||
depth += countDivDepthDelta(lines[j]);
|
||||
if (depth !== 0) continue;
|
||||
|
||||
let replacement = reindentWrapperBody(lines.slice(i + 1, j), indent).join('\n');
|
||||
if (expandSingleLineContainer) {
|
||||
replacement = expandAcceptedVariantMarkup(replacement, indent);
|
||||
}
|
||||
lines.splice(i, j - i + 1, ...replacement.split('\n'));
|
||||
return lines.join('\n');
|
||||
}
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
function countDivDepthDelta(line) {
|
||||
return countMatches(line, /<div\b/g) - countMatches(line, /<\/div>/g);
|
||||
}
|
||||
|
||||
function countMatches(value, re) {
|
||||
return [...String(value || '').matchAll(re)].length;
|
||||
}
|
||||
|
||||
function reindentWrapperBody(lines, indent) {
|
||||
const firstContentLine = lines.find((line) => line.trim() !== '');
|
||||
const innerIndent = (firstContentLine?.match(/^(\s*)/) || [''])[1] || '';
|
||||
return lines.map((line) => {
|
||||
if (line.trim() === '') return '';
|
||||
if (innerIndent && line.startsWith(innerIndent)) return indent + line.slice(innerIndent.length);
|
||||
return indent + line.trimStart();
|
||||
});
|
||||
}
|
||||
|
||||
function expandAcceptedVariantMarkup(source, indent) {
|
||||
const lines = source.split('\n');
|
||||
if (lines.length !== 1) return source;
|
||||
@@ -2083,3 +2128,7 @@ async function runAccept({ tmp, scriptsDir, id, variant, discard, paramValues, p
|
||||
const last = stdout.trim().split('\n').filter(Boolean).pop();
|
||||
return JSON.parse(last);
|
||||
}
|
||||
|
||||
async function runLiveComplete({ tmp, scriptsDir, id }) {
|
||||
await execFileP(process.execPath, [path.join(scriptsDir, 'live-complete.mjs'), '--id', id], { cwd: tmp });
|
||||
}
|
||||
|
||||
@@ -51,9 +51,27 @@ export function stageFixture(name, fixture) {
|
||||
return tmp;
|
||||
}
|
||||
|
||||
export function runInstall(tmp, command) {
|
||||
export function runInstall(tmp, command, { timeoutMs = readTimeoutEnv('IMPECCABLE_E2E_INSTALL_TIMEOUT_MS', 180_000) } = {}) {
|
||||
const [cmd, ...args] = command;
|
||||
execFileSync(cmd, args, { cwd: tmp, stdio: 'inherit' });
|
||||
const installArgs = addNpmInstallDefaults(cmd, args);
|
||||
try {
|
||||
execFileSync(cmd, installArgs, { cwd: tmp, stdio: 'inherit', timeout: timeoutMs });
|
||||
} catch (err) {
|
||||
if (err.signal === 'SIGTERM' || err.signal === 'SIGKILL' || err.killed) {
|
||||
err.message = `fixture dependency install timed out after ${timeoutMs}ms: ${cmd} ${installArgs.join(' ')}`;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function addNpmInstallDefaults(cmd, args) {
|
||||
if (cmd !== 'npm') return args;
|
||||
if (!['install', 'ci'].includes(args[0])) return args;
|
||||
const out = [...args];
|
||||
for (const flag of ['--prefer-offline', '--no-progress']) {
|
||||
if (!out.some((arg) => arg === flag || arg.startsWith(flag + '='))) out.push(flag);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -121,11 +139,15 @@ export function startDevServer(tmp, runtime) {
|
||||
child.stderr.on('data', capture);
|
||||
|
||||
const ready = new Promise((resolve, reject) => {
|
||||
const readyTimeoutMs = readTimeoutEnv(
|
||||
'IMPECCABLE_E2E_DEV_READY_TIMEOUT_MS',
|
||||
runtime.readyTimeoutMs ?? 120_000,
|
||||
);
|
||||
const timeout = setTimeout(() => {
|
||||
reject(new Error(
|
||||
`dev server ready timeout (${runtime.readyTimeoutMs}ms). Tail:\n${bufLog.join('')}`,
|
||||
`dev server ready timeout (${readyTimeoutMs}ms). Tail:\n${bufLog.join('')}`,
|
||||
));
|
||||
}, runtime.readyTimeoutMs ?? 120_000);
|
||||
}, readyTimeoutMs);
|
||||
|
||||
const checkMatch = (buf) => {
|
||||
const m = buf.toString().match(readyRe);
|
||||
@@ -145,13 +167,27 @@ export function startDevServer(tmp, runtime) {
|
||||
return { child, ready, log: () => bufLog.join('') };
|
||||
}
|
||||
|
||||
function readTimeoutEnv(name, fallback) {
|
||||
const raw = process.env[name];
|
||||
if (raw == null || raw === '') return fallback;
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
export async function stopDevServer(child) {
|
||||
if (!child || child.killed) return;
|
||||
const exited = new Promise((resolve) => child.once('exit', resolve));
|
||||
if (!child || child.exitCode != null || child.signalCode != null) return;
|
||||
let didExit = false;
|
||||
const exited = new Promise((resolve) => child.once('exit', () => {
|
||||
didExit = true;
|
||||
resolve();
|
||||
}));
|
||||
child.kill('SIGTERM');
|
||||
const timeoutPromise = new Promise((resolve) => setTimeout(resolve, 5_000));
|
||||
await Promise.race([exited, timeoutPromise]);
|
||||
if (!child.killed) child.kill('SIGKILL');
|
||||
if (!didExit && child.exitCode == null && child.signalCode == null) {
|
||||
child.kill('SIGKILL');
|
||||
await Promise.race([exited, new Promise((resolve) => setTimeout(resolve, 1_000))]);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -196,20 +232,27 @@ export async function bootFixtureSession({ name, fixture, browser, agent, wrapTa
|
||||
};
|
||||
|
||||
try {
|
||||
const startedAt = Date.now();
|
||||
log(`installing deps`);
|
||||
runInstall(tmp, runtime.install);
|
||||
log(`deps installed in ${formatDuration(Date.now() - startedAt)}`);
|
||||
|
||||
const liveStartedAt = Date.now();
|
||||
log(`starting live-server`);
|
||||
live = startLiveServer(tmp);
|
||||
log(`live-server ready in ${formatDuration(Date.now() - liveStartedAt)}`);
|
||||
|
||||
const injectStartedAt = Date.now();
|
||||
log(`live-inject --port ${live.port}`);
|
||||
const injectResult = runInject(tmp, live.port);
|
||||
if (!injectResult.ok) throw new Error('live-inject failed: ' + JSON.stringify(injectResult));
|
||||
log(`live-inject complete in ${formatDuration(Date.now() - injectStartedAt)}`);
|
||||
|
||||
const devStartedAt = Date.now();
|
||||
log(`spawning dev server: ${runtime.devCommand.join(' ')}`);
|
||||
dev = startDevServer(tmp, runtime);
|
||||
const { port: devPort } = await dev.ready;
|
||||
log(`dev server ready on ${devPort}`);
|
||||
log(`dev server ready on ${devPort} in ${formatDuration(Date.now() - devStartedAt)}`);
|
||||
|
||||
// Agent loop runs concurrently — abort on teardown.
|
||||
agentAbort = new AbortController();
|
||||
@@ -239,10 +282,12 @@ export async function bootFixtureSession({ name, fixture, browser, agent, wrapTa
|
||||
if (msg.type() === 'error') consoleErrors.push(`console.error: ${msg.text()}`);
|
||||
});
|
||||
|
||||
const pageStartedAt = Date.now();
|
||||
await page.goto(`${scheme}://127.0.0.1:${devPort}`, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 30_000,
|
||||
});
|
||||
log(`page loaded in ${formatDuration(Date.now() - pageStartedAt)}`);
|
||||
|
||||
return {
|
||||
tmp,
|
||||
@@ -260,3 +305,8 @@ export async function bootFixtureSession({ name, fixture, browser, agent, wrapTa
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function formatDuration(ms) {
|
||||
if (ms < 1_000) return `${ms}ms`;
|
||||
return `${(ms / 1_000).toFixed(1)}s`;
|
||||
}
|
||||
|
||||
+51
-23
@@ -139,9 +139,21 @@ function installLiveQueryHelpersInPage() {
|
||||
};
|
||||
}
|
||||
|
||||
export async function installLiveQueryHelpers(page) {
|
||||
export async function installLiveQueryHelpers(page, { timeout = 5_000 } = {}) {
|
||||
await page.addInitScript(installLiveQueryHelpersInPage).catch(() => {});
|
||||
await page.evaluate(installLiveQueryHelpersInPage);
|
||||
await withTimeout(
|
||||
page.evaluate(installLiveQueryHelpersInPage),
|
||||
timeout,
|
||||
'install live query helpers',
|
||||
);
|
||||
}
|
||||
|
||||
function withTimeout(promise, timeout, label) {
|
||||
let timer;
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeout}ms`)), timeout);
|
||||
});
|
||||
return Promise.race([promise, timeoutPromise]).finally(() => clearTimeout(timer));
|
||||
}
|
||||
|
||||
async function clickLiveControl(page, selector) {
|
||||
@@ -605,14 +617,14 @@ export async function clickPrev(page) {
|
||||
}
|
||||
|
||||
async function clickBarButton(page, label) {
|
||||
await installLiveQueryHelpers(page);
|
||||
const button = page.locator(`${BAR_ID} button`, { hasText: label });
|
||||
const textMatch = label instanceof RegExp
|
||||
? { kind: 'regex', source: label.source, flags: label.flags }
|
||||
: { kind: 'text', value: String(label) };
|
||||
let lastErr;
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try {
|
||||
await installLiveQueryHelpers(page);
|
||||
const button = page.locator(`${BAR_ID} button`, { hasText: label });
|
||||
await button.click({ timeout: 5_000 });
|
||||
return;
|
||||
} catch (err) {
|
||||
@@ -637,11 +649,19 @@ async function clickBarButton(page, label) {
|
||||
}
|
||||
|
||||
async function dispatchBarButton(page, label) {
|
||||
await installLiveQueryHelpers(page);
|
||||
const textMatch = label instanceof RegExp
|
||||
? { kind: 'regex', source: label.source, flags: label.flags }
|
||||
: { kind: 'text', value: String(label) };
|
||||
return page.evaluate(findAndClickBarButton, { barSel: BAR_ID, textMatch });
|
||||
try {
|
||||
await installLiveQueryHelpers(page);
|
||||
const textMatch = label instanceof RegExp
|
||||
? { kind: 'regex', source: label.source, flags: label.flags }
|
||||
: { kind: 'text', value: String(label) };
|
||||
return await withTimeout(
|
||||
page.evaluate(findAndClickBarButton, { barSel: BAR_ID, textMatch }),
|
||||
5_000,
|
||||
'dispatch bar button',
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function findAndClickBarButton({ barSel, textMatch }) {
|
||||
@@ -662,20 +682,28 @@ function findAndClickBarButton({ barSel, textMatch }) {
|
||||
* Read the currently visible variant index (the "i" in "i/N").
|
||||
*/
|
||||
export async function getVisibleVariant(page) {
|
||||
await installLiveQueryHelpers(page);
|
||||
return page.evaluate((barSel) => {
|
||||
const wrapper = window.__impeccableLiveQuery('[data-impeccable-variants]');
|
||||
if (wrapper) {
|
||||
const variants = [...wrapper.querySelectorAll('[data-impeccable-variant]:not([data-impeccable-variant="original"])')];
|
||||
const visible = variants.find((variant) => variant.style.display !== 'none');
|
||||
const idx = visible ? parseInt(visible.dataset.impeccableVariant || '0', 10) : 0;
|
||||
if (idx > 0) return idx;
|
||||
}
|
||||
const bar = window.__impeccableLiveQuery(barSel);
|
||||
if (!bar) return null;
|
||||
const m = (bar.textContent || '').match(/(\d+)\s*\/\s*(\d+)/);
|
||||
return m ? parseInt(m[1], 10) : null;
|
||||
}, BAR_ID);
|
||||
try {
|
||||
await installLiveQueryHelpers(page);
|
||||
return await withTimeout(
|
||||
page.evaluate((barSel) => {
|
||||
const wrapper = window.__impeccableLiveQuery('[data-impeccable-variants]');
|
||||
if (wrapper) {
|
||||
const variants = [...wrapper.querySelectorAll('[data-impeccable-variant]:not([data-impeccable-variant="original"])')];
|
||||
const visible = variants.find((variant) => variant.style.display !== 'none');
|
||||
const idx = visible ? parseInt(visible.dataset.impeccableVariant || '0', 10) : 0;
|
||||
if (idx > 0) return idx;
|
||||
}
|
||||
const bar = window.__impeccableLiveQuery(barSel);
|
||||
if (!bar) return null;
|
||||
const m = (bar.textContent || '').match(/(\d+)\s*\/\s*(\d+)/);
|
||||
return m ? parseInt(m[1], 10) : null;
|
||||
}, BAR_ID),
|
||||
5_000,
|
||||
'read visible variant',
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+102
-15
@@ -3,10 +3,11 @@
|
||||
*
|
||||
* Creates real temp directories, runs the CLI, and verifies results.
|
||||
*
|
||||
* Pure blocks (already-installed detection, unprefix migration) run in the
|
||||
* default `bun run test`. Network blocks that download the universal bundle use
|
||||
* `describeNet` and run only under `bun run test:cli-e2e` (IMPECCABLE_CLI_E2E=1),
|
||||
* skipping gracefully when impeccable.style is unreachable.
|
||||
* Deterministic install/update coverage uses a local universal bundle override
|
||||
* and runs in the default suite. Remote smoke blocks that download the
|
||||
* production universal bundle use `describeRemote` and run only under
|
||||
* `bun run test:cli-remote-e2e` (IMPECCABLE_CLI_REMOTE_E2E=1), skipping
|
||||
* gracefully when impeccable.style is unreachable.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { execSync } from 'child_process';
|
||||
@@ -57,6 +58,24 @@ function createFakeLinkSource(root, providers = ['.claude']) {
|
||||
}
|
||||
}
|
||||
|
||||
function createFakeUniversalBundle(root, providers = ['.claude', '.agents', '.cursor']) {
|
||||
const bundleRoot = join(root, 'universal-bundle');
|
||||
for (const provider of providers) {
|
||||
const skillDir = join(bundleRoot, provider, 'skills', 'impeccable');
|
||||
mkdirSync(join(skillDir, 'scripts'), { recursive: true });
|
||||
writeFileSync(join(skillDir, 'SKILL.md'), [
|
||||
'---',
|
||||
'name: impeccable',
|
||||
'version: 9.9.9-local',
|
||||
'---',
|
||||
'',
|
||||
`Local deterministic bundle for ${provider}.`,
|
||||
].join('\n'));
|
||||
writeFileSync(join(skillDir, 'scripts', 'context.mjs'), 'console.log("local bundle context");\n');
|
||||
}
|
||||
return bundleRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulate an install from the era when the CLI offered a command prefix: the
|
||||
* skill lives at `<prefix>impeccable`. Optionally drop in a third-party skill
|
||||
@@ -71,19 +90,19 @@ function createPrefixedInstall(root, { prefix = 'i-', providers = ['.claude'], f
|
||||
|
||||
// ─── Already-installed detection ─────────────────────────────────────────────
|
||||
|
||||
// Network e2e blocks (real bundle downloads from impeccable.style) run only
|
||||
// under `bun run test:cli-e2e` (IMPECCABLE_CLI_E2E=1). The default `bun run test`
|
||||
// skips them so it stays fast and works offline; when opted in they still skip
|
||||
// gracefully if the bundle endpoint is unreachable.
|
||||
const WANT_CLI_E2E = process.env.IMPECCABLE_CLI_E2E === '1';
|
||||
// Remote e2e blocks (real bundle downloads from impeccable.style) run only
|
||||
// under `bun run test:cli-remote-e2e` (IMPECCABLE_CLI_REMOTE_E2E=1). The default
|
||||
// suite skips them so it stays offline and stable; when opted in they still
|
||||
// skip gracefully if the bundle endpoint is unreachable.
|
||||
const WANT_CLI_REMOTE_E2E = process.env.IMPECCABLE_CLI_REMOTE_E2E === '1';
|
||||
let bundleReachable = false;
|
||||
if (WANT_CLI_E2E) {
|
||||
if (WANT_CLI_REMOTE_E2E) {
|
||||
try {
|
||||
execSync('curl -sfIL --max-time 10 https://impeccable.style/api/download/bundle/universal -o /dev/null', { stdio: 'pipe' });
|
||||
bundleReachable = true;
|
||||
} catch {}
|
||||
}
|
||||
const describeNet = (WANT_CLI_E2E && bundleReachable) ? describe : describe.skip;
|
||||
const describeRemote = (WANT_CLI_REMOTE_E2E && bundleReachable) ? describe : describe.skip;
|
||||
|
||||
describe('skills install: already-installed detection', () => {
|
||||
test('detects impeccable sentinel and bails', () => {
|
||||
@@ -291,9 +310,77 @@ describe('skills: unprefix migration', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Update fallback (direct download) ───────────────────────────────────────
|
||||
// ─── Install/update from local universal bundle ──────────────────────────────
|
||||
|
||||
describeNet('skills update: refreshes from the universal bundle', () => {
|
||||
describe('skills install/update: local universal bundle e2e', () => {
|
||||
test('installs provider-specific skills into a fresh project', () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'imp-test-local-install-'));
|
||||
execSync('git init', { cwd: tmp });
|
||||
const bundleRoot = createFakeUniversalBundle(tmp);
|
||||
|
||||
const output = run('skills install -y --providers=claude,codex,cursor', {
|
||||
cwd: tmp,
|
||||
env: { ...process.env, IMPECCABLE_BUNDLE_PATH: bundleRoot },
|
||||
});
|
||||
expect(output).toContain('Done!');
|
||||
|
||||
for (const provider of ['.claude', '.agents', '.cursor']) {
|
||||
const skillDir = join(tmp, provider, 'skills', 'impeccable');
|
||||
expect(existsSync(join(skillDir, 'SKILL.md'))).toBe(true);
|
||||
expect(readFileSync(join(skillDir, 'SKILL.md'), 'utf8')).toContain(`Local deterministic bundle for ${provider}.`);
|
||||
expect(existsSync(join(skillDir, 'scripts', 'context.mjs'))).toBe(true);
|
||||
}
|
||||
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}, 15000);
|
||||
|
||||
test('updates stale copied skills from the local bundle', () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'imp-test-local-update-'));
|
||||
execSync('git init', { cwd: tmp });
|
||||
const bundleRoot = createFakeUniversalBundle(tmp, ['.claude']);
|
||||
|
||||
const skillDir = join(tmp, '.claude', 'skills', 'impeccable');
|
||||
mkdirSync(skillDir, { recursive: true });
|
||||
writeFileSync(join(skillDir, 'SKILL.md'), '---\nname: impeccable\nstale: true\n---\nOld content.\n');
|
||||
|
||||
const output = run('skills update -y', {
|
||||
cwd: tmp,
|
||||
env: { ...process.env, IMPECCABLE_BUNDLE_PATH: bundleRoot },
|
||||
});
|
||||
expect(output).toContain('Updated');
|
||||
|
||||
const content = readFileSync(join(skillDir, 'SKILL.md'), 'utf8');
|
||||
expect(content).not.toContain('stale: true');
|
||||
expect(content).toContain('version: 9.9.9-local');
|
||||
expect(existsSync(join(skillDir, 'scripts', 'context.mjs'))).toBe(true);
|
||||
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}, 15000);
|
||||
|
||||
test('--force reinstall over an old prefixed install lands on canonical impeccable', () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'imp-test-local-force-'));
|
||||
execSync('git init', { cwd: tmp });
|
||||
const bundleRoot = createFakeUniversalBundle(tmp, ['.claude']);
|
||||
const prefixed = join(tmp, '.claude', 'skills', 'i-impeccable');
|
||||
mkdirSync(prefixed, { recursive: true });
|
||||
writeFileSync(join(prefixed, 'SKILL.md'), '---\nname: i-impeccable\n---\n');
|
||||
|
||||
run('skills install -y --force --providers=claude', {
|
||||
cwd: tmp,
|
||||
env: { ...process.env, IMPECCABLE_BUNDLE_PATH: bundleRoot },
|
||||
});
|
||||
|
||||
const skills = readdirSync(join(tmp, '.claude', 'skills'));
|
||||
expect(skills).toContain('impeccable');
|
||||
expect(skills).not.toContain('i-impeccable');
|
||||
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}, 15000);
|
||||
});
|
||||
|
||||
// ─── Update fallback (remote direct download smoke) ──────────────────────────
|
||||
|
||||
describeRemote('skills update: refreshes from the production universal bundle', () => {
|
||||
let tmp;
|
||||
|
||||
beforeAll(() => {
|
||||
@@ -328,9 +415,9 @@ describeNet('skills update: refreshes from the universal bundle', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Full install e2e (downloads the universal bundle) ───────────────────────
|
||||
// ─── Full install remote smoke (downloads the production universal bundle) ───
|
||||
|
||||
describeNet('skills install: full e2e (universal bundle download)', () => {
|
||||
describeRemote('skills install: production universal bundle download', () => {
|
||||
let tmp;
|
||||
|
||||
beforeAll(() => {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
DEFAULT_SUITES,
|
||||
OPT_IN_SUITES,
|
||||
SUITES,
|
||||
expandSuites,
|
||||
findTestFiles,
|
||||
suiteFiles,
|
||||
} from '../scripts/test-suites.mjs';
|
||||
|
||||
describe('test suite registry', () => {
|
||||
it('assigns every test file to a default or opt-in suite', () => {
|
||||
const allDiscovered = findTestFiles();
|
||||
const allRegistered = new Set(suiteFiles([...DEFAULT_SUITES, ...OPT_IN_SUITES]));
|
||||
const missing = allDiscovered.filter((file) => !allRegistered.has(file));
|
||||
|
||||
assert.deepEqual(
|
||||
missing,
|
||||
[],
|
||||
'new test files must be added to scripts/test-suites.mjs, either in a default suite or an opt-in suite',
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps default local suites free of duplicate test files', () => {
|
||||
const files = suiteFiles(DEFAULT_SUITES);
|
||||
const duplicates = files.filter((file, index) => files.indexOf(file) !== index);
|
||||
|
||||
assert.deepEqual(duplicates, []);
|
||||
});
|
||||
|
||||
it('keeps opt-in suites out of the default alias', () => {
|
||||
const expanded = expandSuites(['default']);
|
||||
for (const suite of expanded) {
|
||||
assert.equal(SUITES[suite].optIn, undefined, `${suite} should not be opt-in`);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user