Refresh the Impeccable product experience

Rework the landing page proof, steering demo, feature grid, slop catalog, detector coverage, theming, Live workflow, and responsive behavior.\n\nAI-assisted implementation by OpenAI Codex.
This commit is contained in:
Paul Bakaus
2026-07-15 23:29:47 -07:00
parent 8682c85c57
commit bbed6eef08
553 changed files with 8903 additions and 97987 deletions
+6 -6
View File
@@ -41,28 +41,28 @@ describe('gatherSignals', () => {
const s = await gatherSignals(scratch);
assert.equal(s.setup.hasProduct, false);
assert.equal(s.setup.hasDesign, false);
assert.equal(s.setup.register, null);
assert.equal(Object.hasOwn(s.setup, 'register'), false);
assert.equal(s.setup.hasCode, false);
assert.equal(s.critique.latest, null);
});
it('detects PRODUCT.md, register, and code presence', async () => {
write('PRODUCT.md', '# Product\n\n## Register\n\nbrand\n');
it('detects PRODUCT.md, platform, and code presence', async () => {
write('PRODUCT.md', '# Product\n\n## Platform\n\nweb\n');
write('package.json', '{"name":"x"}');
const s = await gatherSignals(scratch);
assert.equal(s.setup.hasProduct, true);
assert.equal(s.setup.register, 'brand');
assert.equal(s.setup.platform, 'web');
assert.equal(s.setup.hasCode, true);
});
it('flags missing DESIGN.md when code exists', async () => {
write('PRODUCT.md', '# Product\n\n## Register\n\nproduct\n');
write('PRODUCT.md', '# Product\n\n## Platform\n\nweb\n');
write('src/App.tsx', 'export default 1;');
const s = await gatherSignals(scratch);
assert.equal(s.setup.hasProduct, true);
assert.equal(s.setup.hasDesign, false);
assert.equal(s.setup.hasCode, true);
assert.equal(s.setup.register, 'product');
assert.equal(s.setup.platform, 'web');
});
it('reads the newest critique snapshot score', async () => {
+61 -34
View File
@@ -21,7 +21,7 @@ import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import { loadContext, resolveContextDir, resolveProjectRoot, extractRegister, extractPlatform } from '../skill/scripts/context.mjs';
import { loadContext, resolveContextDir, resolveProjectRoot, extractPlatform, hasVisualImplementation } from '../skill/scripts/context.mjs';
import { fileURLToPath } from 'node:url';
const SCRIPT_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'skill', 'scripts', 'context.mjs');
@@ -447,7 +447,7 @@ describe('loadContext (monorepo project context)', () => {
it('supports --target in the CLI', async () => {
writeMonorepo();
write('apps/dashboard/PRODUCT.md', '# Dashboard product\n\n## Register\n\nproduct\n');
write('apps/dashboard/PRODUCT.md', '# Dashboard product\n\n## Platform\n\nweb\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH, '--target', 'apps/dashboard/src/App.jsx'], {
cwd: scratch,
@@ -461,7 +461,7 @@ describe('loadContext (monorepo project context)', () => {
assert.match(res.stdout, /"targetPath": "apps\/dashboard\/src\/App\.jsx"/);
assert.match(res.stdout, /"productPath": "apps\/dashboard\/PRODUCT\.md"/);
assert.match(res.stdout, /"designPath": "DESIGN\.md"/);
assert.match(res.stdout, /REGISTER: `product`/);
assert.doesNotMatch(res.stdout, /REGISTER:/);
});
it('asks for an app when the CLI runs from a monorepo root without selection', () => {
@@ -765,9 +765,6 @@ describe('extractPlatform', () => {
// `## Platform notes` must not be mistaken for the `## Platform` field.
const product = '## Platform notes\n\nsome prose here\n\n## Platform\n\nios\n';
assert.equal(extractPlatform(product), 'ios');
// Same precision for the register heading.
const reg = '## Register guidelines\n\nblah\n\n## Register\n\nbrand\n';
assert.equal(extractRegister(reg), 'brand');
});
it('reads the first non-empty line after the heading', () => {
@@ -779,13 +776,6 @@ describe('extractPlatform', () => {
// (which would surface a nonsense "value `## Product Purpose` is not
// recognized" warning from the CLI).
assert.equal(extractPlatform('## Platform\n\n## Product Purpose\n\nAn app.\n'), null);
assert.equal(extractRegister('## Register\n\n## Users\n\nAnglers.\n'), null);
});
it('is independent of the register field', () => {
const product = '# P\n\n## Register\n\nproduct\n\n## Platform\n\nandroid\n';
assert.equal(extractRegister(product), 'product');
assert.equal(extractPlatform(product), 'android');
});
});
@@ -796,6 +786,8 @@ describe('context.mjs CLI', () => {
assert.equal(res.status, 0);
assert.match(res.stdout, /^NO_PRODUCT_MD:/);
assert.match(res.stdout, /reference\/init\.md/);
assert.match(res.stdout, /structured simulated-user interview/);
assert.match(res.stdout, /IDENTITY_INIT_REQUIRED:/);
});
it('prints a PRODUCT.md markdown block when only PRODUCT.md exists', async () => {
@@ -807,9 +799,53 @@ describe('context.mjs CLI', () => {
assert.match(res.stdout, /# Acme/);
assert.equal(res.stdout.includes('# DESIGN.md'), false);
// Directives are appended after `---`; with no DESIGN.md the
// new-work gate directive fires.
// init identity gate directive fires before the task concept flow.
assert.match(res.stdout, /\n---\n\n/);
assert.match(res.stdout, /NEW_WORK: PRODUCT\.md exists but no DESIGN\.md/);
assert.match(res.stdout, /IDENTITY_INIT_REQUIRED: PRODUCT\.md exists but no DESIGN\.md/);
});
it('treats tokenized code as incumbent design authority when DESIGN.md is missing', () => {
write('PRODUCT.md', '# Acme\n');
write('src/app.css', ':root { --color-brand: red; --color-surface: white; --color-text: black; }\nbody { font-family: system-ui; background: var(--color-surface); color: var(--color-text); }\n');
assert.equal(hasVisualImplementation(scratch), true);
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8', env: { ...process.env, IMPECCABLE_NO_UPDATE_CHECK: '1' } });
assert.equal(res.status, 0);
assert.match(res.stdout, /BUILD_DESIGN_DOCUMENT_REQUIRED:/);
assert.match(res.stdout, /Before `craft` or `shape`, load reference\/init\.md Step 5/);
assert.doesNotMatch(res.stdout, /IDENTITY_INIT_REQUIRED:/);
assert.match(res.stdout, /"hasVisualImplementation": true/);
});
it('does not mistake the empty Astro eval scaffold for an incumbent identity', () => {
write('src/styles/global.css', '@import "tailwindcss";\n');
write('src/pages/index.astro', `---\nimport "../styles/global.css";\n---\n<!doctype html><html><head><title>Eval Workspace</title></head><body></body></html>\n`);
assert.equal(hasVisualImplementation(scratch), false);
});
it('recognizes one substantive authored Astro surface', () => {
write('src/pages/index.astro', `<main class="shell"><h1>Field notes for the night shift</h1><p>Specific authored content.</p></main>\n<style>\n:root { --ink: #111; --paper: #fff; --accent: #c40; }\n.shell { color: var(--ink); background-color: var(--paper); border-color: var(--accent); font-family: serif; padding: 4rem; min-height: 100vh; }\n</style>\n`);
assert.equal(hasVisualImplementation(scratch), true);
});
it('does not let irrelevant or vendored files exhaust or satisfy the visual scan', () => {
for (let i = 0; i < 300; i++) write(`src/data/item-${String(i).padStart(3, '0')}.txt`, 'not visual\n');
write('public/vendor/framework.min.css', ':root { --a: 1; --b: 2; --c: 3; } body { color: red; background: blue; border-color: green; font-family: sans-serif; }\n');
write('styles/z-theme.css', ':root { --brand: #124; --surface: #fff; --text: #111; } main { color: var(--text); background-color: var(--surface); border-color: var(--brand); }\n');
assert.equal(hasVisualImplementation(scratch), true);
});
it('routes craft through init but keeps narrow refinements non-blocking when visual code exists without PRODUCT.md', () => {
write('styles/theme.css', ':root { --brand: #124; --surface: #fff; --text: #111; }\nmain { color: var(--text); background-color: var(--surface); border-color: var(--brand); }\n');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8', env: { ...process.env, IMPECCABLE_NO_UPDATE_CHECK: '1' } });
assert.equal(res.status, 0);
assert.match(res.stdout, /^NO_PRODUCT_MD:/);
assert.match(res.stdout, /EXISTING_VISUAL_SYSTEM:/);
assert.match(res.stdout, /BUILD_INIT_REQUIRED:/);
assert.match(res.stdout, /SCOPED_EXISTING_ALLOWED:/);
assert.match(res.stdout, /proceed without blocking/);
assert.match(res.stdout, /For `init`, `teach`, `craft`, or `shape`/);
assert.match(res.stdout, /For a redesign\/rebrand.*old look only as evidence and anti-reference/s);
assert.doesNotMatch(res.stdout, /IDENTITY_INIT_REQUIRED:/);
});
it('concatenates PRODUCT.md and DESIGN.md with a --- separator', async () => {
@@ -821,7 +857,7 @@ describe('context.mjs CLI', () => {
assert.match(res.stdout, /^# PRODUCT\.md/);
assert.match(res.stdout, /\n---\n/);
assert.match(res.stdout, /# DESIGN\.md\n\n# Acme design/);
assert.equal(res.stdout.includes('NEW_WORK:'), false);
assert.equal(res.stdout.includes('IDENTITY_INIT_REQUIRED:'), false);
});
it('reads from a fallback dir when cwd is clean', async () => {
@@ -833,26 +869,17 @@ describe('context.mjs CLI', () => {
assert.match(res.stdout, /# fallback product/);
});
it('names the register-specific reference when PRODUCT.md declares one', async () => {
it('ignores a legacy Register field because visitor mode is task-scoped', async () => {
write('PRODUCT.md', '# Acme\n\n## Register\n\nbrand\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8', env: { ...process.env, IMPECCABLE_NO_UPDATE_CHECK: '1' } });
assert.equal(res.status, 0);
assert.match(res.stdout, /REGISTER: `brand`/);
assert.match(res.stdout, /Derive the visitor's mode per SKILL\.md/);
});
it('falls back to a generic register directive when no register field is present', async () => {
write('PRODUCT.md', '# Acme\n\n(no register field)\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8', env: { ...process.env, IMPECCABLE_NO_UPDATE_CHECK: '1' } });
assert.equal(res.status, 0);
assert.match(res.stdout, /NEW_WORK: PRODUCT\.md exists but no DESIGN\.md/);
assert.equal(res.stdout.includes('REGISTER:'), false);
assert.doesNotMatch(res.stdout, /REGISTER:/);
assert.match(res.stdout, /IDENTITY_INIT_REQUIRED: PRODUCT\.md exists but no DESIGN\.md/);
});
it('appends a native platform directive for an ios project', async () => {
write('PRODUCT.md', '# Acme\n\n## Register\n\nproduct\n\n## Platform\n\nios\n');
write('PRODUCT.md', '# Acme\n\n## Platform\n\nios\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8', env: { ...process.env, IMPECCABLE_NO_UPDATE_CHECK: '1' } });
assert.equal(res.status, 0);
@@ -861,7 +888,7 @@ describe('context.mjs CLI', () => {
});
it('appends both native directives for an adaptive project', async () => {
write('PRODUCT.md', '# Acme\n\n## Register\n\nproduct\n\n## Platform\n\nadaptive\n');
write('PRODUCT.md', '# Acme\n\n## Platform\n\nadaptive\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8', env: { ...process.env, IMPECCABLE_NO_UPDATE_CHECK: '1' } });
assert.equal(res.status, 0);
@@ -870,7 +897,7 @@ describe('context.mjs CLI', () => {
});
it('appends no native platform directive for a web project', async () => {
write('PRODUCT.md', '# Acme\n\n## Register\n\nproduct\n\n## Platform\n\nweb\n');
write('PRODUCT.md', '# Acme\n\n## Platform\n\nweb\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8', env: { ...process.env, IMPECCABLE_NO_UPDATE_CHECK: '1' } });
assert.equal(res.status, 0);
@@ -879,7 +906,7 @@ describe('context.mjs CLI', () => {
});
it('appends a native platform directive for an android project', async () => {
write('PRODUCT.md', '# Acme\n\n## Register\n\nproduct\n\n## Platform\n\nandroid\n');
write('PRODUCT.md', '# Acme\n\n## Platform\n\nandroid\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8', env: { ...process.env, IMPECCABLE_NO_UPDATE_CHECK: '1' } });
assert.equal(res.status, 0);
@@ -891,7 +918,7 @@ describe('context.mjs CLI', () => {
// The likeliest misconfiguration is a toolchain name where the target
// belongs. Silent fallback to web would give web guidance to the exact
// projects that tried to declare themselves native.
write('PRODUCT.md', '# Acme\n\n## Register\n\nproduct\n\n## Platform\n\nflutter\n');
write('PRODUCT.md', '# Acme\n\n## Platform\n\nflutter\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8', env: { ...process.env, IMPECCABLE_NO_UPDATE_CHECK: '1' } });
assert.equal(res.status, 0);
@@ -901,7 +928,7 @@ describe('context.mjs CLI', () => {
});
it('emits no warning for an empty Platform section', async () => {
write('PRODUCT.md', '# Acme\n\n## Register\n\nproduct\n\n## Platform\n\n## Users\n\nAnglers.\n');
write('PRODUCT.md', '# Acme\n\n## Platform\n\n## Users\n\nAnglers.\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8', env: { ...process.env, IMPECCABLE_NO_UPDATE_CHECK: '1' } });
assert.equal(res.status, 0);
@@ -6,7 +6,6 @@
*/
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'path';
import { fileURLToPath } from 'url';
import {
@@ -18,38 +17,6 @@ import {
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const FIXTURES = path.join(__dirname, 'fixtures', 'antipatterns');
describe('detectText - Astro structural CSS fixtures', () => {
const SHOULD_FLAG = [
'Kinpaku Edge',
'Patina Edge',
'Accent Edge',
'Signal Blue Edge',
];
const SHOULD_PASS = [
'Neutral Shadow Token',
'Current Color Edge',
'Selected State Edge',
'Hairline Edge',
'Thick Fill Edge',
'Blurred Edge',
'Narrow Artwork',
];
it('Astro style blocks flag unresolved chromatic inset stripes only', () => {
const filePath = path.join(FIXTURES, 'astro-inset-shadow-stripe.astro');
const source = fs.readFileSync(filePath, 'utf8');
const findings = detectText(source, filePath).filter(r => r.antipattern === 'side-tab');
const snippets = findings.map(r => r.snippet || '').join(' | ');
for (const heading of SHOULD_FLAG) {
assert.match(snippets, new RegExp(`data-case=${JSON.stringify(heading)}`), `expected "${heading}" to flag`);
}
for (const heading of SHOULD_PASS) {
assert.doesNotMatch(snippets, new RegExp(`data-case=${JSON.stringify(heading)}`), `"${heading}" should pass`);
}
});
});
describe('detectHtml — static HTML/CSS fixtures', () => {
it('should-flag: catches border anti-patterns', async () => {
const f = await detectHtml(path.join(FIXTURES, 'should-flag.html'));
-15
View File
@@ -1405,21 +1405,6 @@ describe('inset box-shadow stripe', () => {
expect(scanCssTextForInsetStripe(selectedOnly)).toHaveLength(0);
});
test('flags semantically chromatic external tokens without guessing neutral tokens', () => {
const css = `
.kinpaku { box-shadow: inset 3px 0 0 var(--ks-kinpaku-deep); }
.patina { box-shadow: inset 3px 0 0 var(--ks-patina-deep); }
.accent { box-shadow: inset 3px 0 0 var(--brand-accent); }
.signal { box-shadow: inset 3px 0 0 var(--signal-blue); }
.neutral { box-shadow: inset 3px 0 0 var(--shadow-color); }
.current { box-shadow: inset 3px 0 0 currentColor; }
`;
const findings = scanCssTextForInsetStripe(css);
expect(findings).toHaveLength(4);
expect(findings.map(f => f.snippet).join(' ')).not.toContain('.neutral');
expect(findings.map(f => f.snippet).join(' ')).not.toContain('.current');
});
test('static tab strip: chromatic border on every tab flags, selected-only underline stays silent', async () => {
// All-tabs variant: every tab in the group carries the stripe.
await withStaticFixture({
+34
View File
@@ -89,6 +89,40 @@ function routeFromUrl(url) {
}
describe('docs integrity', () => {
test('separates durable world creation from collaborative surface concepts', () => {
const init = fs.readFileSync(path.join(ROOT, 'skill/reference/init.md'), 'utf8');
const newWork = fs.readFileSync(path.join(ROOT, 'skill/reference/new-work.md'), 'utf8');
const shape = fs.readFileSync(path.join(ROOT, 'skill/reference/shape.md'), 'utf8');
const document = fs.readFileSync(path.join(ROOT, 'skill/reference/document.md'), 'utf8');
const codex = fs.readFileSync(path.join(ROOT, 'skill/reference/codex.md'), 'utf8');
const typeset = fs.readFileSync(path.join(ROOT, 'skill/reference/typeset.md'), 'utf8');
expect(init).toContain('## Audience World');
expect(init).toContain('## Cultural Context');
expect(init).toContain('## Pinned Direction');
expect(init).toContain('Ask the user to choose');
expect(init).toContain('user-approved visual world');
expect(newWork).toContain('A committed world does not decide the new surface');
expect(newWork).toContain('concept-seed.mjs');
expect(newWork).toContain('Present two or three materially different surface concepts');
expect(newWork).toContain('DIRECTION CONTRACT');
expect(newWork).toContain('[codex.md](codex.md)');
expect(newWork).toContain('Commit before correcting');
expect(newWork).toContain('Judge the skeleton skin-blind');
expect(newWork).not.toContain('palette.mjs');
expect(shape).toContain('follow [new-work.md](new-work.md)');
expect(shape).toContain('shape never writes code or a direction contract');
expect(init).toContain('“Redesign this page/site” is enough authorization');
expect(init).toContain('Do not offer “the old look, polished”');
expect(document).toContain('load **Step 5: Establish the visual world**');
expect(codex).toContain('this file must not reopen it');
expect(codex).toContain('Do not generate a palette artifact');
expect(typeset).toContain('New identity work belongs to [init.md](init.md)');
expect(typeset).not.toContain('[new-work.md](new-work.md)');
});
test('internal docs links point at canonical local routes', () => {
const routes = knownRoutes();
const broken = [];
@@ -1,41 +0,0 @@
---
const title = 'Astro inset shadow stripe regression';
---
<main>
<h1>{title}</h1>
<section aria-labelledby="should-flag">
<h2 id="should-flag">Should flag</h2>
<article data-case="Kinpaku Edge"><h3>Kinpaku Edge</h3></article>
<article data-case="Patina Edge"><h3>Patina Edge</h3></article>
<article data-case="Accent Edge"><h3>Accent Edge</h3></article>
<article data-case="Signal Blue Edge"><h3>Signal Blue Edge</h3></article>
</section>
<section aria-labelledby="should-pass">
<h2 id="should-pass">Should pass</h2>
<article data-case="Neutral Shadow Token"><h3>Neutral Shadow Token</h3></article>
<article data-case="Current Color Edge"><h3>Current Color Edge</h3></article>
<article data-case="Selected State Edge" aria-current="page"><h3>Selected State Edge</h3></article>
<article data-case="Hairline Edge"><h3>Hairline Edge</h3></article>
<article data-case="Thick Fill Edge"><h3>Thick Fill Edge</h3></article>
<article data-case="Blurred Edge"><h3>Blurred Edge</h3></article>
<article data-case="Narrow Artwork"><h3>Narrow Artwork</h3></article>
</section>
</main>
<style is:inline>
[data-case="Kinpaku Edge"] { box-shadow: inset 3px 0 0 var(--ks-kinpaku-deep); }
[data-case="Patina Edge"] { box-shadow: inset 3px 0 0 var(--ks-patina-deep); }
[data-case="Accent Edge"] { box-shadow: inset -4px 0 0 var(--brand-accent); }
[data-case="Signal Blue Edge"] { box-shadow: inset 0 5px 0 var(--signal-blue); }
[data-case="Neutral Shadow Token"] { box-shadow: inset 3px 0 0 var(--shadow-color); }
[data-case="Current Color Edge"] { box-shadow: inset 3px 0 0 currentColor; }
[data-case="Selected State Edge"][aria-current="page"] { box-shadow: inset 3px 0 0 var(--brand-accent); }
[data-case="Hairline Edge"] { box-shadow: inset 2px 0 0 var(--brand-accent); }
[data-case="Thick Fill Edge"] { box-shadow: inset 14px 0 0 var(--brand-accent); }
[data-case="Blurred Edge"] { box-shadow: inset 3px 0 5px var(--brand-accent); }
[data-case="Narrow Artwork"] { width: 24px; box-shadow: inset 3px 0 0 var(--brand-accent); }
</style>
-24
View File
@@ -117,26 +117,16 @@ for (const name of listFixtures()) {
const ignored = execFileSync('git', [
'check-ignore',
'.impeccable/live/server.json',
'.impeccable/live/codex-worker.json',
'.impeccable/live/codex-worker.log',
'.impeccable/live/sessions/example.jsonl',
'.impeccable/live/previews/example/v1.html',
'.impeccable/live/artifacts/example-r1.jsx',
'.impeccable/live/accept-receipts/example.json',
'.impeccable/live/locks/example.lock',
'.impeccable/live/deferred-svelte-component-accepts.json',
'src/lib/impeccable/ImpeccableLiveRoot.svelte',
'src/lib/impeccable/__runtime.js',
'src/lib/impeccable/a4ac4e74/v3.svelte',
], { cwd: tmp, encoding: 'utf-8' });
assert.match(ignored, /\.impeccable\/live\/server\.json/);
assert.match(ignored, /\.impeccable\/live\/codex-worker\.json/);
assert.match(ignored, /\.impeccable\/live\/codex-worker\.log/);
assert.match(ignored, /\.impeccable\/live\/sessions\/example\.jsonl/);
assert.match(ignored, /\.impeccable\/live\/previews\/example\/v1\.html/);
assert.match(ignored, /\.impeccable\/live\/artifacts\/example-r1\.jsx/);
assert.match(ignored, /\.impeccable\/live\/accept-receipts\/example\.json/);
assert.match(ignored, /\.impeccable\/live\/locks\/example\.lock/);
assert.match(ignored, /\.impeccable\/live\/deferred-svelte-component-accepts\.json/);
assert.match(ignored, /src\/lib\/impeccable\/ImpeccableLiveRoot\.svelte/);
assert.match(ignored, /src\/lib\/impeccable\/__runtime\.js/);
@@ -152,15 +142,6 @@ for (const name of listFixtures()) {
assert.match(root, /localhost:9999\/live\.js/, 'SvelteKit root component loads live.js');
return;
}
if (result.adapter === 'nuxt') {
const plugin = result.results[0];
const body = readFileSync(join(tmp, plugin.file), 'utf-8');
assert.equal(plugin.inserted, true, 'Nuxt client plugin was created');
assert.match(body, /impeccable-live-nuxt-plugin/);
assert.match(body, /if \(!import\.meta\.dev/);
assert.match(body, /localhost:9999\/live\.js/);
return;
}
for (const r of result.results) {
assert.ok(r.inserted, `${r.file} got the tag (result: ${JSON.stringify(r)})`);
const body = readFileSync(join(tmp, r.file), 'utf-8');
@@ -188,11 +169,6 @@ for (const name of listFixtures()) {
assert.equal(existsSync(join(tmp, 'src/lib/impeccable/ImpeccableLiveRoot.svelte')), false);
return;
}
if (result.adapter === 'nuxt') {
assert.equal(result.results[0].removed, true);
assert.equal(existsSync(join(tmp, result.results[0].file)), false, 'Nuxt client plugin was removed');
return;
}
for (const r of result.results) {
const body = readFileSync(join(tmp, r.file), 'utf-8');
assert.doesNotMatch(body, /impeccable-live-start/);
-42
View File
@@ -112,7 +112,6 @@ When `preActions` is omitted, steer smoke inherits `runtime.preActions` to revea
| `nextjs-app/` | `app/layout.tsx` as JSX inject target (commentSyntax `jsx`). |
| `astro/` | `src/layouts/Layout.astro` as inject target. HTML comments. |
| `sveltekit/` | `src/app.html` shell + `src/routes/+page.svelte`. |
| `nuxt-vite7/` | Nuxt 4 `app/` structure + Vue 3 SFC. Live loads through a generated dev-only client plugin. |
| `multipage-with-generator/` | `src/` tracked, `dist/` gitignored. Exercises the is-generated guard and `element_not_in_source` fallback. |
| `nextjs-turborepo/` | Monorepo with shared CSP helper (`createBaseNextConfig`). CSP shape `append-arrays`. |
| `nextjs-inline-csp/` | App-level `next.config.js` with a literal CSP string. CSP shape `append-string`. |
@@ -120,44 +119,3 @@ When `preActions` is omitted, steer smoke inherits `runtime.preActions` to revea
| `nuxt-csp/` | Nuxt `routeRules` with literal CSP header in `nuxt.config.ts`. CSP shape `append-string`. |
Add new fixtures by cloning a directory, swapping files, and updating `fixture.json`.
## External quality-eval fixtures
The public Live benchmark can execute a fixture owned by another repository
without copying its task corpus or rubric into Impeccable:
```sh
bun run bench:live -- \
--fixture-dir=/absolute/path/to/private-fixture \
--agent=codex \
--action=bolder \
--iterations=1 \
--evidence-bundle=/absolute/path/to/output-bundle
```
An external fixture has the same shape as a directory in this folder:
`fixture.json`, `gitignore.txt`, and `files/`. Use the optional
`evidenceCapture` block in `fixture.json` for rubric-free capture metadata:
```json
{
"evidenceCapture": {
"captureSelector": "section.case-study",
"mode": "target",
"viewport": { "width": 1440, "height": 1080 },
"action": "bolder"
}
}
```
Use `"mode": "target"` when `captureSelector` is the picked element itself;
the original resolves through that selector and each variant resolves through
its exact Live wrapper. Omit it when the selector is a stable ancestor used as
shared page context for every capture.
The bundle contains `report.json`, the original capture, each progressively
delivered variant capture, geometry/overflow facts, hashes, and timing data.
It deliberately cannot run `--judge-rendered`; comparative rubrics, private
fixtures, human calibration, and quality decisions belong in the consuming
evaluation harness. The normal public E2E suite remains responsible for Live
protocol, framework, source-commit, cleanup, and recovery correctness.
@@ -0,0 +1,10 @@
<template>
<html lang="en">
<head>
<title>Nuxt + Vite 7 Fixture</title>
</head>
<body>
<NuxtPage />
</body>
</html>
</template>
@@ -1,3 +0,0 @@
<template>
<NuxtPage />
</template>
@@ -1,5 +1,4 @@
export default defineNuxtConfig({
compatibilityDate: '2025-07-15',
devtools: { enabled: false },
ssr: false,
});
@@ -1,38 +1,18 @@
{
"name": "Nuxt 4 + Vue 3",
"name": "Nuxt 4 + Vue 3 (static fixture only — runtime inject unsupported)",
"config": {
"files": ["app/app.vue"],
"insertBefore": "</template>",
"files": ["app.vue"],
"insertBefore": "</body>",
"commentSyntax": "html"
},
"sourceFiles": ["app/app.vue", "app/pages/index.vue", "nuxt.config.ts"],
"sourceFiles": ["app.vue", "pages/index.vue", "nuxt.config.ts"],
"generatedFiles": [],
"wrapCases": [
{
"name": "wraps hero in pages/index.vue",
"args": { "classes": "hero-title", "tag": "h1" },
"expectedFile": "app/.impeccable-live/wraptest0/manifest.json",
"expectedSourceFile": "app/pages/index.vue",
"expectedPreviewMode": "vue-component"
"expectedFile": "pages/index.vue"
}
],
"runtime": {
"styling": "vue-scoped-css",
"install": ["npm", "install", "--no-audit", "--no-fund"],
"devCommand": ["npm", "run", "dev"],
"scheme": "http",
"ignoreHTTPSErrors": false,
"readyPattern": "Local:\\s+http://[^:]+:(\\d+)",
"readyTimeoutMs": 120000,
"pickSelector": "h1.hero-title",
"steer": {
"message": "steer-e2e mark hero",
"sourceFile": "app/pages/index.vue",
"expectSelector": "h1.hero-title[data-impeccable-steer=\"e2e\"]"
},
"probe": {
"expectLiveInit": true,
"expectConsoleClean": true
}
}
"_runtimeOmitted": "Nuxt's app.vue is a Vue template that compiles to a render function — a <script> tag inserted there renders as a DOM node but does not execute. Nuxt needs a config-based inject (nuxt.config.ts -> app.head.script), which live-inject.mjs does not currently support. Static checks (is-generated, inject syntax, wrap routing) still validate."
}
@@ -1,7 +0,0 @@
# Design system
- Warm paper, dark ink, moss, and brass only.
- Georgia display type with a restrained sans body.
- Editorial, practical, quiet, and tactile.
- Reuse the existing CSS custom properties. Do not add colors, fonts, gradients, shadows, glow, glass, or decorative effects.
- Square, rule-led compositions are preferred to card stacks and rounded containers.
@@ -1,7 +0,0 @@
# Northstar Field Journal
An independent quarterly field guide for design-conscious weekend walkers. Readers value practical detail, editorial restraint, and objects worth keeping. The offer card should make issue eight feel collectible without becoming luxurious or loud.
## Platform
web
@@ -1,12 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Northstar Field Journal</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.jsx"></script>
</body>
</html>
@@ -1,18 +0,0 @@
{
"name": "vite8-react-brand-fidelity-fixture",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"dev": "vite --host 127.0.0.1",
"build": "vite build"
},
"dependencies": {
"react": "^19.0.0",
"react-dom": "^19.0.0"
},
"devDependencies": {
"@vitejs/plugin-react": "^6.0.0",
"vite": "^8.0.0"
}
}
@@ -1,26 +0,0 @@
function ActionLink({ children }) {
return <a className="action-link" href="#edition">{children}</a>;
}
export default function App() {
return (
<main className="page-shell">
<header className="masthead">
<p className="masthead__kicker">Northstar Field Journal</p>
<h1>Useful observations from the long way around.</h1>
</header>
<section className="edition" id="edition" aria-labelledby="edition-title">
<p className="edition__number">Edition 08 · Coastal paths</p>
<article className="offer-card" aria-labelledby="field-notes-title">
<div className="offer-card__copy">
<p className="offer-card__eyebrow">Quarterly print edition</p>
<h2 className="offer-card__title" id="field-notes-title">Field Notes</h2>
<p className="offer-card__body">Four routes, annotated maps, and practical details for unhurried weekends.</p>
</div>
<ActionLink>Reserve issue eight</ActionLink>
</article>
</section>
</main>
);
}
@@ -1,10 +0,0 @@
import { StrictMode } from 'react';
import { createRoot } from 'react-dom/client';
import App from './App.jsx';
import './styles.css';
createRoot(document.getElementById('root')).render(
<StrictMode>
<App />
</StrictMode>,
);
@@ -1,111 +0,0 @@
:root {
--color-paper: #f3efe4;
--color-paper-deep: #e7dfcf;
--color-ink: #20251f;
--color-moss: #526248;
--color-brass: #9b6b2f;
--font-display: Georgia, "Times New Roman", serif;
--font-body: Inter, Arial, sans-serif;
--space-1: 0.5rem;
--space-2: 1rem;
--space-3: 1.5rem;
--space-4: 2.5rem;
--radius-control: 0.25rem;
}
* { box-sizing: border-box; }
body {
margin: 0;
background: var(--color-paper);
color: var(--color-ink);
font-family: var(--font-body);
}
.page-shell {
width: min(70rem, calc(100% - 2rem));
margin: 0 auto;
padding: 5rem 0;
}
.masthead {
max-width: 50rem;
margin-bottom: 4rem;
}
.masthead__kicker,
.edition__number,
.offer-card__eyebrow {
color: var(--color-moss);
font-size: 0.75rem;
font-weight: 700;
letter-spacing: 0.12em;
text-transform: uppercase;
}
h1,
h2 {
font-family: var(--font-display);
font-weight: 400;
text-wrap: balance;
}
h1 {
margin: var(--space-2) 0 0;
font-size: clamp(3rem, 7vw, 5.5rem);
line-height: 0.98;
}
.edition {
border-top: 1px solid var(--color-brass);
padding-top: var(--space-2);
}
.offer-card {
display: grid;
grid-template-columns: minmax(0, 1fr) auto;
gap: var(--space-4);
align-items: end;
margin-top: var(--space-2);
padding: var(--space-4);
background: var(--color-paper-deep);
border-left: 0.25rem solid var(--color-moss);
}
.offer-card__eyebrow,
.offer-card__body {
margin: 0;
}
.offer-card__title {
margin: var(--space-1) 0;
font-size: 2.5rem;
line-height: 1;
}
.offer-card__body {
max-width: 58ch;
line-height: 1.65;
}
.action-link {
display: inline-flex;
min-height: 2.75rem;
align-items: center;
padding: 0 var(--space-3);
border: 1px solid var(--color-ink);
border-radius: var(--radius-control);
color: var(--color-ink);
font-weight: 700;
text-decoration: none;
}
.action-link:focus-visible {
outline: 0.2rem solid var(--color-brass);
outline-offset: 0.2rem;
}
@media (max-width: 42rem) {
.offer-card { grid-template-columns: 1fr; }
.action-link { justify-content: center; }
}
@@ -1,10 +0,0 @@
import { defineConfig } from 'vite';
import react from '@vitejs/plugin-react';
export default defineConfig({
plugins: [react()],
server: {
host: '127.0.0.1',
strictPort: false,
},
});
@@ -1,67 +0,0 @@
{
"name": "Vite 8 + React + brand fidelity",
"config": {
"files": ["index.html"],
"insertBefore": "</body>",
"commentSyntax": "html"
},
"sourceFiles": ["PRODUCT.md", "DESIGN.md", "index.html", "src/App.jsx", "src/main.jsx", "src/styles.css", "vite.config.js"],
"generatedFiles": [],
"renderedQuality": {
"remoteSafe": true,
"captureSelector": "main.page-shell",
"viewport": { "width": 1280, "height": 900 },
"action": "bolder",
"brief": "Make the Field Notes offer materially bolder while preserving Northstar's restrained editorial system.",
"reviewFocus": "Hierarchy, proportion, composition, brand fidelity, usability, and copy preservation.",
"constraints": [
"Warm paper, dark ink, moss, and brass only",
"Georgia display type with a restrained sans body",
"No gradients, shadows, glow, or invented content",
"Preserve every word and the ActionLink"
],
"tokens": {
"--color-paper": "#f3efe4",
"--color-paper-deep": "#e7dfcf",
"--color-ink": "#20251f",
"--color-moss": "#526248",
"--color-brass": "#9b6b2f",
"--font-display": "Georgia, Times New Roman, serif",
"--font-body": "Inter, Arial, sans-serif"
},
"componentRoles": {
"ActionLink": "Quiet outlined control; preserve its label, border, radius, and interaction role",
"offer-card": "Warm-paper offer surface with dark ink, moss structure, and optional brass rules"
},
"redactSelectors": []
},
"wrapCases": [
{
"name": "wraps the benchmark offer card in source JSX",
"args": { "classes": "offer-card", "tag": "article", "text": "Field Notes" },
"expectedFile": "src/App.jsx"
}
],
"runtime": {
"styling": "plain-css",
"install": ["npm", "install", "--no-audit", "--no-fund", "--loglevel=error"],
"devCommand": ["npx", "vite", "--host", "127.0.0.1"],
"readyPattern": "Local:\\s+https?://[^:]+:(\\d+)",
"readyTimeoutMs": 120000,
"pickSelector": "article.offer-card",
"pickPosition": { "x": 8, "y": 8 },
"expectedPick": { "tagName": "article", "classes": ["offer-card"] },
"acceptedSourcePattern": "<article[^>]*(class|className)=\"[^\"]*\\boffer-card\\b[^\"]*\"",
"steer": {
"message": "steer-e2e mark offer",
"target": { "classes": "offer-card", "tag": "article" },
"expectSelector": "article.offer-card[data-impeccable-steer=\"e2e\"]",
"expectSourceContains": "data-impeccable-steer=\"e2e\"",
"sourceFile": "src/App.jsx"
},
"probe": {
"expectLiveInit": true,
"expectConsoleClean": true
}
}
}
@@ -1,3 +0,0 @@
node_modules
dist
.impeccable
@@ -6,10 +6,3 @@
<article class="feature-card">Two</article>
</section>
</main>
<style>
.feature-card {
min-height: 64px;
padding: 12px;
}
</style>
+45 -1
View File
@@ -54,6 +54,7 @@ import {
splitFindingsByTier,
perEditTieringActive,
extractDirectionContract,
missingDirectionContractFields,
renderContractAudit,
CONTRACT_MAX_CHARS,
CONTRACT_HEAD_CHARS,
@@ -2802,6 +2803,22 @@ describe('extractDirectionContract()', () => {
assert.match(extractDirectionContract(html) || '', /boarding pass/);
});
it('extracts a JSX direction-contract block', () => {
const jsx = `{/*\nDIRECTION CONTRACT\nUNIQUE: the timeline folds around the evidence.\n*/}\nexport default function Page() { return <main />; }`;
assert.match(extractDirectionContract(jsx) || '', /timeline folds/);
});
it('reports missing contract fields deterministically', () => {
assert.deepEqual(
missingDirectionContractFields('DIRECTION CONTRACT\nUNIQUE: own idea\nFORM: evidence timeline'),
['NOT-TEMPLATE', 'OWN-WORLD', 'STORY', 'FIRST VIEWPORT'],
);
assert.deepEqual(missingDirectionContractFields([
'UNIQUE: x', 'NOT-TEMPLATE: x', 'OWN-WORLD: x',
'STORY: x', 'FIRST VIEWPORT: x', 'FORM: x',
].join('\n')), []);
});
it('returns null when there is no comment at all', () => {
assert.equal(extractDirectionContract('<!doctype html><html></html>'), null);
assert.equal(extractDirectionContract(''), null);
@@ -2848,6 +2865,12 @@ describe('extractDirectionContract()', () => {
assert.doesNotMatch(text, new RegExp(`DIRECTION CONTRACT ${CONTRACT_AUDIT_MAX_FILES}\\b`));
assert.equal(renderContractAudit([], {}), '');
});
it('renderContractAudit flags an incomplete promise contract', () => {
const text = renderContractAudit([{ filePath: '/tmp/page.html', contract: 'DIRECTION CONTRACT\nUNIQUE: one idea' }], { cwd: '/tmp' });
assert.match(text, /Contract integrity defect/);
assert.match(text, /NOT-TEMPLATE/);
});
});
describe('runStopHook() — direction-contract audit', () => {
@@ -2934,6 +2957,27 @@ describe('runStopHook() — direction-contract audit', () => {
assert.doesNotMatch(out.hookSpecificOutput.additionalContext, /findings requiring review/);
});
it('audits contracts in Astro, Svelte, Vue, JSX, and TSX artifacts', async () => {
const htmlFamily = ['astro', 'svelte', 'vue'];
const jsxFamily = ['jsx', 'tsx'];
for (const ext of htmlFamily) {
const sid = `contract-${ext}`;
const file = write(`src/Page.${ext}`, `<!--\nDIRECTION CONTRACT\nUNIQUE: ${ext} evidence ribbon.\n-->\n<main />`);
await touch(file, sid, fakeDetector([]));
const stop = await runStopHook({ stdinJson: JSON.stringify(stopEvent(sid)), env: {}, cwd, detector: fakeDetector([]) });
assert.match(stop.stdout, new RegExp(`${ext} evidence ribbon`));
}
for (const ext of jsxFamily) {
const sid = `contract-${ext}`;
const file = write(`src/Page.${ext}`, `{/*\nDIRECTION CONTRACT\nUNIQUE: ${ext} evidence ribbon.\n*/}\nexport default function Page() { return <main />; }`);
await touch(file, sid, fakeDetector([]));
const stop = await runStopHook({ stdinJson: JSON.stringify(stopEvent(sid)), env: {}, cwd, detector: fakeDetector([]) });
assert.match(stop.stdout, new RegExp(`${ext} evidence ribbon`));
}
});
it('adds no audit section when the HTML has no contract comment', async () => {
const sid = 'no-contract';
const file = write('index.html', '<!doctype html><html><body>plain</body></html>');
@@ -2968,7 +3012,7 @@ describe('runStopHook() — direction-contract audit', () => {
assert.doesNotMatch(text, /Direction-contract audit/);
});
it('ignores contract-looking comments in non-HTML touched files', async () => {
it('ignores marker text outside a supported contract comment block', async () => {
const sid = 'contract-non-html';
const file = write('src/Card.tsx', '{/* stub */}\n// <!-- DIRECTION CONTRACT: not an artifact -->\nexport default 1;\n');
const det = fakeDetector([finding('em-dash-overuse', 3)]);
+1 -78
View File
@@ -5,12 +5,11 @@
import { describe, it, beforeEach, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { existsSync, mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import { tmpdir } from 'node:os';
import { fileURLToPath } from 'node:url';
import { execFileSync } from 'node:child_process';
import { scaffoldSourceArtifactSession } from '../skill/scripts/live/source-artifact.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const ACCEPT = resolve(__dirname, '..', 'skill/scripts/live-accept.mjs');
@@ -30,55 +29,6 @@ function runAccept(cwd, args) {
}
}
describe('live-accept — isolated source artifacts', () => {
let tmp;
beforeEach(() => { tmp = mkdtempSync(join(tmpdir(), 'impeccable-accept-isolated-')); });
afterEach(() => { rmSync(tmp, { recursive: true, force: true }); });
function scaffold(id) {
const original = '<main>\n <section class="hero"><h1>Original</h1></section>\n</main>\n';
writeFileSync(join(tmp, 'page.html'), original);
const session = scaffoldSourceArtifactSession({
id,
count: 2,
sourceFile: 'page.html',
sourceStartLine: 2,
sourceEndLine: 2,
originalSource: '<section class="hero"><h1>Original</h1></section>',
previewContent: `<main>
<!-- impeccable-variants-start ${id} -->
<div data-impeccable-variants="${id}" data-impeccable-variant-count="2" style="display: contents">
<div data-impeccable-variant="original"><section class="hero"><h1>Original</h1></section></div>
<div data-impeccable-variant="1"><section class="hero"><h1>Accepted one</h1></section></div>
<div data-impeccable-variant="2"><section class="hero"><h1>Accepted two</h1></section></div>
</div>
<!-- impeccable-variants-end ${id} -->
</main>
`,
cwd: tmp,
});
return { original, session };
}
it('accepts one preview into true source exactly once', () => {
const { session } = scaffold('isolatedaccept');
const result = runAccept(tmp, ['--id', 'isolatedaccept', '--variant', '2']);
assert.equal(result.handled, true, JSON.stringify(result));
const source = readFileSync(join(tmp, 'page.html'), 'utf-8');
assert.match(source, /Accepted two/);
assert.doesNotMatch(source, /Accepted one|data-impeccable-variant/);
assert.equal(existsSync(join(tmp, session.sessionDir)), false);
});
it('discards the preview instantly without touching true source', () => {
const { original, session } = scaffold('isolateddiscard');
const result = runAccept(tmp, ['--id', 'isolateddiscard', '--discard']);
assert.equal(result.handled, true, JSON.stringify(result));
assert.equal(readFileSync(join(tmp, 'page.html'), 'utf-8'), original);
assert.equal(existsSync(join(tmp, session.sessionDir)), false);
});
});
describe('live-accept — style-element edge cases', () => {
let tmp;
beforeEach(() => { tmp = mkdtempSync(join(tmpdir(), 'impeccable-accept-test-')); });
@@ -124,33 +74,6 @@ describe('live-accept — style-element edge cases', () => {
assert.ok(!after.includes('original text'), 'original content dropped');
});
it('replays a durable receipt when Accept is retried after source was already written', () => {
const html = `<body>
<!-- impeccable-variants-start RECEIPT1 -->
<div data-impeccable-variants="RECEIPT1" data-impeccable-variant-count="2" style="display: contents">
<div data-impeccable-variant="original"><p>original</p></div>
<style data-impeccable-css="RECEIPT1" />
<div data-impeccable-variant="1"><p>accepted once</p></div>
<div data-impeccable-variant="2" style="display: none"><p>other</p></div>
</div>
<!-- impeccable-variants-end RECEIPT1 -->
</body>`;
writeFileSync(join(tmp, 'page.html'), html);
const first = runAccept(tmp, ['--id', 'RECEIPT1', '--variant', '1']);
const afterFirst = readFileSync(join(tmp, 'page.html'), 'utf-8');
const replay = runAccept(tmp, ['--id', 'RECEIPT1', '--variant', '1']);
assert.equal(first.handled, true);
assert.equal(replay.handled, true);
assert.equal(replay.alreadyApplied, true);
assert.equal(replay.file, 'page.html');
assert.equal(readFileSync(join(tmp, 'page.html'), 'utf-8'), afterFirst);
const conflict = runAccept(tmp, ['--id', 'RECEIPT1', '--variant', '2']);
assert.equal(conflict.handled, false);
assert.equal(conflict.error, 'accept_receipt_conflict');
});
// Variant: same-line <style>…</style> block should also be treated as a
// single skipped unit; the line has both open and close tags.
it('finds the accepted variant after a single-line <style>…</style> block', () => {
-252
View File
@@ -1,252 +0,0 @@
import assert from 'node:assert/strict';
import { describe, it } from 'node:test';
import {
assembleSplitProgressiveOutput,
buildInteractionRun,
compareModelBackedReports,
createTraceRecorder,
deriveJournalGenerationMetrics,
durationBetween,
parseLiveBenchmarkArgs,
resolveLiveBenchmarkPaths,
summarizeRuns,
} from '../scripts/lib/live-benchmark.mjs';
describe('live benchmark metrics', () => {
it('normalizes documented kebab-case CLI flags', () => {
assert.deepEqual(parseLiveBenchmarkArgs([
'--accept-first',
'--judge-rendered=true',
'--worker-timeout-ms=25000',
'--accept-variant=2',
]), {
acceptFirst: true,
judgeRendered: 'true',
workerTimeoutMs: '25000',
acceptVariant: '2',
});
});
it('resolves an external fixture into a portable rubric-free evidence bundle', () => {
const paths = resolveLiveBenchmarkPaths({
fixtureDir: '../impeccable-evals/fixtures/live/tidewater',
evidenceBundle: '/tmp/live-tidewater',
}, {
root: '/workspace/impeccable',
fixturesDir: '/workspace/impeccable/tests/framework-fixtures',
});
assert.deepEqual(paths, {
fixtureName: 'tidewater',
fixtureDir: '/workspace/impeccable-evals/fixtures/live/tidewater',
fixtureOrigin: 'external',
evidenceRoot: '/tmp/live-tidewater',
artifactRoot: '/tmp/live-tidewater',
outputPath: '/tmp/live-tidewater/report.json',
});
});
it('keeps evidence bundles atomic and unambiguous', () => {
const options = { root: '/repo', fixturesDir: '/repo/tests/framework-fixtures' };
assert.throws(
() => resolveLiveBenchmarkPaths({ evidenceBundle: 'bundle', artifacts: 'shots' }, options),
/replaces --artifacts/,
);
assert.throws(
() => resolveLiveBenchmarkPaths({ evidenceBundle: 'bundle', output: 'report.json' }, options),
/writes report.json itself/,
);
});
it('derives production worker phases from the durable session journal', () => {
const metrics = deriveJournalGenerationMetrics({
generationTimings: {
picked_up: { at: 100 },
source_ready: { at: 124 },
first_variant_generating: { at: 130 },
first_variant_validating: { at: 210 },
first_reviewable: { at: 240 },
second_variant_generating: { at: 245 },
second_variant_validating: { at: 300 },
second_reviewable: { at: 315 },
remaining_variants_generating: { at: 320 },
remaining_variants_validating: { at: 400 },
all_variants_ready: { at: 430 },
},
});
assert.equal(metrics.workerPickupToSourceReadyMs, 24);
assert.equal(metrics.workerFirstGenerationToReviewableMs, 110);
assert.equal(metrics.workerFirstValidationToReviewableMs, 30);
assert.equal(metrics.workerSecondGenerationToReviewableMs, 70);
assert.equal(metrics.workerSecondValidationToReviewableMs, 15);
assert.equal(metrics.workerRemainingGenerationToReadyMs, 110);
assert.equal(metrics.workerRemainingValidationToReadyMs, 30);
assert.deepEqual(metrics.journalTimingErrors, []);
});
it('surfaces non-monotonic worker phases instead of reporting a false zero', () => {
const metrics = deriveJournalGenerationMetrics({
generationTimings: {
first_variant_generating: { at: 300 },
first_reviewable: { at: 200 },
},
});
assert.equal(metrics.workerFirstGenerationToReviewableMs, null);
assert.deepEqual(metrics.journalTimingErrors, ['first_reviewable_before_first_variant_generating']);
});
it('keeps published progressive CSS byte-stable and carries deferred params', () => {
const firstCss = '@scope ([data-impeccable-variant="1"]) { .offer { color: red; } }';
const laterCss = [
'@scope ([data-impeccable-variant="2"]) { .offer { color: green; } }',
'@scope ([data-impeccable-variant="3"]) { .offer { color: blue; } }',
].join('\n');
const firstVariant = { innerHtml: '<article class="offer">One</article>', params: [] };
const deferredParams = [{ name: 'density', type: 'range', min: 0, max: 1, default: 0.5 }];
const assembled = assembleSplitProgressiveOutput(
{ scopedCss: firstCss, variants: [firstVariant] },
{
scopedCss: laterCss,
variants: [
{ innerHtml: firstVariant.innerHtml, params: deferredParams },
{ innerHtml: '<article class="offer">Two</article>', params: [] },
{ innerHtml: '<article class="offer">Three</article>', params: [] },
],
},
);
assert.equal(assembled.scopedCss, `${firstCss}\n${laterCss}`);
assert.equal(assembled.scopedCss.slice(0, firstCss.length), firstCss);
assert.equal(assembled.variants[0].innerHtml, firstVariant.innerHtml);
assert.equal(assembled.variants[0].params, deferredParams);
});
it('rejects tail CSS that would reproduce published_variant_css_changed', () => {
const first = {
scopedCss: '@scope ([data-impeccable-variant="1"]) { .offer { color: red; } }',
variants: [{ innerHtml: '<article class="offer">One</article>', params: [] }],
};
const conflictingTail = {
scopedCss: [
'@scope ([data-impeccable-variant="1"]) { .offer { color: purple; } }',
'@scope ([data-impeccable-variant="2"]) { .offer { color: green; } }',
].join('\n'),
variants: [
{ innerHtml: first.variants[0].innerHtml, params: [] },
{ innerHtml: '<article class="offer">Two</article>', params: [] },
],
};
assert.throws(
() => assembleSplitProgressiveOutput(first, conflictingTail),
/must not repeat or conflict with published variant 1 CSS/,
);
});
it('separates model generation from Impeccable overhead', () => {
const events = [
{ name: 'ui.go.start', at: 100, iteration: 1 },
{ name: 'browser.generate_post', at: 108, id: 'abc', selectedTagName: 'article', selectedClasses: ['offer-card'], hasScreenshotPath: false, commentCount: 0, strokeCount: 0 },
{ name: 'agent.event.received', at: 110, id: 'abc', type: 'generate' },
{ name: 'agent.scaffold.start', at: 112, id: 'abc' },
{ name: 'agent.scaffold.end', at: 132, id: 'abc' },
{ name: 'agent.generate.start', at: 132, id: 'abc' },
{ name: 'agent.generate.first_ready', at: 1132, id: 'abc' },
{ name: 'agent.generate.end', at: 1132, id: 'abc' },
{ name: 'agent.write.start', at: 1132, id: 'abc' },
{ name: 'agent.write.end', at: 1142, id: 'abc' },
{ name: 'agent.reply.start', at: 1142, id: 'abc' },
{ name: 'agent.reply.end', at: 1147, id: 'abc' },
{ name: 'browser.first_variant', at: 1200, iteration: 1 },
{ name: 'browser.second_variant', at: 1200, iteration: 1 },
{ name: 'browser.all_variants', at: 1200, iteration: 1 },
];
const run = buildInteractionRun(events, {
iteration: 1,
scenario: 'plain',
goStartedAt: 100,
browserTiming: { goAt: 50, generateAt: 52.5 },
});
assert.equal(run.goToFirstVariantMs, 1094.5);
assert.equal(run.goToSecondVariantMs, 1094.5);
assert.equal(run.browserPreparationMs, 8);
assert.equal(run.browserDispatchMs, 2.5);
assert.equal(run.automationClickMs, 5.5);
assert.deepEqual(run.annotationEvidence, { screenshotPath: false, comments: 0, strokes: 0 });
assert.deepEqual(run.selectionEvidence, { tagName: 'article', classes: ['offer-card'] });
assert.equal(run.serverPickupMs, 2);
assert.equal(run.generationMs, 1000);
assert.equal(run.impeccableOverheadMs, 94.5);
assert.equal(run.deliveryGapMs, 0);
assert.equal(run.scaffoldMs, 20);
});
it('reports interpolated medians and p95 values', () => {
const summary = summarizeRuns([
{ goToFirstVariantMs: 100, generationMs: 70 },
{ goToFirstVariantMs: 200, generationMs: 140 },
{ goToFirstVariantMs: 300, generationMs: 210 },
]);
assert.equal(summary.metrics.goToFirstVariantMs.median, 200);
assert.equal(summary.metrics.goToFirstVariantMs.p95, 290);
});
it('records monotonic trace events and returns null for missing boundaries', () => {
let now = 0;
const recorder = createTraceRecorder(() => ++now);
recorder.trace('start');
recorder.trace('end');
assert.equal(durationBetween(recorder.events, 'start', 'end'), 1);
assert.equal(durationBetween(recorder.events, 'missing', 'end'), null);
});
it('proves model-backed first-reviewable thresholds with comparable reports', () => {
const atomic = modelReport('atomic', 1000, 1200, 1400, 1500);
const progressive = modelReport('progressive', 500, 700, 1450, 1550);
const comparison = compareModelBackedReports(atomic, progressive);
assert.equal(comparison.passed, true);
assert.equal(comparison.target.medianImprovement, 0.5);
assert.equal(comparison.target.p95Improvement, 0.4167);
});
it('rejects fake, simulated, and mismatched model reports', () => {
const atomic = modelReport('atomic', 1000, 1200, 1400, 1500);
const progressive = modelReport('progressive', 500, 700, 1450, 1550);
assert.throws(
() => compareModelBackedReports({ ...atomic, benchmark: { ...atomic.benchmark, agent: 'fake' } }, progressive),
/model-backed/,
);
assert.throws(
() => compareModelBackedReports(atomic, { ...progressive, benchmark: { ...progressive.benchmark, simulation: { remainingGenerationMs: 1 } } }),
/simulated latency/,
);
assert.throws(
() => compareModelBackedReports(atomic, { ...progressive, benchmark: { ...progressive.benchmark, model: 'other-model' } }),
/benchmark mismatch for model/,
);
});
});
function modelReport(delivery, firstMedian, firstP95, allMedian, allP95) {
return {
benchmark: {
fixture: 'vite8-react-plain',
agent: 'llm',
provider: 'anthropic',
model: 'claude-haiku-4-5',
scenario: 'plain',
variants: 3,
delivery,
promptMode: 'synthetic-element-contract',
simulation: null,
},
summary: {
count: 5,
metrics: {
goToFirstVariantMs: { median: firstMedian, p95: firstP95 },
goToAllVariantsMs: { median: allMedian, p95: allP95 },
},
},
};
}
+27 -82
View File
@@ -74,7 +74,7 @@ describe('live-browser.js regression guards', () => {
);
});
it('uses a framework-component-gated painted-ancestor crop proxy for shader capture', () => {
it('uses a Svelte-gated painted-ancestor crop proxy for shader capture', () => {
assert.match(
SOURCE,
/function findShaderProxyCaptureRoot\(el\) \{[\s\S]{0,500}?let node = el\.parentElement;[\s\S]{0,700}?containsElement && paintsShaderProxySurface\(node\)[\s\S]{0,120}?return null;/,
@@ -87,8 +87,8 @@ describe('live-browser.js regression guards', () => {
);
assert.match(
SOURCE,
/function shouldUseAncestorCropShaderProxy\(el\) \{[\s\S]{0,260}?window\.__IMPECCABLE_LIVE_ADAPTER__[\s\S]{0,280}?isFrameworkComponentPreviewMode\(currentPreviewMode\) \|\| svelteComponentSession[\s\S]{0,260}?isFrameworkComponentPreviewMode\(wrapper\?\.dataset\?\.impeccablePreview\);/,
'ancestor crop proxy must be gated to Svelte/Vue component previews',
/function shouldUseAncestorCropShaderProxy\(el\) \{[\s\S]{0,260}?window\.__IMPECCABLE_LIVE_ADAPTER__[\s\S]{0,280}?currentPreviewMode === 'svelte-component' \|\| svelteComponentSession[\s\S]{0,260}?dataset\?\.impeccablePreview === 'svelte-component';/,
'ancestor crop proxy must be gated to the Svelte adapter / Svelte component previews',
);
assert.match(
SOURCE,
@@ -141,37 +141,11 @@ describe('live-browser.js regression guards', () => {
it('restores unsaved inline edit drafts before hideBar tears editing down', () => {
assert.match(
SOURCE,
/function hideBar\(instant\) \{[\s\S]{0,720}?if \(state === 'EDITING'\) restoreInlineEditDrafts\(\);[\s\S]{0,80}?disableInlineEdit\(\);/,
/function hideBar\(\) \{[\s\S]{0,620}?if \(state === 'EDITING'\) restoreInlineEditDrafts\(\);[\s\S]{0,80}?disableInlineEdit\(\);/,
'hideBar should not leave unsaved contenteditable drafts in the DOM when an external event hides the bar',
);
});
it('discards variants without hiding the original or animating stale chrome', () => {
assert.match(SOURCE, /function showOriginalDuringDiscard\(sessionId\)[\s\S]{0,900}?data-impeccable-variant="original"/);
assert.match(SOURCE, /function handleDiscard\(\)[\s\S]{0,420}?cleanup\(\{ restoreOriginal: true, instantChrome: true \}\)/);
assert.match(SOURCE, /if \(instant\) barEl\.style\.display = 'none'/);
assert.match(
SOURCE,
/if \(restoreOriginal\) showOriginalDuringDiscard\(cleanupSessionId\);\s*else wrapper\.style\.display = 'none';/,
'only non-discard cleanup may blank the wrapper while waiting for HMR',
);
});
it('stores live state off the document root and preserves the selected anchor top', () => {
assert.match(SOURCE, /window\.__IMPECCABLE_LIVE_STATE__ = next/);
assert.doesNotMatch(SOURCE, /document\.documentElement\.dataset\.impeccableLiveState/);
assert.match(SOURCE, /pickedAnchorViewportTop: Number\.isFinite\(pickedAnchorViewportTop\)/);
assert.match(SOURCE, /scrollLockAnchorTop = typeof initialAnchorTop === 'number' && isFinite\(initialAnchorTop\)/);
assert.match(SOURCE, /const anchorDelta = anchorTop - scrollLockAnchorTop/);
});
it('injects source-artifact previews immediately instead of waiting for HMR', () => {
assert.match(
SOURCE,
/else if \(isSourceArtifactPreviewMode\(msg\.previewMode\) && msg\.previewFile\) \{\s*injectVariantsFromSource\(msg\.previewFile/,
);
});
it('does not autofocus the steering chat while inline editing', () => {
assert.match(
SOURCE,
@@ -867,58 +841,6 @@ describe('live-browser.js regression guards', () => {
);
});
it('makes every arrived progressive variant immediately actionable', () => {
assert.match(
SOURCE,
/if \(arrivedVariants > 0\) \{[\s\S]{0,180}?setLiveState\('CYCLING'\)/,
'the first arrived variant should leave the generating-only state',
);
assert.doesNotMatch(
SOURCE,
/arrivedVariants < expectedVariants\) \{[\s\S]{0,180}?accept\.style\.pointerEvents = 'none'/,
'Accept must not wait for variants the user did not choose',
);
assert.doesNotMatch(
SOURCE,
/arrivedVariants < expectedVariants\) \{[\s\S]{0,180}?discard\.style\.pointerEvents = 'none'/,
'Discard must cancel remaining work immediately',
);
assert.match(
SOURCE,
/const resumedState = arrivedVariants > 0 \? 'CYCLING' : 'GENERATING'/,
'reload recovery should preserve a partially delivered review state',
);
assert.match(
SOURCE,
/arrivedVariants >= expectedVariants && expectedVariants > 0[\s\S]{0,100}?\? 'variants_ready'[\s\S]{0,60}?: 'variants_progress'/,
'checkpoint timing must distinguish partial review from complete delivery by counts',
);
});
it('keeps deferred Tune controls visible and refreshes params-only publications', () => {
assert.match(
SOURCE,
/const paramsPending = !hasParams && \(parameterGenerationState === 'pending' \|\| parameterGenerationState === 'loading'\)/,
'the cycling bar must expose Tune while parameter generation is outstanding',
);
assert.match(SOURCE, /tune\.disabled = true/, 'pending Tune must be visibly loading but non-interactive');
assert.match(SOURCE, /Tune controls are ready\./, 'parameter arrival needs a clear ready indication');
assert.match(
SOURCE,
/msg\.publicationKind !== 'params' && arrivedVariants >= targetArrived/,
'a params-only publication must refresh even though the variant count is unchanged',
);
assert.match(SOURCE, /revisionDomain: 'browser'/, 'browser checkpoints must use their own revision domain');
});
it('promotes an early-accepted Svelte preview before releasing the picker', () => {
assert.match(
SOURCE,
/function scheduleAcceptCleanup\(accepted\) \{[\s\S]{0,420}?if \(accepted\?\.isSvelteComponent\) \{[\s\S]{0,120}?commitAcceptedSvelteComponentToDom\(accepted\.id\);[\s\S]{0,120}?cleanupAcceptedSession\(\);/,
'Svelte early accept must tear down its adapter mount before the next picking session starts',
);
});
it('variant injection resolves the picked anchor before entering recovery', () => {
assert.match(
SOURCE,
@@ -937,6 +859,29 @@ describe('live-browser.js regression guards', () => {
);
});
it('uses dark ink for every control filled with kinpaku gold in the Tune drawer', () => {
assert.match(
SOURCE,
/background: tuneOpen \? C\.brand : BP\.hairline,[\s\S]{0,100}?color: tuneOpen \? C\.ink : 'inherit'/,
'the active Tune count badge needs dark ink on gold',
);
assert.match(
SOURCE,
/background: active \? C\.brand : 'transparent',[\s\S]{0,100}?color: active \? C\.ink : P\.text/,
'active segmented options need dark ink on gold',
);
assert.match(
SOURCE,
/btn\.style\.background = on \? C\.brand : 'transparent';\s*btn\.style\.color = on \? C\.ink : P\.text;/,
'segmented options must keep dark ink after interaction',
);
assert.match(
SOURCE,
/const knob = el\('span',[\s\S]{0,300}?background: C\.ink/,
'the toggle knob needs a visible dark edge against gold',
);
});
it('editing focus timeout does not read a stale inline edit row', () => {
assert.doesNotMatch(
SOURCE,
+8 -65
View File
@@ -5,57 +5,8 @@ import { join } from 'node:path';
const SOURCE = readFileSync(join(process.cwd(), 'skill/scripts/live-browser.js'), 'utf-8');
const PENDING_DOCK_POSITION_SOURCE = SOURCE.match(/function positionPendingDock\(\) \{[\s\S]*?\n \}/)?.[0] || '';
const CAPTURE_AND_EMIT_SOURCE = SOURCE.match(/async function captureAndEmit\([\s\S]*?\n \}/)?.[0] || '';
describe('live-browser source contracts', () => {
it('surfaces missing Codex CLI fallback without treating the agent as disconnected', () => {
assert.match(
SOURCE,
/syncAgentPollingUi\(!!msg\.agentPolling, msg\.codexWorker\)/,
'the initial SSE state should include the dedicated worker status',
);
assert.match(
SOURCE,
/cliUnavailable = codexWorkerStatus\?\.error === 'codex_cli_unavailable'[\s\S]*?using foreground generation/,
'a missing CLI should keep Live usable while exposing foreground mode accessibly',
);
assert.match(
SOURCE,
/Codex CLI is unavailable\. Live is using the main agent, so generation may take longer\.[\s\S]*?codex login/,
'the one-time fallback notice should explain the performance impact and recovery action',
);
});
it('routes Nuxt Vue preview modules through the Vite build-assets base', () => {
assert.match(
SOURCE,
/function resolveComponentModuleUrl\(manifest, modulePath\)[\s\S]*?manifest\?\.previewMode === 'vue-component'[\s\S]*?window\.__NUXT__\?\.config\?\.app\?\.buildAssetsDir[\s\S]*?pathValue\.slice\('\/@fs\/'.length\)/,
'Nuxt must not send app-local preview modules through the page-route fallback',
);
assert.match(
SOURCE,
/const moduleBase = manifest\.componentModuleBase[\s\S]*?resolveComponentModuleUrl\(manifest, modulePath\)/,
'Vue SFC variants should use the manifest Vite module base rather than componentDir as a route URL',
);
});
it('dispatches plain generation before screenshot capture without bypassing annotated evidence', () => {
const dispatchIndex = CAPTURE_AND_EMIT_SOURCE.indexOf('await sendEvent(basePayload);');
const captureIndex = CAPTURE_AND_EMIT_SOURCE.indexOf('await captureElementToBlob');
assert.ok(dispatchIndex >= 0, 'plain generation should dispatch immediately');
assert.ok(captureIndex > dispatchIndex, 'plain generation dispatch must happen before capture begins');
assert.match(
CAPTURE_AND_EMIT_SOURCE,
/if \(blob && hasAnnotations\)[\s\S]*?\/annotation\?token=/,
'annotation screenshots should still upload before annotated generation dispatch',
);
assert.match(
CAPTURE_AND_EMIT_SOURCE,
/if \(hasAnnotations\) \{[\s\S]*?basePayload\.clientSentAt = Date\.now\(\);\s*sendEvent\(screenshotPath \? \{ \.\.\.basePayload, screenshotPath \} : basePayload\);\s*\}/,
'annotated generation should dispatch exactly after capture and upload resolve',
);
});
it('saves copy edits to the staged buffer with rich AI context', () => {
assert.doesNotMatch(
SOURCE,
@@ -334,7 +285,7 @@ describe('live-browser source contracts', () => {
assert.match(SOURCE, /sendEvent\(\{ type: 'discard', id: currentSessionId \}, \{ throwOnError: true \}\)/);
});
it('releases the foreground picker after deterministic accept while carbonize finishes', () => {
it('waits for post-carbonize completion before final accepted DOM cleanup', () => {
assert.match(
SOURCE,
/let pendingAcceptedSession = null;/,
@@ -358,8 +309,8 @@ describe('live-browser source contracts', () => {
const agentDoneStart = SOURCE.indexOf("case 'agent_done':");
const errorCaseStart = SOURCE.indexOf("case 'error':", agentDoneStart);
const agentDoneSource = SOURCE.slice(agentDoneStart, errorCaseStart);
assert.match(agentDoneSource, /must not hold the foreground picker hostage/);
assert.match(agentDoneSource, /maybeCompleteAcceptedSession\(msg\)/);
assert.match(agentDoneSource, /Carbonize accepts are not terminal/);
assert.match(agentDoneSource, /break;/);
assert.match(
SOURCE,
/function handleGo\(\)[\s\S]{0,900}?pendingAcceptedSession = null;[\s\S]{0,80}?currentSessionId = id8\(\);/,
@@ -368,15 +319,15 @@ describe('live-browser source contracts', () => {
const handleAcceptStart = SOURCE.indexOf('function handleAccept()');
const maybeCompleteStart = SOURCE.indexOf('function maybeCompleteAcceptedSession', handleAcceptStart);
const handleAcceptSource = SOURCE.slice(handleAcceptStart, maybeCompleteStart);
assert.match(
assert.doesNotMatch(
handleAcceptSource,
/sendEvent\(acceptPayload, \{ throwOnError: true \}\)[\s\S]*?markSessionHandled\(\);[\s\S]*?setLiveState\('CONFIRMED'\);[\s\S]*?scheduleAcceptCleanup\(pending\);/,
'durable accept intent should release the foreground picker before background source cleanup completes',
/state = 'CONFIRMED'|cleanupAcceptedSession\(|hideBar\(\)/,
'accept enqueue should not clear or confirm the browser session before source cleanup completes',
);
assert.match(
SOURCE,
/function scheduleAcceptCleanup\(accepted\)[\s\S]*?queueMicrotask\(function\(\) \{[\s\S]*?cleanupAcceptedSession\(\);[\s\S]*?setTimeout\(function\(\) \{[\s\S]*?ensureAcceptedDomClean\(accepted\);[\s\S]*?\}, 1200\);/,
'foreground cleanup should be immediate while the no-HMR DOM fallback stays deferred',
/function scheduleAcceptCleanup\(accepted\)[\s\S]*?acceptedDomAlreadyClean\(accepted\)[\s\S]*?setTimeout\(function\(\) \{[\s\S]*?ensureAcceptedDomClean\(accepted\);[\s\S]*?cleanupAcceptedSession\(\);[\s\S]*?\}, 1800\);/,
'post-cleanup fallback should give HMR a second chance before mutating React-owned DOM',
);
assert.match(
SOURCE,
@@ -442,12 +393,4 @@ describe('live-browser source contracts', () => {
'source fallback should translate simple JSX style objects such as display:none',
);
});
it('loads progressive source checkpoints through the no-HMR fallback', () => {
assert.match(
SOURCE,
/case 'variant_progress':[\s\S]{0,1400}?msg\.previewMode === 'source'[\s\S]{0,1000}?arrivedVariants >= targetArrived[\s\S]{0,260}?injectVariantsFromSource\(msg\.previewFile \|\| msg\.file, msg\.id\)/,
'source-mode progress should let framework HMR settle before using the no-HMR fallback',
);
});
});
-435
View File
@@ -1,435 +0,0 @@
import assert from 'node:assert/strict';
import { EventEmitter } from 'node:events';
import { PassThrough, Writable } from 'node:stream';
import { describe, it } from 'node:test';
import {
CodexAppServerClient,
CodexAppServerError,
selectFastCodexModel,
selectLowestReasoningEffort,
selectQualityCodexModel,
} from '../skill/scripts/live/codex-app-server-client.mjs';
class FakeChild extends EventEmitter {
constructor(onMessage) {
super();
this.stdout = new PassThrough();
this.stderr = new PassThrough();
this.messages = [];
this.killedWith = null;
this.stdinEnded = false;
let buffer = '';
this.stdin = new Writable({
write: (chunk, _encoding, callback) => {
buffer += String(chunk);
let newline;
while ((newline = buffer.indexOf('\n')) !== -1) {
const line = buffer.slice(0, newline).trim();
buffer = buffer.slice(newline + 1);
if (line) {
const message = JSON.parse(line);
this.messages.push(message);
onMessage?.(message, this);
}
}
callback();
},
final: (callback) => {
this.stdinEnded = true;
callback();
},
});
}
send(message) {
this.stdout.write(`${JSON.stringify(message)}\n`);
}
sendRaw(text) {
this.stdout.write(text);
}
respond(request, result) {
this.send({ id: request.id, result });
}
fail(request, error) {
this.send({ id: request.id, error });
}
kill(signal) {
this.killedWith = signal;
queueMicrotask(() => this.emit('exit', null, signal));
return true;
}
}
function createHarness(handler = () => {}) {
const children = [];
const spawnCalls = [];
const spawnFactory = (command, args, options) => {
spawnCalls.push({ command, args, options });
const child = new FakeChild((message, process) => {
if (message.method === 'initialize' && message.id !== undefined) {
process.respond(message, { userAgent: 'fake-app-server' });
return;
}
handler(message, process, children.length);
});
children.push(child);
return child;
};
return { children, spawnCalls, spawnFactory };
}
function makeClient(harness, options = {}) {
let now = 0;
return new CodexAppServerClient({
command: '/fake/codex',
cwd: '/workspace',
spawnFactory: harness.spawnFactory,
clock: () => ++now,
requestTimeoutMs: 1_000,
turnTimeoutMs: 1_000,
...options,
});
}
async function connectClient(handler, options) {
const harness = createHarness(handler);
const client = makeClient(harness, options);
await client.connect();
return { client, harness, child: harness.children[0] };
}
describe('Codex app-server model selection', () => {
it('prefers visible Codex Spark, then Codex mini, other mini, and the default', () => {
const defaultModel = { id: 'gpt-5', isDefault: true };
const otherMini = { id: 'gpt-5-mini' };
const codexMini = { id: 'gpt-5-codex-mini' };
const spark = { id: 'gpt-5.3-codex-spark' };
assert.equal(selectFastCodexModel([
{ ...spark, hidden: true }, defaultModel, otherMini, codexMini, spark,
]), spark);
assert.equal(selectFastCodexModel([defaultModel, otherMini, codexMini]), codexMini);
assert.equal(selectFastCodexModel([defaultModel, otherMini]), otherMini);
assert.equal(selectFastCodexModel([defaultModel]), defaultModel);
assert.equal(selectFastCodexModel([{ id: 'first' }]).id, 'first');
assert.equal(selectFastCodexModel([]), null);
});
it('chooses none, minimal, or low before the catalog fallback', () => {
assert.equal(selectLowestReasoningEffort({
supportedReasoningEfforts: [{ reasoningEffort: 'high' }, { reasoningEffort: 'none' }],
}), 'none');
assert.equal(selectLowestReasoningEffort({
supportedReasoningEfforts: ['high', 'minimal', 'low'],
}), 'minimal');
assert.equal(selectLowestReasoningEffort({
supportedReasoningEfforts: [{ reasoningEffort: 'medium' }],
defaultReasoningEffort: 'medium',
}), 'medium');
assert.equal(selectLowestReasoningEffort({}), 'low');
});
it('prefers the visible 5.6 Sol model for design-sensitive generation', () => {
const spark = { id: 'gpt-5.3-codex-spark', isDefault: true };
const mini = { id: 'gpt-5.4-mini' };
const sol = { id: 'gpt-5.6-sol' };
assert.equal(selectQualityCodexModel([spark, mini, sol]), sol);
assert.equal(selectQualityCodexModel([{ ...sol, hidden: true }, spark, { id: 'gpt-5.5' }]).id, 'gpt-5.5');
assert.equal(selectQualityCodexModel([]), null);
});
});
describe('Codex app-server transport', () => {
it('spawns stdio JSONL and completes initialize/initialized exactly once', async () => {
const { client, harness, child } = await connectClient();
assert.equal(client.connected, true);
assert.deepEqual(harness.spawnCalls[0], {
command: '/fake/codex',
args: ['app-server', '--stdio'],
options: {
cwd: '/workspace',
env: process.env,
stdio: ['pipe', 'pipe', 'pipe'],
},
});
assert.equal(child.messages[0].method, 'initialize');
assert.equal(child.messages[0].params.clientInfo.name, 'impeccable_live');
assert.deepEqual(child.messages[1], { method: 'initialized', params: {} });
assert.equal(client.initializeResult.userAgent, 'fake-app-server');
assert.equal(client.startupMs > 0, true);
await client.connect();
assert.equal(harness.children.length, 1);
await client.close();
});
it('maps out-of-order responses, exposes notifications, and isolates listener errors', async () => {
const pending = [];
const { client, child } = await connectClient((message, process) => {
if (message.method === 'first' || message.method === 'second') {
pending.push(message);
if (pending.length === 2) {
process.respond(pending[1], { value: 2 });
process.respond(pending[0], { value: 1 });
}
}
});
const notifications = [];
client.onNotification('turn/started', () => { throw new Error('consumer failure'); });
const unsubscribe = client.onNotification('turn/started', (notification) => {
notifications.push(notification);
});
const [first, second] = await Promise.all([
client.request('first'),
client.request('second'),
]);
child.send({ method: 'turn/started', params: { threadId: 't1' } });
child.sendRaw('{not valid json}\n');
await new Promise((resolve) => setImmediate(resolve));
assert.deepEqual(first, { value: 1 });
assert.deepEqual(second, { value: 2 });
assert.equal(notifications.length, 1);
assert.equal(typeof notifications[0].receivedAt, 'number');
unsubscribe();
await client.close();
});
it('lists models and surfaces structured request errors', async () => {
const models = [{ id: 'gpt-5.3-codex-spark', hidden: false }];
const { client } = await connectClient((message, process) => {
if (message.method === 'model/list') process.respond(message, { data: models });
if (message.method === 'explode') {
process.fail(message, { code: -32_000, message: 'bad request', data: { retry: false } });
}
});
assert.deepEqual(await client.listModels(), models);
assert.equal((await client.selectFastModel()).id, models[0].id);
await assert.rejects(client.request('explode'), (error) => {
assert.equal(error instanceof CodexAppServerError, true);
assert.equal(error.code, -32_000);
assert.deepEqual(error.data, { retry: false });
return true;
});
await client.close();
});
it('rejects every pending request immediately when the process exits', async () => {
const { client, child } = await connectClient();
const first = client.request('never-returns');
const second = client.request('also-never-returns');
child.emit('exit', 17, null);
await assert.rejects(first, /exited with code 17/);
await assert.rejects(second, /exited with code 17/);
assert.equal(client.connected, false);
assert.equal(client.lastExit.code, 17);
});
});
describe('dedicated Codex worker threads', () => {
it('starts and resumes only explicit dedicated thread IDs, with no discovery request', async () => {
const methods = [];
const { client } = await connectClient((message, process) => {
methods.push(message.method);
if (message.method === 'thread/start') {
process.respond(message, { thread: { id: 'live-worker-1', ephemeral: false } });
}
if (message.method === 'thread/resume') {
process.respond(message, { thread: { id: message.params.threadId } });
}
});
await assert.rejects(
client.startTurn({ threadId: 'desktop-thread', input: 'work' }),
/not owned by this client/,
);
await assert.rejects(client.resumeDedicatedThread('', {}), /non-empty string/);
await assert.rejects(
client.resumeDedicatedThread('live-worker-1', { path: '/desktop/rollout' }),
/only be resumed by explicit threadId/,
);
const started = await client.startDedicatedThread({
cwd: '/workspace',
serviceName: 'impeccable_live_worker',
ephemeral: false,
});
assert.equal(started.id, 'live-worker-1');
const resumed = await client.resumeDedicatedThread('live-worker-1', { cwd: '/workspace' });
assert.equal(resumed.id, 'live-worker-1');
assert.deepEqual(client.dedicatedThreadIds, ['live-worker-1']);
assert.equal(methods.includes('thread/list'), false);
assert.equal(methods.includes('thread/read'), false);
await client.close();
});
it('collects early and late agent messages through turn completion', async () => {
const { client } = await connectClient((message, process) => {
if (message.method === 'thread/start') {
process.respond(message, { thread: { id: 'worker' } });
}
if (message.method === 'turn/start') {
const common = { threadId: 'worker', turnId: 'turn-1' };
process.send({
method: 'turn/started',
params: { threadId: 'worker', turn: { id: 'turn-1', status: 'inProgress' } },
});
process.send({
method: 'item/completed',
params: { ...common, item: { type: 'agentMessage', text: 'first fragment' } },
});
process.respond(message, { turn: { id: 'turn-1', status: 'inProgress' } });
queueMicrotask(() => {
process.send({
method: 'item/completed',
params: { ...common, item: { type: 'agentMessage', text: 'final answer' } },
});
process.send({
method: 'thread/tokenUsage/updated',
params: {
...common,
tokenUsage: {
last: { inputTokens: 100, cachedInputTokens: 60, outputTokens: 20, reasoningOutputTokens: 4, totalTokens: 124 },
total: { inputTokens: 100, cachedInputTokens: 60, outputTokens: 20, reasoningOutputTokens: 4, totalTokens: 124 },
},
},
});
process.send({
method: 'turn/completed',
params: { threadId: 'worker', turn: { id: 'turn-1', status: 'completed' } },
});
});
}
});
await client.startDedicatedThread({ serviceName: 'impeccable_live_worker' });
let startedTurnId = null;
const deliveredMessages = [];
const result = await client.startTurn({
threadId: 'worker',
input: 'Reply exactly',
model: 'gpt-5.3-codex-spark',
effort: 'low',
onStarted: (turnId) => { startedTurnId = turnId; },
onAgentMessage: async (message) => { deliveredMessages.push(message); },
});
assert.equal(startedTurnId, 'turn-1');
assert.equal(result.turnId, 'turn-1');
assert.equal(result.status, 'completed');
assert.deepEqual(result.agentMessages, ['first fragment', 'final answer']);
assert.deepEqual(deliveredMessages, ['first fragment', 'final answer']);
assert.equal(result.message, 'final answer');
assert.equal(result.firstAgentMessageMs >= 0, true);
assert.equal(result.tokenUsage.last.inputTokens, 100);
assert.equal(result.started.method, 'turn/started');
assert.equal(result.durationMs > 0, true);
await client.close();
});
it('rejects failed turn completions instead of treating them as usable output', async () => {
const { client } = await connectClient((message, process) => {
if (message.method === 'thread/start') {
process.respond(message, { thread: { id: 'worker' } });
}
if (message.method === 'turn/start') {
process.respond(message, { turn: { id: 'turn-failed', status: 'inProgress' } });
queueMicrotask(() => {
process.send({
method: 'turn/completed',
params: {
threadId: 'worker',
turn: { id: 'turn-failed', status: 'failed', error: { message: 'model unavailable' } },
},
});
});
}
});
await client.startDedicatedThread({ serviceName: 'impeccable_live_worker' });
await assert.rejects(
client.startTurn({ threadId: 'worker', input: 'work' }),
(error) => error.code === 'TURN_FAILED' && /status failed/.test(error.message),
);
await client.close();
});
it('interrupts, unsubscribes, archives, and cleanly closes', async () => {
const methods = [];
const { client, child } = await connectClient((message, process) => {
methods.push(message.method);
if (message.method === 'thread/start') process.respond(message, { thread: { id: 'worker' } });
if (message.method === 'turn/interrupt') process.respond(message, {});
if (message.method === 'thread/unsubscribe') {
process.respond(message, { status: 'unsubscribed' });
}
if (message.method === 'thread/archive') process.respond(message, {});
});
await client.startDedicatedThread({ serviceName: 'impeccable_live_worker' });
await client.interruptTurn('worker', 'turn-1');
assert.deepEqual(await client.unsubscribeThread('worker'), { status: 'unsubscribed' });
await client.close({ threadId: 'worker', archive: true });
assert.equal(methods.includes('turn/interrupt'), true);
assert.equal(methods.includes('thread/unsubscribe'), true);
assert.equal(methods.includes('thread/archive'), true);
assert.equal(child.stdinEnded, true);
assert.equal(child.killedWith, 'SIGTERM');
assert.equal(client.connected, false);
assert.deepEqual(client.dedicatedThreadIds, []);
});
it('reconnects to a new process and explicitly resumes the dedicated worker', async () => {
const methods = [];
const harness = createHarness((message, process, childCount) => {
methods.push({ method: message.method, childCount });
if (message.method === 'thread/start') {
process.respond(message, { thread: { id: 'worker' } });
}
if (message.method === 'thread/resume') {
process.respond(message, { thread: { id: message.params.threadId } });
}
});
const client = makeClient(harness);
await client.connect();
await client.startDedicatedThread({ serviceName: 'impeccable_live_worker' });
const resumed = await client.reconnect({
threadId: 'worker',
resumeParams: { cwd: '/workspace' },
});
assert.equal(harness.children.length, 2);
assert.equal(resumed.id, 'worker');
assert.equal(client.connectionGeneration, 2);
assert.equal(methods.some((entry) => entry.method === 'thread/resume' && entry.childCount === 2), true);
await client.close();
});
it('rejects an in-flight turn when the transport exits', async () => {
const { client, child } = await connectClient((message, process) => {
if (message.method === 'thread/start') process.respond(message, { thread: { id: 'worker' } });
if (message.method === 'turn/start') {
process.respond(message, { turn: { id: 'turn-1', status: 'inProgress' } });
}
});
await client.startDedicatedThread({ serviceName: 'impeccable_live_worker' });
const turn = client.startTurn({ threadId: 'worker', input: 'work' });
await new Promise((resolve) => setImmediate(resolve));
child.emit('exit', 9, null);
await assert.rejects(turn, /exited with code 9/);
});
});
-44
View File
@@ -1,44 +0,0 @@
import assert from 'node:assert/strict';
import { EventEmitter } from 'node:events';
import { PassThrough } from 'node:stream';
import { describe, it } from 'node:test';
import {
runCodexExecBenchmark,
summarizeArchitectureRuns,
} from '../scripts/lib/codex-exec-benchmark.mjs';
describe('direct Codex architecture benchmark', () => {
it('records JSONL lifecycle and token events from codex exec', async () => {
const child = new EventEmitter();
child.stdout = new PassThrough();
child.stderr = new PassThrough();
child.kill = () => true;
const resultPromise = runCodexExecBenchmark({
args: ['exec', '--json', 'work'],
spawnFactory: () => child,
});
child.stdout.write('{"type":"thread.started","thread_id":"one"}\n');
child.stdout.write('{"type":"turn.started"}\n');
child.stdout.write('{"type":"item.completed","item":{"type":"agent_message","text":"done"}}\n');
child.stdout.write('{"type":"turn.completed","usage":{"input_tokens":100,"cached_input_tokens":60,"output_tokens":20}}\n');
child.emit('exit', 0, null);
const result = await resultPromise;
assert.equal(result.events.length, 4);
assert.equal(result.usage.input_tokens, 100);
assert.ok(result.threadStartedMs >= 0);
assert.ok(result.firstAgentMessageMs >= result.turnStartedMs);
});
it('summarizes startup, generation, total, quality, and token medians', () => {
const summary = summarizeArchitectureRuns([
{ passed: true, startupMs: 10, generationMs: 100, firstUsableMs: 110, totalMs: 110, usage: { input_tokens: 1000, cached_input_tokens: 500, output_tokens: 100 } },
{ passed: false, startupMs: 20, generationMs: 200, firstUsableMs: 220, totalMs: 220, usage: { input_tokens: 2000, cached_input_tokens: 1000, output_tokens: 200 } },
]);
assert.equal(summary.runs, 2);
assert.equal(summary.passed, 1);
assert.equal(summary.medianTotalMs, 165);
assert.equal(summary.medianFirstUsableMs, 165);
assert.equal(summary.medianInputTokens, 1500);
});
});
@@ -1,89 +0,0 @@
import assert from 'node:assert/strict';
import path from 'node:path';
import { describe, it } from 'node:test';
import {
buildCodexQualityPrompt,
createCodexQualityTasks,
parseJudgeResult,
scoreCodexQualityOutput,
summarizeCodexQualityRuns,
} from '../scripts/lib/live-codex-quality-benchmark.mjs';
const repoRoot = path.resolve(import.meta.dirname, '..');
const tasks = createCodexQualityTasks({ repoRoot });
describe('Codex Live quality benchmark', () => {
it('covers full bolder and polish tasks with project and action context', () => {
assert.deepEqual(tasks.map((task) => `${task.action}:${task.id}`), [
'bolder:editorial-bolder',
'polish:operations-polish',
'polish:operations-annotated',
]);
const prompt = buildCodexQualityPrompt(tasks[0], { actionReference: 'BOLDER', fullContext: true });
assert.match(prompt, /Impeccable skill is attached/);
assert.match(prompt, /<product_context>/);
assert.match(prompt, /<design_context>/);
assert.match(prompt, /BOLDER/);
assert.match(prompt, /src\/App\.jsx/);
assert.equal(tasks[2].annotation.strokes, 1);
});
it('rejects no-op, contract-breaking, and design-system-drifting output', () => {
const task = tasks[0];
const noOp = { files: Object.entries(task.files).map(([filePath, content]) => ({ path: filePath, content })) };
assert.equal(scoreCodexQualityOutput(task, noOp).passed, false);
const cssOnly = {
files: [
{ path: 'src/App.jsx', content: task.files['src/App.jsx'] },
{ path: 'src/styles.css', content: `${task.files['src/styles.css']}\n.offer-card { min-height: 30rem; }` },
],
};
assert.equal(scoreCodexQualityOutput(task, cssOnly).passed, true, 'CSS-only design work is a material implementation change');
const splitVisibleCopy = {
files: [
{ path: 'src/App.jsx', content: task.files['src/App.jsx'].replace('Field Notes', '<span>Field</span> <span>Notes</span>') },
{ path: 'src/styles.css', content: `${task.files['src/styles.css']}\n.offer-card { min-height: 30rem; }` },
],
};
assert.equal(scoreCodexQualityOutput(task, splitVisibleCopy).checks.copyPreserved, true);
const drift = {
files: [
{ path: 'src/App.jsx', content: task.files['src/App.jsx'].replace('Field Notes', 'Neon Notes') },
{ path: 'src/styles.css', content: `${task.files['src/styles.css']}\n.offer-card { background: linear-gradient(red, blue); }` },
],
};
const score = scoreCodexQualityOutput(task, drift);
assert.equal(score.checks.copyPreserved, false);
assert.equal(score.checks.noForbiddenDrift, false);
});
it('allows an annotation-scoped semantic risk rail but still rejects decorative shadows', () => {
const task = tasks[2];
const withRiskRail = {
files: [
{ path: 'src/App.jsx', content: task.files['src/App.jsx'] },
{ path: 'src/styles.css', content: `${task.files['src/styles.css']}\n.metric--warning { box-shadow: inset 0.1875rem 0 0 var(--warning); }` },
],
};
assert.equal(scoreCodexQualityOutput(task, withRiskRail).checks.noForbiddenDrift, true);
withRiskRail.files[1].content += '\n.queue { box-shadow: 0 1rem 3rem rgb(0 0 0 / 0.2); }';
assert.equal(scoreCodexQualityOutput(task, withRiskRail).checks.noForbiddenDrift, false);
});
it('parses strict judge results and summarizes latency and quality', () => {
const judge = parseJudgeResult('{"commandFidelity":8,"brandAndSystemFidelity":9,"frontendQuality":7,"taskCompletion":8,"criticalFailure":false,"summary":"Good."}');
assert.equal(judge.passed, true);
const summary = summarizeCodexQualityRuns([
{ durationMs: 100, passed: true, judge },
{ durationMs: 300, passed: false, judge: { ...judge, frontendQuality: 6 } },
]);
assert.equal(summary.medianDurationMs, 200);
assert.equal(summary.passed, 1);
assert.equal(summary.averageJudgeScores.frontendQuality, 6.5);
});
});
-830
View File
@@ -1,830 +0,0 @@
import assert from 'node:assert/strict';
import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { describe, it } from 'node:test';
import { CODEX_WORKER_OWNER } from '../skill/scripts/live/codex-worker.mjs';
import {
CODEX_WORKER_EVENT_LEASE_MS,
CODEX_WORKER_EVENT_TYPES,
CodexLiveWorkerSupervisor,
buildDetectorRepairPrompt,
buildDeterministicScaffoldCommand,
resolveDetectorFindingWaivers,
} from '../skill/scripts/live/codex-worker-supervisor.mjs';
import { createLiveSessionStore } from '../skill/scripts/live/session-store.mjs';
import { selectAvailablePendingEvent } from '../skill/scripts/live/poll-lanes.mjs';
describe('Codex Live worker supervisor ownership and lifecycle', () => {
it('partitions worker and foreground control events without overlapping leases', () => {
const entries = [
{ event: { type: 'steer' }, leaseUntil: 0, seq: 1 },
{ event: { type: 'generate' }, leaseUntil: 0, seq: 2 },
{ event: { type: 'manual_edit_apply' }, leaseUntil: 0, seq: 3 },
{ event: { type: 'accept' }, leaseUntil: 0, seq: 4 },
{ event: { type: 'carbonize_cleanup' }, leaseUntil: 0, seq: 5 },
{ event: { type: 'exit' }, leaseUntil: 0, seq: 6 },
];
assert.equal(selectAvailablePendingEvent(entries, { types: CODEX_WORKER_EVENT_TYPES }).event.type, 'accept');
assert.equal(selectAvailablePendingEvent(entries, {
types: ['steer', 'manual_edit_apply', 'carbonize_cleanup', 'exit'],
}).event.type, 'exit');
assert.equal(CODEX_WORKER_EVENT_TYPES.includes('steer'), false);
assert.equal(CODEX_WORKER_EVENT_TYPES.includes('manual_edit_apply'), false);
assert.equal(CODEX_WORKER_EVENT_TYPES.includes('carbonize_cleanup'), false);
assert.equal(CODEX_WORKER_EVENT_TYPES.includes('exit'), false);
});
it('builds the same deterministic wrap/insert target contract as foreground Live', () => {
const replace = buildDeterministicScaffoldCommand({
id: 'abc12345',
count: 3,
element: { id: 'hero', classes: ['hero', 'title'], tagName: 'H1', textContent: ' Exact hero copy ' },
}, '/scripts');
assert.equal(replace.script, '/scripts/live-wrap.mjs');
assert.deepEqual(replace.args, [
'--id', 'abc12345', '--count', '3', '--isolated', '--element-id', 'hero',
'--classes', 'hero,title', '--tag', 'h1', '--text', 'Exact hero copy',
]);
const insert = buildDeterministicScaffoldCommand({
id: 'abc12346',
count: 2,
mode: 'insert',
insert: { position: 'before', anchor: { tag: 'section', text: 'Anchor' } },
}, '/scripts');
assert.equal(insert.script, '/scripts/live-insert.mjs');
assert.deepEqual(insert.args, [
'--id', 'abc12346', '--count', '2', '--position', 'before',
'--tag', 'section', '--query', 'Anchor', '--text', 'Anchor',
]);
});
it('never resumes a desktop or otherwise unowned thread record', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-owner-'));
const statePath = path.join(cwd, '.impeccable/live/codex-worker.json');
mkdirSync(path.dirname(statePath), { recursive: true });
writeFileSync(statePath, JSON.stringify({ owner: 'desktop', cwd, threadId: 'desktop-thread' }));
const client = fakeClient();
const supervisor = createSupervisor({ cwd, statePath, client });
await supervisor.initialize();
assert.equal(client.calls.resumeDedicatedThread.length, 0);
assert.equal(client.calls.startDedicatedThread.length, 1);
assert.equal(client.calls.startDedicatedThread[0].ephemeral, false);
assert.equal(client.calls.startDedicatedThread[0].sandbox, 'read-only');
await supervisor.shutdown();
});
it('resumes only a durable Live-owned worker thread', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-resume-'));
const statePath = path.join(cwd, '.impeccable/live/codex-worker.json');
mkdirSync(path.dirname(statePath), { recursive: true });
writeFileSync(statePath, JSON.stringify({
owner: CODEX_WORKER_OWNER,
cwd,
threadId: 'live-worker-thread',
status: 'ready',
threadPrimed: true,
}));
const client = fakeClient();
const supervisor = createSupervisor({ cwd, statePath, client });
await supervisor.initialize();
assert.equal(client.calls.resumeDedicatedThread.length, 1);
assert.equal(client.calls.resumeDedicatedThread[0].threadId, 'live-worker-thread');
assert.equal(client.calls.startDedicatedThread.length, 0);
assert.equal(supervisor.threadPrimed, true, 'a resumed thread must not receive the skill attachment again');
assert.equal(JSON.parse(readFileSync(statePath, 'utf-8')).threadPrimed, true);
await supervisor.shutdown();
});
it('interrupts the active dedicated turn on early Accept or Discard', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-interrupt-'));
const client = fakeClient();
const supervisor = createSupervisor({
cwd,
statePath: path.join(cwd, 'state.json'),
client,
});
supervisor.thread = { id: 'live-worker-thread' };
supervisor.model = client.models[0];
supervisor.active = { eventId: 'generation-1', turnId: 'turn-1' };
await supervisor.cancelActive('accept', 'generation-1');
assert.deepEqual(client.calls.interruptTurn, [{ threadId: 'live-worker-thread', turnId: 'turn-1' }]);
assert.equal(supervisor.canceled.has('generation-1'), true);
});
it('does not block deterministic Accept on the app-server interrupt round trip', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-fast-accept-'));
const client = fakeClient();
let releaseInterrupt;
const interruptReleased = new Promise((resolve) => { releaseInterrupt = resolve; });
client.interruptTurn = async (threadId, turnId) => {
client.calls.interruptTurn.push({ threadId, turnId });
await interruptReleased;
};
let acceptStarted = false;
const events = [
{ type: 'accept', id: 'generation-1', variantId: '1' },
{ type: 'exit' },
];
const supervisor = new CodexLiveWorkerSupervisor({
cwd,
base: 'http://localhost:1',
token: 'token',
client,
config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
statePath: path.join(cwd, 'state.json'),
scriptsDir: path.join(cwd, 'skill/scripts'),
fetchEvent: async () => events.shift(),
handleAccept: async () => {
acceptStarted = true;
releaseInterrupt();
return { _acceptResult: { handled: true, carbonize: false } };
},
});
supervisor.thread = { id: 'live-worker-thread' };
supervisor.model = client.models[0];
supervisor.active = { eventId: 'generation-1', turnId: 'turn-1', threadId: 'live-worker-thread' };
await supervisor.run();
assert.equal(acceptStarted, true);
assert.equal(client.calls.interruptTurn.length >= 1, true);
});
it('rotates a busy thread so the next generation does not wait for the canceled tail', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-tail-rotation-'));
const client = fakeClient();
client.startDedicatedThread = async (params) => {
client.calls.startDedicatedThread.push(params);
await new Promise((resolve) => setImmediate(resolve));
return { id: 'replacement-live-thread' };
};
const supervisor = createSupervisor({ cwd, statePath: path.join(cwd, 'state.json'), client });
supervisor.model = client.models[0];
supervisor.thread = { id: 'draining-live-thread' };
let releaseDrainingQueue;
let drainingQueueFinished = false;
supervisor.queue = new Promise((resolve) => {
releaseDrainingQueue = () => {
drainingQueueFinished = true;
resolve();
};
});
let observedThread = null;
supervisor.runGenerationPhase = async () => {
observedThread = supervisor.thread.id;
};
supervisor.reply = async () => {};
supervisor.rotateWorkerThread('accept');
await supervisor.processGeneration({
type: 'generate',
id: 'next-generation',
count: 1,
scaffold: { file: 'src/App.jsx' },
});
assert.equal(observedThread, 'replacement-live-thread');
assert.equal(drainingQueueFinished, false, 'the next generation must not join the canceled tail queue');
releaseDrainingQueue();
await new Promise((resolve) => setImmediate(resolve));
assert.deepEqual(client.calls.archiveThread, [{ threadId: 'draining-live-thread' }]);
});
it('interrupts a canceled turn whose id arrives after Accept', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-late-turn-'));
const client = fakeClient();
const supervisor = createSupervisor({ cwd, statePath: path.join(cwd, 'state.json'), client });
supervisor.thread = { id: 'live-worker-thread' };
supervisor.model = client.models[0];
supervisor.active = { eventId: 'generation-1', turnId: null };
supervisor.canceled.add('generation-1');
client.startTurn = async ({ onStarted }) => {
onStarted('late-turn');
await new Promise((resolve) => setImmediate(resolve));
return { message: '{"files":[]}' };
};
await supervisor.runTurnWithReconnect({ input: 'work', outputSchema: {} });
assert.deepEqual(client.calls.interruptTurn, [{ threadId: 'live-worker-thread', turnId: 'late-turn' }]);
});
it('queues carbonize cleanup onto the foreground control lane', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-carbonize-'));
const cleanups = [];
const order = [];
const client = fakeClient();
const supervisor = new CodexLiveWorkerSupervisor({
cwd,
base: 'http://localhost:1',
token: 'token',
client,
config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
statePath: path.join(cwd, 'state.json'),
scriptsDir: path.join(cwd, 'skill/scripts'),
handleAccept: async (event) => ({
...event,
_acceptResult: { handled: true, carbonize: true, file: 'src/App.jsx' },
_completionAck: { ok: false, deferred: true },
}),
postCleanup: async (_base, _token, event) => { cleanups.push(event); order.push('cleanup'); },
completeAccept: async () => { order.push('accept_ack'); },
});
supervisor.running = true;
supervisor.thread = { id: 'live-worker-thread' };
supervisor.fetchEvent = async (_base, _token, options) => {
assert.deepEqual(options.types, CODEX_WORKER_EVENT_TYPES);
assert.equal(options.leaseMs, CODEX_WORKER_EVENT_LEASE_MS);
return cleanups.length === 0
? { type: 'accept', id: 'abc12345', variantId: '1' }
: { type: 'exit' };
};
await supervisor.run();
assert.deepEqual(cleanups, [{
id: 'abc12345',
sessionId: 'abc12345',
file: 'src/App.jsx',
variantId: '1',
acceptResult: { handled: true, carbonize: true, file: 'src/App.jsx' },
}]);
assert.deepEqual(order, ['cleanup', 'accept_ack']);
});
it('renews but never queues the same long-running generation twice', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-duplicate-lease-'));
const client = fakeClient();
const supervisor = createSupervisor({
cwd,
statePath: path.join(cwd, 'state.json'),
client,
});
supervisor.thread = { id: 'live-worker-thread' };
let generations = 0;
supervisor.processGeneration = async () => {
generations += 1;
await new Promise((resolve) => setImmediate(resolve));
};
const events = [
{ type: 'generate', id: 'generation-1', count: 3 },
{ type: 'generate', id: 'generation-1', count: 3 },
{ type: 'exit' },
];
supervisor.fetchEvent = async () => events.shift();
await supervisor.run();
assert.equal(generations, 1);
assert.equal(supervisor.queuedGenerationIds.size, 0);
});
it('reconnects and resumes the owned worker once after app-server loss', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-reconnect-'));
const client = fakeClient();
let attempts = 0;
client.startTurn = async () => {
attempts += 1;
if (attempts === 1) throw new Error('app-server exited');
return { message: '{"files":[]}' };
};
const supervisor = createSupervisor({
cwd,
statePath: path.join(cwd, 'state.json'),
client,
});
supervisor.thread = { id: 'live-worker-thread' };
supervisor.model = client.models[0];
supervisor.active = { eventId: 'generation-1', turnId: null };
const result = await supervisor.runTurnWithReconnect({ input: 'work', outputSchema: {} });
assert.equal(result.answer, '{"files":[]}');
assert.equal(client.calls.reconnect, 1);
assert.equal(client.calls.resumeDedicatedThread.length, 1);
});
it('relinquishes Generate and advertises foreground fallback after permanent failure', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-fallback-'));
const replies = [];
const statePath = path.join(cwd, 'state.json');
const supervisor = new CodexLiveWorkerSupervisor({
cwd,
base: 'http://localhost:1',
token: 'token',
client: fakeClient(),
config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
statePath,
scriptsDir: path.join(cwd, 'skill/scripts'),
reply: async (_base, _token, value) => { replies.push(value); },
});
supervisor.running = true;
supervisor.pollAbortController = new AbortController();
await supervisor.handleGenerationFailure(
{ type: 'generate', id: 'recoverable-generation' },
new Error('app-server remained unavailable after reconnect'),
);
assert.equal(supervisor.running, false);
assert.equal(supervisor.pollAbortController.signal.aborted, true);
assert.deepEqual(replies, [{
id: 'recoverable-generation',
type: 'retry',
sourceEventType: 'generate',
}]);
const state = JSON.parse(readFileSync(statePath, 'utf-8'));
assert.equal(state.status, 'failed');
assert.equal(state.eventId, 'recoverable-generation');
assert.match(state.error, /app-server remained unavailable/);
});
it('re-prepares once when foreground cleanup changes source during generation', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-source-race-'));
const supervisor = createSupervisor({
cwd,
statePath: path.join(cwd, 'state.json'),
client: fakeClient(),
});
let attempts = 0;
supervisor.runGenerationPhaseOnce = async () => {
attempts += 1;
if (attempts === 1) {
const error = new Error('publish_source_hash_mismatch');
error.code = 'publish_source_hash_mismatch';
throw error;
}
};
await supervisor.runGenerationPhase({ id: 'source-race' }, 'first', 1);
assert.equal(attempts, 2);
});
it('repairs new detector findings on the same persistent thread before publication', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-detector-repair-'));
mkdirSync(path.join(cwd, 'src'), { recursive: true });
const sessionId = 'detectorrepair';
writeFileSync(path.join(cwd, 'src/App.jsx'), [
'<main>',
` <div data-impeccable-variants="${sessionId}" data-impeccable-variant-count="3">`,
` <style data-impeccable-css="${sessionId}"></style>`,
' <div data-impeccable-variant="original"><h1>Original</h1></div>',
` {/* impeccable-variants-end ${sessionId} */}`,
' </div>',
'</main>',
].join('\n'));
createLiveSessionStore({ cwd, sessionId }).appendEvent({
type: 'generate', id: sessionId, count: 3, generationEpoch: 1,
});
const plan = {
identityLock: ['Preserve identity'],
directions: [
{ variantId: 1, name: 'One', axis: 'hierarchy', intent: 'Strengthen hierarchy' },
{ variantId: 2, name: 'Two', axis: 'layout', intent: 'Recompose layout' },
{ variantId: 3, name: 'Three', axis: 'density', intent: 'Adjust density' },
],
};
const client = fakeClient();
const turnThreadIds = [];
const prompts = [];
const outputSchemas = [];
let turn = 0;
client.startTurn = async ({ threadId, input, outputSchema, onStarted }) => {
turn += 1;
turnThreadIds.push(threadId);
prompts.push(input.find((item) => item.type === 'text').text);
outputSchemas.push(outputSchema);
onStarted?.(`turn-${turn}`);
return { message: JSON.stringify({
sourceDelta: {
variantId: 1,
markup: `<h1>${turn === 1 ? 'Flagged' : 'Repaired'}</h1>`,
css: '@scope ([data-impeccable-variant="1"]) { h1 { color: currentColor; } }',
},
plan,
...(turn === 1 ? {} : { detectorWaivers: [] }),
}) };
};
let detectorCall = 0;
const supervisor = new CodexLiveWorkerSupervisor({
cwd,
base: 'http://localhost:1',
token: 'token',
client,
config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
statePath: path.join(cwd, '.impeccable/live/codex-worker.json'),
scriptsDir: path.resolve('skill/scripts'),
detectCandidate: () => {
detectorCall += 1;
return detectorCall === 2
? [{ antipattern: 'gradient-text', name: 'Gradient text', snippet: 'flagged candidate', file: 'App.jsx' }]
: [];
},
publishCheckpoint: async () => {},
publishPhase: async () => {},
});
supervisor.thread = { id: 'persistent-live-thread' };
supervisor.model = client.models[0];
await supervisor.runGenerationPhaseOnce({
type: 'generate',
id: sessionId,
count: 3,
scaffold: { file: 'src/App.jsx', styleMode: 'scoped' },
}, 'first', 1);
assert.deepEqual(turnThreadIds, ['persistent-live-thread', 'persistent-live-thread']);
assert.match(prompts[1], /new Impeccable detector findings/);
assert.equal(outputSchemas[0].properties.detectorWaivers, undefined);
assert.equal(outputSchemas[1].properties.detectorWaivers.type, 'array');
assert.match(readFileSync(path.join(cwd, 'src/App.jsx'), 'utf-8'), /Repaired/);
assert.doesNotMatch(readFileSync(path.join(cwd, 'src/App.jsx'), 'utf-8'), /Flagged/);
assert.equal(detectorCall, 3);
});
it('accepts only explicit narrow detector false-positive waivers', () => {
const findings = [
{
antipattern: 'gradient-text',
file: '/project/src/App.jsx',
snippet: 'intentional campaign wordmark',
ignoreValue: '',
},
{
antipattern: 'overused-font',
file: '/project/src/App.jsx',
snippet: 'font-family: Inter',
ignoreValue: 'Inter',
},
];
const resolved = resolveDetectorFindingWaivers(findings, [
{
rule: 'gradient-text',
file: 'App.jsx',
snippet: 'intentional campaign wordmark',
ignoreValue: '',
reason: 'The selected element is the established campaign wordmark.',
},
{
rule: 'overused-font',
file: 'App.jsx',
snippet: '',
ignoreValue: 'Roboto',
reason: 'Wrong value must not waive the finding.',
},
]);
assert.deepEqual(resolved.accepted.map(({ waiver }) => waiver.rule), ['gradient-text']);
assert.deepEqual(resolved.unresolved, [findings[1]]);
assert.match(buildDetectorRepairPrompt('first', findings), /Fix real defects/);
assert.match(buildDetectorRepairPrompt('first', findings), /contextually intentional or a detector false positive/);
assert.match(buildDetectorRepairPrompt('first', findings), /do not.*persist project detector config/i);
});
it('publishes a candidate when the repair turn explicitly waives the remaining false positive', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-detector-waiver-'));
mkdirSync(path.join(cwd, 'src'), { recursive: true });
const sessionId = 'detectorwaiver';
writeFileSync(path.join(cwd, 'src/App.jsx'), [
'<main>',
` <div data-impeccable-variants="${sessionId}" data-impeccable-variant-count="1">`,
` <style data-impeccable-css="${sessionId}"></style>`,
' <div data-impeccable-variant="original"><h1>Original</h1></div>',
` {/* impeccable-variants-end ${sessionId} */}`,
' </div>',
'</main>',
].join('\n'));
createLiveSessionStore({ cwd, sessionId }).appendEvent({
type: 'generate', id: sessionId, count: 1, generationEpoch: 1,
});
const finding = {
antipattern: 'gradient-text',
name: 'Gradient text',
snippet: 'intentional campaign wordmark',
ignoreValue: '',
file: 'App.jsx',
};
const client = fakeClient();
let turn = 0;
client.startTurn = async ({ onStarted }) => {
turn += 1;
onStarted?.(`turn-${turn}`);
return { message: JSON.stringify({
sourceDelta: {
variantId: 1,
markup: '<h1>Intentional wordmark</h1>',
css: '@scope ([data-impeccable-variant="1"]) { h1 { color: currentColor; } }',
},
...(turn === 1 ? {} : {
detectorWaivers: [{
rule: 'gradient-text',
file: 'App.jsx',
snippet: finding.snippet,
ignoreValue: '',
reason: 'This selected element is the established campaign wordmark.',
}],
}),
}) };
};
let detectorCall = 0;
const supervisor = new CodexLiveWorkerSupervisor({
cwd,
base: 'http://localhost:1',
token: 'token',
client,
config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
statePath: path.join(cwd, '.impeccable/live/codex-worker.json'),
scriptsDir: path.resolve('skill/scripts'),
detectCandidate: () => (++detectorCall === 1 ? [] : [finding]),
publishCheckpoint: async () => {},
publishPhase: async () => {},
});
supervisor.thread = { id: 'persistent-live-thread' };
supervisor.model = client.models[0];
await supervisor.runGenerationPhaseOnce({
type: 'generate',
id: sessionId,
count: 1,
scaffold: { file: 'src/App.jsx', styleMode: 'scoped' },
}, 'first', 1);
assert.equal(turn, 2);
assert.equal(detectorCall, 3);
assert.match(readFileSync(path.join(cwd, 'src/App.jsx'), 'utf-8'), /Intentional wordmark/);
const snapshot = createLiveSessionStore({ cwd, sessionId }).getSnapshot(sessionId, { includeCompleted: true });
assert.deepEqual(snapshot.detectorWaivers, [{
rule: 'gradient-text',
file: 'App.jsx',
snippet: finding.snippet,
ignoreValue: '',
reason: 'This selected element is the established campaign wordmark.',
}]);
});
it('resumes progressive delivery from durable variant checkpoints', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-checkpoint-resume-'));
const phases = [];
const replies = [];
const supervisor = new CodexLiveWorkerSupervisor({
cwd,
base: 'http://localhost:1',
token: 'token',
client: fakeClient(),
config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
statePath: path.join(cwd, 'state.json'),
scriptsDir: path.join(cwd, 'skill/scripts'),
reply: async (_base, _token, value) => { replies.push(value); },
});
supervisor.thread = { id: 'live-worker-thread' };
supervisor.threadReady = Promise.resolve(supervisor.thread);
supervisor.runGenerationPhase = async (_event, phase, arrivedVariants) => {
phases.push({ phase, arrivedVariants });
};
const partialId = 'resume-partial';
const store = createLiveSessionStore({ cwd, sessionId: partialId });
store.appendEvent({ type: 'generate', id: partialId, count: 3, generationEpoch: 1 });
store.appendEvent({ type: 'checkpoint', id: partialId, phase: 'cycling', revision: 1, arrivedVariants: 1 });
await supervisor.processGeneration({
type: 'generate',
id: partialId,
count: 3,
generationEpoch: 1,
scaffold: { file: 'src/App.jsx' },
});
assert.deepEqual(phases, [{ phase: 'remainder', arrivedVariants: 3 }]);
assert.equal(replies.at(-1).type, 'done');
phases.length = 0;
const completeId = 'resume-complete';
const completeStore = createLiveSessionStore({ cwd, sessionId: completeId });
completeStore.appendEvent({ type: 'generate', id: completeId, count: 3, generationEpoch: 1 });
completeStore.appendEvent({ type: 'checkpoint', id: completeId, phase: 'cycling', revision: 2, arrivedVariants: 3 });
await supervisor.processGeneration({
type: 'generate',
id: completeId,
count: 3,
generationEpoch: 1,
scaffold: { file: 'src/App.jsx' },
});
assert.deepEqual(phases, [{ phase: 'params', arrivedVariants: 3 }]);
assert.equal(replies.at(-1).id, completeId);
assert.equal(replies.at(-1).type, 'done');
});
it('archives its dedicated thread during clean Live shutdown', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-close-'));
const client = fakeClient();
const supervisor = createSupervisor({
cwd,
statePath: path.join(cwd, 'state.json'),
client,
});
supervisor.thread = { id: 'live-worker-thread' };
await supervisor.shutdown({ archive: true });
assert.deepEqual(client.calls.archiveThread, [{ threadId: 'live-worker-thread' }]);
assert.equal(client.calls.close, 1);
});
it('reports stopped rather than archived when thread archival fails', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-archive-fail-'));
const client = fakeClient();
client.archiveThread = async () => { throw new Error('archive unavailable'); };
const statePath = path.join(cwd, 'state.json');
const supervisor = createSupervisor({ cwd, statePath, client });
supervisor.thread = { id: 'live-worker-thread' };
await supervisor.shutdown({ archive: true });
const state = JSON.parse(readFileSync(statePath, 'utf-8'));
assert.equal(state.status, 'stopped');
assert.equal(state.archived, false);
});
it('treats an unused thread with no persisted rollout as already archived', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-empty-thread-'));
const statePath = path.join(cwd, 'state.json');
const client = fakeClient();
client.archiveThread = async () => { throw new Error('thread/archive: no rollout found for thread id empty'); };
const supervisor = createSupervisor({ cwd, statePath, client });
supervisor.thread = { id: 'empty' };
await supervisor.shutdown({ archive: true });
assert.equal(JSON.parse(readFileSync(statePath, 'utf-8')).status, 'archived');
});
it('publishes progressive source checkpoints only through the fenced publisher', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-publish-'));
mkdirSync(path.join(cwd, 'src'), { recursive: true });
mkdirSync(path.join(cwd, 'skill'), { recursive: true });
writeFileSync(path.join(cwd, 'PRODUCT.md'), '# Product\nStable product context');
writeFileSync(path.join(cwd, 'DESIGN.md'), '# Design\nStable design context');
writeFileSync(path.join(cwd, 'skill/SKILL.md'), '# Impeccable skill');
const sessionId = 'codexprogress';
const original = '<main><div data-impeccable-variants="codexprogress"><style data-impeccable-css="codexprogress"></style><div data-impeccable-variant="original"><h1>Original</h1></div></div></main>';
writeFileSync(path.join(cwd, 'src/App.jsx'), original);
createLiveSessionStore({ cwd, sessionId }).appendEvent({
type: 'generate',
id: sessionId,
count: 3,
generationEpoch: 1,
});
const client = fakeClient();
let turn = 0;
const prompts = [];
const turnInputs = [];
const plan = {
identityLock: ['Preserve copy and shared component roles'],
directions: [
{ variantId: 1, name: 'Hierarchy', axis: 'type scale', intent: 'Strengthen hierarchy' },
{ variantId: 2, name: 'Composition', axis: 'layout', intent: 'Recompose the root' },
{ variantId: 3, name: 'Rhythm', axis: 'spacing', intent: 'Increase rhythm' },
],
};
client.startTurn = async ({ input, onStarted, onAgentMessage }) => {
turn += 1;
turnInputs.push(input);
onStarted?.(`turn-${turn}`);
const prompt = input.find((item) => item.type === 'text').text;
prompts.push(prompt);
const message = turn === 1
? JSON.stringify({
sourceDelta: {
variantId: 1,
markup: '<h1>One</h1>',
css: '@scope ([data-impeccable-variant="1"]) { h1 { color: red; } }',
},
plan,
})
: turn === 2
? JSON.stringify({
sourceDeltas: [
{
variantId: 2,
markup: '<h1>Two</h1>',
css: '@scope ([data-impeccable-variant="2"]) { h1 { color: green; } }',
},
{
variantId: 3,
markup: '<h1>Three</h1>',
css: '@scope ([data-impeccable-variant="3"]) { h1 { color: blue; } }',
},
],
parameterCss: '',
paramsJson: '{"1":[],"2":[],"3":[]}',
})
: JSON.stringify({ parameterCss: '', paramsJson: '{"1":[],"2":[],"3":[]}' });
await Promise.all([
onAgentMessage?.(message),
onAgentMessage?.(message),
]);
return { message };
};
const replies = [];
const checkpoints = [];
const phases = [];
let checkpointAttempts = 0;
const supervisor = new CodexLiveWorkerSupervisor({
cwd,
base: 'http://localhost:1',
token: 'token',
client,
config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
statePath: path.join(cwd, '.impeccable/live/codex-worker.json'),
scriptsDir: path.join(cwd, 'skill/scripts'),
reply: async (_base, _token, value) => { replies.push(value); },
publishCheckpoint: async (_base, _token, value) => {
checkpointAttempts += 1;
if (checkpointAttempts === 1) throw new Error('transient checkpoint transport failure');
checkpoints.push(value);
},
publishPhase: async (_base, _token, value) => { phases.push(value); },
});
supervisor.thread = { id: 'live-worker-thread' };
supervisor.model = client.models[0];
await supervisor.processGeneration({
type: 'generate',
id: sessionId,
count: 3,
action: 'impeccable',
scaffold: { file: 'src/App.jsx', styleMode: 'scoped' },
});
assert.equal(checkpoints.length, 2);
assert.deepEqual(checkpoints.map((item) => item.arrivedVariants), [1, 3]);
assert.deepEqual(phases.map((item) => item.phase), [
'first_variant_generating',
'first_variant_validating',
'remaining_variants_generating',
'remaining_variants_validating',
'parameters_ready',
]);
assert.equal(replies.at(-1).type, 'done');
const publishedSource = readFileSync(path.join(cwd, 'src/App.jsx'), 'utf-8');
assert.equal((publishedSource.match(/data-impeccable-variant="1"/g) || []).length, 2, 'selector and variant 1 remain once each');
assert.match(publishedSource, /<h1>One<\/h1>/);
assert.match(publishedSource, /<h1>Two<\/h1>/);
assert.doesNotMatch(publishedSource, /Mutated/);
const snapshot = createLiveSessionStore({ cwd, sessionId }).getSnapshot(sessionId, { includeCompleted: true });
assert.equal(snapshot.arrivedVariants, 3);
assert.equal(snapshot.publishedRevision, 2);
assert.equal(snapshot.paramsPublished, true);
assert.deepEqual(snapshot.variantPlan, plan);
assert.equal(checkpointAttempts, 3, 'the durable first publication retries only its checkpoint');
assert.match(prompts[1], /"name": "Composition"/);
assert.equal(turnInputs[0].some((item) => item.type === 'skill'), true);
assert.equal(turnInputs[1].some((item) => item.type === 'skill'), false);
assert.equal(JSON.parse(readFileSync(path.join(cwd, '.impeccable/live/codex-worker.json'), 'utf-8')).threadPrimed, true);
assert.equal(client.calls.startDedicatedThread.length, 0, 'both turns stay on the existing durable thread');
assert.equal(client.calls.archiveThread.length, 0);
assert.match(prompts[0], /Stable product context/);
assert.match(prompts[0], /Stable design context/);
assert.doesNotMatch(prompts[0], /<source_neighborhood>/);
assert.doesNotMatch(prompts[1], /Stable product context|<source_neighborhood>/);
assert.match(prompts[1], /parameterCss and paramsJson/);
});
});
function createSupervisor({ cwd, statePath, client }) {
return new CodexLiveWorkerSupervisor({
cwd,
base: 'http://localhost:1',
token: 'token',
client,
config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
statePath,
scriptsDir: path.join(cwd, 'skill/scripts'),
});
}
function fakeClient() {
const calls = {
connect: 0,
listModels: 0,
startDedicatedThread: [],
resumeDedicatedThread: [],
reconnect: 0,
interruptTurn: [],
archiveThread: [],
close: 0,
};
const models = [{
id: 'gpt-5.3-codex-spark',
model: 'gpt-5.3-codex-spark',
supportedReasoningEfforts: [{ reasoningEffort: 'low' }],
}];
return {
calls,
models,
async connect() { calls.connect += 1; },
async listModels() { calls.listModels += 1; return models; },
async startDedicatedThread(params) { calls.startDedicatedThread.push(params); return { id: 'new-live-thread' }; },
async resumeDedicatedThread(threadId, params) {
calls.resumeDedicatedThread.push({ threadId, ...params });
return { id: threadId };
},
async reconnect({ threadId, resumeParams }) {
calls.reconnect += 1;
calls.resumeDedicatedThread.push({ threadId, ...resumeParams });
return { id: threadId };
},
async startTurn() { return { message: 'READY' }; },
async interruptTurn(threadId, turnId) { calls.interruptTurn.push({ threadId, turnId }); },
async archiveThread(threadId) { calls.archiveThread.push({ threadId }); },
async close() { calls.close += 1; },
};
}
-914
View File
@@ -1,914 +0,0 @@
import assert from 'node:assert/strict';
import { spawn, spawnSync } from 'node:child_process';
import { chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { describe, it } from 'node:test';
import {
CODEX_WORKER_OWNER,
applyCodexWorkerOutput,
buildCodexWorkerInstructions,
buildCodexWorkerTurnInputs,
buildGenerationTurnInput,
codexWorkerDetectorRepairSchema,
codexWorkerOutputSchemaForPhase,
codexWorkerProcessStateIsOwned,
codexWorkerStateIsOwned,
isCodexRuntime,
readPreparedArtifact,
resolveCodexExecutable,
resolveCodexWorkerConfig,
} from '../skill/scripts/live/codex-worker.mjs';
describe('Codex Live worker configuration', () => {
it('defaults off everywhere and preserves explicit Codex-only opt-ins', () => {
assert.deepEqual(resolveCodexWorkerConfig({ env: {}, liveConfig: {} }), {
enabled: false,
model: null,
codexPath: 'codex',
effort: 'medium',
profile: 'quality',
delivery: 'progressive',
maxArtifactBytes: 2_000_000,
});
assert.equal(resolveCodexWorkerConfig({
env: { IMPECCABLE_LIVE_CODEX_WORKER: '1' },
liveConfig: {},
}).enabled, true);
assert.equal(resolveCodexWorkerConfig({
env: { IMPECCABLE_LIVE_CODEX_WORKER: 'false' },
liveConfig: { experimentalCodexWorker: { enabled: true } },
}).enabled, false, 'explicit environment disable wins');
assert.equal(resolveCodexWorkerConfig({
env: {},
liveConfig: { experimentalCodexWorker: { enabled: true, delivery: 'atomic' } },
}).enabled, false, 'committed config cannot activate Codex in another harness');
assert.equal(resolveCodexWorkerConfig({ env: { CODEX_THREAD_ID: 'thread-1' } }).enabled, false);
assert.equal(resolveCodexWorkerConfig({
env: { CODEX_THREAD_ID: 'thread-1' },
liveConfig: { experimentalCodexWorker: { enabled: true } },
}).enabled, true);
assert.equal(resolveCodexWorkerConfig({
env: { CODEX_THREAD_ID: 'thread-1', IMPECCABLE_LIVE_CODEX_PROFILE: 'fast' },
}).effort, 'low');
assert.equal(resolveCodexWorkerConfig({
env: { CODEX_THREAD_ID: 'thread-1', IMPECCABLE_LIVE_CODEX_DELIVERY: 'atomic' },
}).delivery, 'atomic');
assert.equal(isCodexRuntime({ CLAUDE_CODE: '1' }), false);
assert.equal(isCodexRuntime({ GEMINI_CLI: '1' }), false);
});
it('recognizes only a Live-owned durable thread record', () => {
const cwd = '/tmp/project';
assert.equal(codexWorkerStateIsOwned({ owner: CODEX_WORKER_OWNER, cwd, threadId: 'worker-1' }, cwd), true);
assert.equal(codexWorkerStateIsOwned({ owner: 'desktop', cwd, threadId: 'desktop-1' }, cwd), false);
assert.equal(codexWorkerStateIsOwned({ owner: CODEX_WORKER_OWNER, cwd: '/tmp/other', threadId: 'worker-1' }, cwd), false);
assert.equal(codexWorkerProcessStateIsOwned({ owner: CODEX_WORKER_OWNER, cwd, pid: 123, status: 'starting' }, cwd), true);
assert.equal(codexWorkerStateIsOwned({ owner: CODEX_WORKER_OWNER, cwd, pid: 123, status: 'starting' }, cwd), false);
});
it('resolves configured Codex executables without spawning a preflight process', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-path-'));
const bin = path.join(cwd, 'bin');
mkdirSync(bin);
const executable = path.join(bin, 'codex');
writeFileSync(executable, '#!/bin/sh\nexit 0\n');
chmodSync(executable, 0o755);
assert.deepEqual(resolveCodexExecutable('./bin/codex', { cwd, env: {} }), {
available: true,
command: './bin/codex',
resolvedPath: executable,
});
assert.deepEqual(resolveCodexExecutable('codex', { cwd, env: { PATH: bin } }), {
available: true,
command: 'codex',
resolvedPath: executable,
});
assert.deepEqual(resolveCodexExecutable('missing-codex', { cwd, env: { PATH: bin } }), {
available: false,
error: 'codex_cli_unavailable',
command: 'missing-codex',
});
});
it('leaves the portable foreground path untouched when the switch is off', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-disabled-'));
const script = path.resolve('skill/scripts/live-codex-worker.mjs');
const result = spawnSync(process.execPath, [script], {
cwd,
encoding: 'utf-8',
env: { ...process.env, IMPECCABLE_LIVE_CODEX_WORKER: '0' },
});
assert.equal(result.status, 0, result.stderr);
assert.deepEqual(JSON.parse(result.stdout), {
ok: false,
error: 'codex_worker_disabled',
fallback: 'foreground',
});
});
it('reports a missing Codex CLI immediately and records actionable foreground fallback', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-missing-cli-'));
const script = path.resolve('skill/scripts/live-codex-worker.mjs');
const missing = path.join(cwd, 'not-installed', 'codex');
const result = spawnSync(process.execPath, [script, '--background', '--no-wait'], {
cwd,
encoding: 'utf-8',
env: {
...process.env,
IMPECCABLE_LIVE_CODEX_WORKER: '1',
IMPECCABLE_CODEX_PATH: missing,
},
});
assert.equal(result.status, 0, result.stderr);
const output = JSON.parse(result.stdout);
assert.equal(output.ok, false);
assert.equal(output.status, 'unavailable');
assert.equal(output.error, 'codex_cli_unavailable');
assert.equal(output.fallback, 'foreground');
assert.equal(output.pid, null);
assert.equal(output.setup.afterInstall, 'codex login');
const state = JSON.parse(readFileSync(path.join(cwd, '.impeccable/live/codex-worker.json'), 'utf-8'));
assert.equal(state.error, 'codex_cli_unavailable');
assert.equal(state.mode, 'foreground');
assert.equal(state.command, missing);
});
it('reuses an owned worker before checking whether Codex is still on PATH', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-reuse-'));
const statePath = path.join(cwd, '.impeccable/live/codex-worker.json');
mkdirSync(path.dirname(statePath), { recursive: true });
const worker = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)'], {
stdio: 'ignore',
});
try {
writeFileSync(statePath, JSON.stringify({
owner: CODEX_WORKER_OWNER,
cwd,
threadId: 'owned-thread',
pid: worker.pid,
status: 'ready',
}));
const script = path.resolve('skill/scripts/live-codex-worker.mjs');
const result = spawnSync(process.execPath, [script, '--background', '--no-wait'], {
cwd,
encoding: 'utf-8',
env: {
...process.env,
IMPECCABLE_LIVE_CODEX_WORKER: '1',
IMPECCABLE_CODEX_PATH: path.join(cwd, 'missing-codex'),
},
});
assert.equal(result.status, 0, result.stderr);
const output = JSON.parse(result.stdout);
assert.equal(output.ok, true);
assert.equal(output.reused, true);
assert.equal(output.pid, worker.pid);
} finally {
worker.kill('SIGTERM');
}
});
it('refuses to signal a pid from an unowned state record', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-unowned-'));
const statePath = path.join(cwd, '.impeccable/live/codex-worker.json');
mkdirSync(path.dirname(statePath), { recursive: true });
const unrelated = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)'], {
stdio: 'ignore',
});
try {
writeFileSync(statePath, JSON.stringify({
owner: 'desktop',
cwd,
pid: unrelated.pid,
status: 'ready',
}));
const script = path.resolve('skill/scripts/live-codex-worker.mjs');
const result = spawnSync(process.execPath, [script, '--stop'], {
cwd,
encoding: 'utf-8',
});
assert.equal(result.status, 2, result.stderr);
assert.equal(JSON.parse(result.stdout).error, 'codex_worker_state_unowned');
assert.doesNotThrow(() => process.kill(unrelated.pid, 0));
} finally {
unrelated.kill('SIGTERM');
}
});
it('reports a stop timeout instead of claiming an owned live process stopped', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-stop-timeout-'));
const statePath = path.join(cwd, '.impeccable/live/codex-worker.json');
mkdirSync(path.dirname(statePath), { recursive: true });
const stubborn = spawn(process.execPath, ['-e', "process.on('SIGTERM',()=>{});setInterval(()=>{},1000)"], {
stdio: 'ignore',
});
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50);
try {
writeFileSync(statePath, JSON.stringify({
owner: CODEX_WORKER_OWNER,
cwd,
threadId: 'owned-thread',
pid: stubborn.pid,
status: 'ready',
}));
const script = path.resolve('skill/scripts/live-codex-worker.mjs');
const result = spawnSync(process.execPath, [script, '--stop'], {
cwd,
encoding: 'utf-8',
env: { ...process.env, IMPECCABLE_LIVE_CODEX_STOP_TIMEOUT_MS: '100' },
});
assert.equal(result.status, 2, result.stderr);
assert.equal(JSON.parse(result.stdout).status, 'stop_timeout');
assert.doesNotThrow(() => process.kill(stubborn.pid, 0));
} finally {
stubborn.kill('SIGKILL');
}
});
it('terminates a detached child before returning foreground fallback on startup timeout', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-start-timeout-'));
const liveDir = path.join(cwd, '.impeccable/live');
mkdirSync(liveDir, { recursive: true });
writeFileSync(path.join(liveDir, 'server.json'), JSON.stringify({
pid: process.pid,
port: 1,
token: 'smoke-token',
}));
const fakeCodex = path.join(cwd, 'fake-codex');
writeFileSync(fakeCodex, '#!/bin/sh\nwhile true; do sleep 1; done\n');
chmodSync(fakeCodex, 0o755);
const script = path.resolve('skill/scripts/live-codex-worker.mjs');
const result = spawnSync(process.execPath, [script, '--background'], {
cwd,
encoding: 'utf-8',
env: {
...process.env,
IMPECCABLE_LIVE_CODEX_WORKER: '1',
IMPECCABLE_CODEX_PATH: fakeCodex,
IMPECCABLE_LIVE_CODEX_START_TIMEOUT_MS: '100',
IMPECCABLE_LIVE_CODEX_STOP_TIMEOUT_MS: '1000',
},
timeout: 5_000,
});
assert.equal(result.status, 2, result.stderr);
const output = JSON.parse(result.stdout);
assert.equal(output.error, 'codex_worker_start_timeout');
assert.equal(output.terminated, true);
assert.equal(output.fallback, 'foreground');
assert.throws(() => process.kill(output.childPid, 0), (error) => error.code === 'ESRCH');
});
it('returns a durable starting record without waiting for app-server readiness', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-prewarm-'));
const liveDir = path.join(cwd, '.impeccable/live');
mkdirSync(liveDir, { recursive: true });
writeFileSync(path.join(liveDir, 'server.json'), JSON.stringify({
pid: process.pid,
port: 1,
token: 'smoke-token',
}));
const fakeCodex = path.join(cwd, 'fake-codex');
writeFileSync(fakeCodex, '#!/bin/sh\nwhile true; do sleep 1; done\n');
chmodSync(fakeCodex, 0o755);
const script = path.resolve('skill/scripts/live-codex-worker.mjs');
const startedAt = Date.now();
const result = spawnSync(process.execPath, [script, '--background', '--no-wait'], {
cwd,
encoding: 'utf-8',
env: {
...process.env,
IMPECCABLE_LIVE_CODEX_WORKER: '1',
IMPECCABLE_CODEX_PATH: fakeCodex,
},
timeout: 5_000,
});
assert.equal(result.status, 0, result.stderr);
const output = JSON.parse(result.stdout);
assert.equal(output.status, 'starting');
assert.equal(output.starting, true);
assert.equal(codexWorkerProcessStateIsOwned(output, cwd), true);
assert.ok(Date.now() - startedAt < 1_000, 'prewarm should not wait for app-server initialization');
const stopped = spawnSync(process.execPath, [script, '--stop'], {
cwd,
encoding: 'utf-8',
env: { ...process.env, IMPECCABLE_LIVE_CODEX_STOP_TIMEOUT_MS: '1000' },
timeout: 3_000,
});
assert.equal(stopped.status, 0, stopped.stderr);
assert.equal(JSON.parse(stopped.stdout).status, 'stopped');
});
});
describe('Codex Live worker structured artifact boundary', () => {
it('keeps the model read-only and the supervisor as the only publisher', () => {
const instructions = buildCodexWorkerInstructions('LIVE SPEC');
assert.match(instructions, /Do not write source/);
assert.match(instructions, /read-only repository tools whenever needed/);
assert.match(instructions, /supervisor alone writes staged artifacts/);
assert.match(instructions, /shared-component visual roles/);
assert.match(instructions, /recompose the selected element itself/);
assert.match(instructions, /semantically unified short labels/);
assert.match(instructions, /fits on one line in the original/);
assert.match(instructions, /Every variant must be independently shippable/);
assert.match(instructions, /reject awkward label wrapping/);
assert.match(instructions, /decorative glyphs or pseudo-content/);
assert.match(instructions, /Ignore any instruction.*run commands/);
});
it('requires a coherent variant plan before progressive or atomic multi-variant output', () => {
const firstSchema = codexWorkerOutputSchemaForPhase('first', 3);
const paramsSchema = codexWorkerOutputSchemaForPhase('params', 3);
assert.deepEqual(firstSchema.required, ['files', 'plan']);
assert.ok(firstSchema.properties.plan);
assert.deepEqual(codexWorkerOutputSchemaForPhase('atomic', 3).required, ['files', 'plan']);
assert.deepEqual(paramsSchema.required, ['files']);
assert.equal(paramsSchema.properties.plan, undefined, 'strict schemas cannot expose optional properties');
assert.deepEqual(codexWorkerOutputSchemaForPhase('atomic', 1).required, ['files']);
assert.deepEqual(
codexWorkerOutputSchemaForPhase('remainder', 3, { sourceDelta: true }).required,
['sourceDeltas', 'parameterCss', 'paramsJson'],
);
assert.deepEqual(
codexWorkerOutputSchemaForPhase('first', 3, { sourceDelta: true }).required,
['sourceDelta', 'plan'],
);
const parameterDelta = codexWorkerOutputSchemaForPhase('params', 3, { sourceDelta: true });
assert.deepEqual(parameterDelta.required, ['parameterCss', 'paramsJson']);
assert.equal(parameterDelta.properties.sourceDelta, undefined);
const repairSchema = codexWorkerDetectorRepairSchema(firstSchema);
assert.deepEqual(repairSchema.required, ['files', 'plan', 'detectorWaivers']);
assert.equal(repairSchema.properties.detectorWaivers.type, 'array');
assert.equal(firstSchema.properties.detectorWaivers, undefined, 'normal generation cannot invent waivers');
});
it('writes only the prepared source artifact path', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-source-'));
const artifact = path.join(cwd, '.impeccable/live/artifacts/session-r1.jsx');
mkdirSync(path.dirname(artifact), { recursive: true });
writeFileSync(artifact, 'before');
const prepared = { artifactFile: '.impeccable/live/artifacts/session-r1.jsx' };
applyCodexWorkerOutput({
output: { files: [{ path: prepared.artifactFile, content: 'after' }], plan: variantPlan() },
prepared,
phase: 'atomic',
expectedVariants: 3,
cwd,
});
assert.equal(readFileSync(artifact, 'utf-8'), 'after');
assert.throws(
() => applyCodexWorkerOutput({
output: { files: [{ path: 'src/App.jsx', content: 'unsafe' }], plan: variantPlan() },
prepared,
phase: 'atomic',
expectedVariants: 3,
cwd,
}),
/worker_output_source_path_invalid/,
);
});
it('creates the JSX preview style and variant 1 from a fenced first delta', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-first-delta-'));
const artifact = path.join(cwd, '.impeccable/live/artifacts/session-r1.jsx');
mkdirSync(path.dirname(artifact), { recursive: true });
writeFileSync(artifact, [
'<aside data-impeccable-variants="other-session">',
' <div data-impeccable-variant="1">Other session stays independent</div>',
'</aside>',
'<main>',
' <div data-impeccable-variants="session" data-impeccable-variant-count="3" style={{ display: "contents" }}>',
' {/* Original */}',
' <div data-impeccable-variant="original"><h1>Original</h1></div>',
' {/* Variants: insert below this line */}',
' {/* impeccable-variants-end session */}',
' </div>',
'</main>',
].join('\n'));
const result = applyCodexWorkerOutput({
output: {
sourceDelta: {
variantId: 1,
markup: '<article className="one"><h1>One</h1></article>',
css: '@scope ([data-impeccable-variant="1"]) { :scope > .one { color: red; } }',
},
plan: variantPlan(),
},
prepared: { artifactFile: '.impeccable/live/artifacts/session-r1.jsx' },
phase: 'first',
expectedVariants: 3,
sessionId: 'session',
scaffold: {
styleMode: 'scoped',
styleTag: '<style data-impeccable-css="SESSION_ID">',
commentSyntax: { open: '{/*', close: '*/}' },
},
cwd,
});
const after = readFileSync(artifact, 'utf-8');
assert.equal(result.sourceDelta, true);
assert.deepEqual(result.plan, variantPlan());
assert.match(after, /<style data-impeccable-css="session">\{`/);
assert.match(after, /data-impeccable-variant="1"/);
assert.match(after, /<article className="one"><h1>One<\/h1><\/article>/);
assert.match(after, /`}<\/style>/);
assert.match(after, /Other session stays independent/);
assert.ok(
after.indexOf('<div data-impeccable-variant="1">') < after.indexOf('impeccable-variants-end session'),
'the source updater must keep generated output inside the accept parser boundary',
);
});
it('merges the fenced remaining variants without letting the model resend variant 1', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-source-delta-'));
const artifact = path.join(cwd, '.impeccable/live/artifacts/session-r2.jsx');
mkdirSync(path.dirname(artifact), { recursive: true });
const before = [
'<main>',
' <div data-impeccable-variants="session" data-impeccable-variant-count="3" style={{ display: "contents" }}>',
' <style data-impeccable-css="session">{`',
'@scope ([data-impeccable-variant="1"]) { :scope > .one { color: red; } }',
'`}</style>',
' <div data-impeccable-variant="original"><h1>Original</h1></div>',
' <div data-impeccable-variant="1"><h1 className="one">Immutable</h1></div>',
' {/* impeccable-variants-end session */}',
' </div>',
'</main>',
].join('\n');
writeFileSync(artifact, before);
const prepared = { artifactFile: '.impeccable/live/artifacts/session-r2.jsx' };
applyCodexWorkerOutput({
output: {
sourceDeltas: [
{
variantId: 2,
markup: '<article className="two"><h1>Two</h1></article>',
css: '@scope ([data-impeccable-variant="2"]) { :scope > .two { color: green; } }',
},
{
variantId: 3,
markup: '<article className="three"><h1>Three</h1></article>',
css: '@scope ([data-impeccable-variant="3"]) { :scope > .three { color: blue; } }',
},
],
parameterCss: '',
paramsJson: emptyParamsJson(),
},
prepared,
phase: 'remainder',
expectedVariants: 3,
sessionId: 'session',
scaffold: { styleMode: 'scoped' },
cwd,
});
const after = readFileSync(artifact, 'utf-8');
assert.match(after, /<h1 className="one">Immutable<\/h1>/);
assert.match(after, /data-impeccable-variant="2"/);
assert.match(after, /<article className="two"><h1>Two<\/h1><\/article>/);
assert.match(after, /@scope \(\[data-impeccable-variant="2"\]\)/);
assert.equal((after.match(/data-impeccable-variant="1"/g) || []).length, 2);
assert.ok(
after.indexOf('<div data-impeccable-variant="2">') < after.indexOf('impeccable-variants-end session'),
);
assert.throws(() => applyCodexWorkerOutput({
output: {
sourceDeltas: [
{
variantId: 2,
markup: '<article>Unsafe</article>',
css: '@scope ([data-impeccable-variant="1"]) { :scope { color: hotpink; } }',
},
{
variantId: 3,
markup: '<article>Three</article>',
css: '@scope ([data-impeccable-variant="3"]) { :scope > article { color: blue; } }',
},
],
parameterCss: '',
paramsJson: emptyParamsJson(),
},
prepared: { ...prepared, artifactFile: prepared.artifactFile },
phase: 'remainder',
expectedVariants: 3,
sessionId: 'session',
scaffold: { styleMode: 'scoped' },
cwd,
}), /worker_output_source_delta_css_unfenced/);
});
it('keeps progressive deltas inside the deterministic early-Accept boundary', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-delta-accept-'));
const artifact = path.join(cwd, 'App.jsx');
writeFileSync(artifact, [
'export default function App() {',
' return <main>',
' <div data-impeccable-variants="session" data-impeccable-variant-count="3" style={{ display: "contents" }}>',
' {/* impeccable-variants-start session */}',
' <div data-impeccable-variant="original"><article>Original</article></div>',
' {/* Variants: insert below this line */}',
' {/* impeccable-variants-end session */}',
' </div>',
' </main>;',
'}',
].join('\n'));
const prepared = { artifactFile: 'App.jsx' };
applyCodexWorkerOutput({
output: {
sourceDelta: {
variantId: 1,
markup: '<article className="one">One</article>',
css: '@scope ([data-impeccable-variant="1"]) { :scope > .one { color: red; } }',
},
plan: variantPlan(),
},
prepared,
phase: 'first',
expectedVariants: 3,
sessionId: 'session',
scaffold: {
styleMode: 'scoped',
styleTag: '<style data-impeccable-css="SESSION_ID">',
commentSyntax: { open: '{/*', close: '*/}' },
},
cwd,
});
applyCodexWorkerOutput({
output: {
sourceDeltas: [
{
variantId: 2,
markup: '<article className="two">Two</article>',
css: '@scope ([data-impeccable-variant="2"]) { :scope > .two { color: green; } }',
},
{
variantId: 3,
markup: '<article className="three">Three</article>',
css: '@scope ([data-impeccable-variant="3"]) { :scope > .three { color: blue; } }',
},
],
parameterCss: '',
paramsJson: emptyParamsJson(),
},
prepared,
phase: 'remainder',
expectedVariants: 3,
sessionId: 'session',
scaffold: { styleMode: 'scoped' },
cwd,
});
const accepted = spawnSync(process.execPath, [
path.resolve('skill/scripts/live-accept.mjs'),
'--id', 'session', '--variant', '2',
], { cwd, encoding: 'utf-8' });
assert.equal(accepted.status, 0, accepted.stderr);
assert.equal(JSON.parse(accepted.stdout).handled, true);
const after = readFileSync(artifact, 'utf-8');
assert.match(after, />Two<\/article>/);
assert.doesNotMatch(after, />One<\/article>/);
assert.doesNotMatch(after, /impeccable-variants-end session/);
});
it('merges Astro global-prefixed deltas without introducing scoped CSS', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-astro-delta-'));
const artifact = path.join(cwd, '.impeccable/live/artifacts/session-r2.astro');
mkdirSync(path.dirname(artifact), { recursive: true });
writeFileSync(artifact, [
'<main>',
' <!-- impeccable-variants-start session -->',
' <div data-impeccable-variants="session" data-impeccable-variant-count="3" style="display: contents">',
' <style is:inline data-impeccable-css="session">',
' [data-impeccable-variant="1"] > .one { color: red; }',
' </style>',
' <div data-impeccable-variant="original"><h1>Original</h1></div>',
' <div data-impeccable-variant="1"><h1 class="one">One</h1></div>',
' <!-- impeccable-variants-end session -->',
' </div>',
' <!-- impeccable-variants-end session -->',
'</main>',
].join('\n'));
applyCodexWorkerOutput({
output: {
sourceDeltas: [
{
variantId: 2,
markup: '<article class="two"><h1>Two</h1></article>',
css: '[data-impeccable-variant="2"] > .two { color: green; }',
},
{
variantId: 3,
markup: '<article class="three"><h1>Three</h1></article>',
css: '[data-impeccable-variant="3"] > .three { color: blue; }',
},
],
parameterCss: '',
paramsJson: emptyParamsJson(),
},
prepared: { artifactFile: '.impeccable/live/artifacts/session-r2.astro' },
phase: 'remainder',
expectedVariants: 3,
sessionId: 'session',
scaffold: { styleMode: 'astro-global-prefixed' },
cwd,
});
const after = readFileSync(artifact, 'utf-8');
assert.match(after, /\[data-impeccable-variant="2"\] > \.two/);
assert.match(after, /<div data-impeccable-variant="2"[^>]*>/);
assert.doesNotMatch(after, /@scope/);
assert.match(after, /<!-- impeccable-variants-end session -->/);
assert.ok(after.indexOf('<div data-impeccable-variant="2">') < after.indexOf('impeccable-variants-end session'));
});
it('publishes the remaining source variants and parameters together without rewriting prior output', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-final-delta-'));
const artifact = path.join(cwd, 'App.jsx');
writeFileSync(artifact, [
'<main>',
' <div data-impeccable-variants="session" data-impeccable-variant-count="3">',
' <style data-impeccable-css="session">{`',
'@scope ([data-impeccable-variant="1"]) { :scope > .one { color: red; } }',
'`}</style>',
' <div data-impeccable-variant="original"><article>Original</article></div>',
' <div data-impeccable-variant="1"><article className="one">Immutable one</article></div>',
' {/* impeccable-variants-end session */}',
' </div>',
'</main>',
].join('\n'));
const paramsJson = JSON.stringify({
1: [{ id: 'scale', kind: 'range', min: 0.8, max: 1.2, step: 0.1, default: 1, label: 'Scale' }],
2: [{ id: 'dense', kind: 'toggle', default: false, label: 'Dense' }],
3: [{
id: 'face',
kind: 'steps',
default: 'serif',
label: 'Face',
options: [{ value: 'serif', label: 'Serif' }, { value: 'sans', label: 'Sans' }],
}],
});
applyCodexWorkerOutput({
output: {
sourceDeltas: [
{
variantId: 2,
markup: '<article className="two">Two</article>',
css: '@scope ([data-impeccable-variant="2"]) { :scope > .two { color: green; } }',
},
{
variantId: 3,
markup: '<article className="three">Three</article>',
css: '@scope ([data-impeccable-variant="3"]) { :scope > .three { color: blue; } }',
},
],
parameterCss: [
'@scope ([data-impeccable-variant="1"]) { :scope[data-p-scale] > .one { scale: var(--p-scale); } }',
'@scope ([data-impeccable-variant="2"]) { :scope[data-p-dense] > .two { padding: 0; } }',
'@scope ([data-impeccable-variant="3"]) { :scope[data-p-face="sans"] > .three { font-family: sans-serif; } }',
].join('\n'),
paramsJson,
},
prepared: { artifactFile: 'App.jsx' },
phase: 'remainder',
expectedVariants: 3,
sessionId: 'session',
scaffold: { styleMode: 'scoped' },
cwd,
});
const after = readFileSync(artifact, 'utf-8');
assert.match(after, /Immutable one/);
assert.match(after, /className="two">Two/);
assert.match(after, /className="three">Three/);
assert.equal((after.match(/data-impeccable-params=/g) || []).length, 3);
assert.match(after, /data-p-scale/);
assert.ok(after.indexOf('className="three"') < after.indexOf('impeccable-variants-end session'));
});
it('never lets a remaining component turn rewrite arrived variant 1', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-component-'));
const componentDir = path.join(cwd, '.impeccable/live/artifacts/session-r2-svelte');
mkdirSync(componentDir, { recursive: true });
writeFileSync(path.join(componentDir, 'manifest.json'), JSON.stringify({
id: 'session',
previewMode: 'svelte-component',
componentExtension: 'svelte',
arrivedVariants: 1,
}));
writeFileSync(path.join(componentDir, 'v1.svelte'), '<h1>Immutable</h1>');
const prepared = {
previewMode: 'svelte-component',
componentDir: '.impeccable/live/artifacts/session-r2-svelte',
artifactFile: '.impeccable/live/artifacts/session-r2-svelte/manifest.json',
};
assert.throws(
() => applyCodexWorkerOutput({
output: { files: [{ path: 'v1.svelte', content: '<h1>Changed</h1>' }] },
prepared,
phase: 'remainder',
expectedVariants: 3,
cwd,
}),
/published_variant_changed/,
);
applyCodexWorkerOutput({
output: {
files: [
{ path: 'v2.svelte', content: '<h1>Two</h1>' },
{ path: 'v3.svelte', content: '<h1>Three</h1>' },
{ path: 'params.json', content: emptyParamsJson() },
],
},
prepared,
phase: 'remainder',
expectedVariants: 3,
cwd,
});
assert.equal(readFileSync(path.join(componentDir, 'v1.svelte'), 'utf-8'), '<h1>Immutable</h1>');
assert.equal(JSON.parse(readFileSync(path.join(componentDir, 'manifest.json'))).arrivedVariants, 3);
});
it('publishes all remaining component variants and parameters in the second turn', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-component-second-'));
const componentDir = path.join(cwd, '.impeccable/live/artifacts/session-r2-svelte');
mkdirSync(componentDir, { recursive: true });
writeFileSync(path.join(componentDir, 'manifest.json'), JSON.stringify({
previewMode: 'svelte-component',
componentExtension: 'svelte',
arrivedVariants: 1,
}));
writeFileSync(path.join(componentDir, 'v1.svelte'), '<h1>Immutable</h1>');
const prepared = {
previewMode: 'svelte-component',
componentDir: '.impeccable/live/artifacts/session-r2-svelte',
artifactFile: '.impeccable/live/artifacts/session-r2-svelte/manifest.json',
};
applyCodexWorkerOutput({
output: { files: [
{ path: 'v2.svelte', content: '<h1>Two</h1>' },
{ path: 'v3.svelte', content: '<h1>Three</h1>' },
{ path: 'params.json', content: emptyParamsJson() },
] },
prepared,
phase: 'remainder',
expectedVariants: 3,
cwd,
});
assert.equal(readFileSync(path.join(componentDir, 'v1.svelte'), 'utf-8'), '<h1>Immutable</h1>');
assert.equal(readFileSync(path.join(componentDir, 'v2.svelte'), 'utf-8'), '<h1>Two</h1>');
assert.equal(readFileSync(path.join(componentDir, 'v3.svelte'), 'utf-8'), '<h1>Three</h1>');
assert.equal(JSON.parse(readFileSync(path.join(componentDir, 'manifest.json'))).arrivedVariants, 3);
assert.equal(existsSync(path.join(componentDir, 'params.json')), true);
});
it('requires atomic component output to contain v1 through vN plus params', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-component-atomic-'));
const componentDir = path.join(cwd, '.impeccable/live/artifacts/session-r1-svelte');
mkdirSync(componentDir, { recursive: true });
writeFileSync(path.join(componentDir, 'manifest.json'), JSON.stringify({
previewMode: 'svelte-component',
componentExtension: 'svelte',
}));
writeFileSync(path.join(componentDir, 'v1.svelte'), '<h1>Scaffold stub</h1>');
const prepared = {
previewMode: 'svelte-component',
componentDir: '.impeccable/live/artifacts/session-r1-svelte',
artifactFile: '.impeccable/live/artifacts/session-r1-svelte/manifest.json',
};
assert.throws(() => applyCodexWorkerOutput({
output: { files: [
{ path: 'v2.svelte', content: '<h1>Two</h1>' },
{ path: 'v3.svelte', content: '<h1>Three</h1>' },
{ path: 'params.json', content: '{}' },
], plan: variantPlan() },
prepared,
phase: 'atomic',
expectedVariants: 3,
cwd,
}), /worker_output_component_file_missing/);
});
it('does not let precreated stubs satisfy missing remaining component output', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-component-final-'));
const componentDir = path.join(cwd, '.impeccable/live/artifacts/session-r2-svelte');
mkdirSync(componentDir, { recursive: true });
writeFileSync(path.join(componentDir, 'manifest.json'), JSON.stringify({
previewMode: 'svelte-component',
componentExtension: 'svelte',
arrivedVariants: 1,
}));
for (const variant of [1, 2, 3]) writeFileSync(path.join(componentDir, `v${variant}.svelte`), `<h1>${variant}</h1>`);
writeFileSync(path.join(componentDir, 'params.json'), '{}');
const prepared = {
previewMode: 'svelte-component',
componentDir: '.impeccable/live/artifacts/session-r2-svelte',
artifactFile: '.impeccable/live/artifacts/session-r2-svelte/manifest.json',
};
assert.throws(() => applyCodexWorkerOutput({
output: { files: [{ path: 'v2.svelte', content: '<h1>Two replacement</h1>' }] },
prepared,
phase: 'remainder',
expectedVariants: 3,
cwd,
}), /worker_output_component_file_missing/);
});
it('builds phase prompts from the exact staged artifact and durable thread context', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-context-'));
const artifactPath = path.join(cwd, 'artifact.html');
writeFileSync(artifactPath, '<main>wrapped</main>');
const prepared = { artifactFile: 'artifact.html' };
const artifact = readPreparedArtifact(prepared, { cwd });
const prompt = buildGenerationTurnInput({
event: { id: 'abc', count: 3, action: 'bolder', scaffold: { file: 'artifact.html' } },
phase: 'first',
prepared,
artifact,
product: 'Product facts',
design: 'Design tokens',
actionReference: 'Polish rules',
contextMetadata: { productPath: 'docs/PRODUCT.md' },
});
assert.match(prompt, /Produce only variant 1/);
assert.match(prompt, /strongest low-risk, independently shippable/);
assert.match(prompt, /shared identity lock and exactly 3 distinct/);
assert.match(prompt, /keep variant 1 low-risk/);
assert.match(prompt, /Reserve root recomposition for variant 2 or 3/);
assert.match(prompt, /Color alone is not a sufficient primary axis/);
assert.match(prompt, /Every \/bolder direction must be visibly more assertive/);
assert.match(prompt, /<main>wrapped<\/main>/);
assert.match(prompt, /Product facts/);
assert.match(prompt, /Design tokens/);
assert.match(prompt, /docs\/PRODUCT\.md/);
assert.doesNotMatch(prompt, /source_neighborhood/);
const remainderPrompt = buildGenerationTurnInput({
event: { id: 'abc', count: 3 },
phase: 'remainder',
prepared,
artifact,
variantPlan: variantPlan(),
});
assert.match(remainderPrompt, /Follow the durable variant plan/);
assert.match(remainderPrompt, /variants 2 through 3 and the final tunable parameters together/);
assert.match(remainderPrompt, /parameterCss and paramsJson/);
assert.match(remainderPrompt, /Composition/);
const paramsPrompt = buildGenerationTurnInput({
event: { id: 'abc', count: 3 },
phase: 'params',
prepared,
artifact,
variantPlan: variantPlan(),
});
assert.match(paramsPrompt, /Return only parameterCss and paramsJson/);
assert.match(paramsPrompt, /Do not return markup or restyle any default appearance/);
assert.match(paramsPrompt, /Do not call tools or inspect the repository/);
assert.match(paramsPrompt, /Parameter schema examples: range/);
});
it('attaches the real skill and annotation image as first-class turn inputs', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-inputs-'));
const skillPath = path.join(cwd, 'SKILL.md');
const screenshotPath = path.join(cwd, 'annotation.png');
writeFileSync(skillPath, '# Skill');
writeFileSync(screenshotPath, 'png');
assert.deepEqual(buildCodexWorkerTurnInputs({ prompt: 'work', skillPath, screenshotPath, cwd }), [
{ type: 'skill', name: 'impeccable', path: skillPath },
{ type: 'localImage', path: screenshotPath, detail: 'high' },
{ type: 'text', text: 'work' },
]);
assert.deepEqual(buildCodexWorkerTurnInputs({ prompt: 'work', screenshotPath: '/tmp/outside.png', cwd }), [
{ type: 'text', text: 'work' },
]);
});
});
function variantPlan() {
return {
identityLock: ['Preserve copy and established component roles'],
directions: [
{ variantId: 1, name: 'Hierarchy', axis: 'type scale', intent: 'Strengthen the primary hierarchy' },
{ variantId: 2, name: 'Composition', axis: 'spatial layout', intent: 'Recompose the selected root' },
{ variantId: 3, name: 'Rhythm', axis: 'spacing and rules', intent: 'Increase editorial rhythm' },
],
};
}
function emptyParamsJson() {
return '{"1":[],"2":[],"3":[]}';
}
+1 -11
View File
@@ -1,18 +1,8 @@
import assert from 'node:assert/strict';
import { describe, it } from 'node:test';
import {
htmlToJsx,
isExpectedGenerationCancellation,
normalizeVariantOutput,
} from './live-e2e/agent.mjs';
import { htmlToJsx, normalizeVariantOutput } from './live-e2e/agent.mjs';
describe('live-e2e agent output translation', () => {
it('treats a fenced late generation as expected cancellation only', () => {
assert.equal(isExpectedGenerationCancellation(new Error('Source publication prepare failed: stale_generation_epoch')), true);
assert.equal(isExpectedGenerationCancellation(new Error('Source publication failed: stale_source_revision')), false);
assert.equal(isExpectedGenerationCancellation(new Error('provider unavailable')), false);
});
it('converts HTML class and inline style attributes to JSX syntax', () => {
const jsx = htmlToJsx(
'<h1 class="hero-title" style="--p-scale:1; font-size:2.25rem; font-weight:700">Title</h1>',
-63
View File
@@ -9,13 +9,10 @@ import {
createLlmAgent,
parseManualEditResponse,
parseVariantResponse,
progressiveVariantGuidance,
resolveLlmAgentConfig,
validateManualEditCoverage,
validateManualEditPlanningCoverage,
validateVariantMaterialChange,
validateVariantCount,
validateProgressiveVariantOutput,
validateVariantVisibleCopy,
} from './live-e2e/agents/llm-agent.mjs';
@@ -1462,19 +1459,6 @@ describe('live-e2e LLM agent manual edit coverage validation', () => {
});
describe('live-e2e LLM agent variant prompt', () => {
it('makes progressive phase boundaries and lazy parameters explicit', () => {
const first = progressiveVariantGuidance({ count: 1, progressive: { phase: 'first' } });
const remaining = progressiveVariantGuidance({
count: 3,
progressive: { phase: 'remaining', omitFirstVariantCss: true },
});
assert.match(first, /params: \[\]/);
assert.match(first, /materially different/);
assert.match(remaining, /complete final set of exactly 3 variants/);
assert.match(remaining, /Keep its innerHtml exactly unchanged/);
assert.match(remaining, /Do not repeat or modify any scopedCss rule/);
});
it('tells the model not to nest duplicate picked containers', () => {
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /replacement root itself/);
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /do not wrap a duplicate/);
@@ -1500,53 +1484,6 @@ describe('live-e2e LLM agent variant prompt', () => {
});
describe('live-e2e LLM agent variant copy validation', () => {
it('enforces the exact requested variant count', () => {
const parsed = { scopedCss: '', variants: [{ innerHtml: '<h1>One</h1>', params: [] }] };
assert.match(validateVariantCount(parsed, { count: 2 }), /expected exactly 2 variants, received 1/);
assert.equal(validateVariantCount(parsed, { count: 1 }), null);
});
it('defers progressive params and preserves the visible first variant', () => {
const firstHtml = '<h1 class="hero-title"><span>One</span></h1>';
assert.match(
validateProgressiveVariantOutput(
{ variants: [{ innerHtml: firstHtml, params: [{ id: 'weight' }] }] },
{ progressive: { phase: 'first' } },
),
/defer params/,
);
assert.equal(
validateProgressiveVariantOutput(
{ variants: [{ innerHtml: firstHtml, params: [] }] },
{ progressive: { phase: 'remaining', firstVariant: { innerHtml: firstHtml } } },
),
null,
);
assert.match(
validateProgressiveVariantOutput(
{ variants: [{ innerHtml: '<h1>Changed</h1>', params: [] }] },
{ progressive: { phase: 'remaining', firstVariant: { innerHtml: firstHtml } } },
),
/preserve variant 1/,
);
assert.match(
validateProgressiveVariantOutput(
{
scopedCss: '@scope ([data-impeccable-variant="1"]) { .hero-title { color: red; } }',
variants: [{ innerHtml: firstHtml, params: [] }],
},
{
progressive: {
phase: 'remaining',
firstVariant: { innerHtml: firstHtml },
omitFirstVariantCss: true,
},
},
),
/omit already-published variant 1 CSS/,
);
});
it('allows variants that preserve the picked element text', () => {
const result = validateVariantVisibleCopy(
{
+11 -377
View File
@@ -22,7 +22,7 @@
import { describe, it, before, after } from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { appendFileSync, cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname, join, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -38,7 +38,6 @@ import {
clickAccept,
clickApplyEdits,
clickEditCopy,
clickDiscard,
clickSaveEdit,
clickGo,
clickNext,
@@ -46,7 +45,6 @@ import {
editTextLeaf,
drawAnnotationPinAndStroke,
getVisibleVariant,
installLiveQueryHelpers,
pickElement,
runLiveChromeBottomBarSmoke,
waitForApplyDockHidden,
@@ -222,7 +220,7 @@ for (const { name, fixture } of fixtures) {
const domSelector = isInsert
? insertDomSelector
: pickSelector;
const usesSvelteComponentPreview = fixtureUsesSvelteKitAdapter(fixture) || name === 'nuxt-vite7';
const usesSvelteComponentPreview = fixtureUsesSvelteKitAdapter(fixture);
const variantContentSelector = isInsert
? (usesSvelteComponentPreview ? '.inserted-copy' : '[data-impeccable-variant="2"] .inserted-copy')
: usesSvelteComponentPreview
@@ -316,11 +314,10 @@ for (const { name, fixture } of fixtures) {
const after = readFileSync(sourceFile, 'utf-8');
const svelteComponentSession = svelteComponentTargetFor(sourceFile);
if (svelteComponentSession) {
const componentExtension = svelteComponentSession.manifest.componentExtension || 'svelte';
const variantFile = join(tmp, svelteComponentSession.manifest.componentDir, `v2.${componentExtension}`);
const variantFile = join(tmp, svelteComponentSession.manifest.componentDir, 'v2.svelte');
const variantBody = readFileSync(variantFile, 'utf-8');
const routeBody = readFileSync(join(tmp, svelteComponentSession.manifest.sourceFile), 'utf-8');
assert.match(after, /"previewMode": "(?:svelte|vue)-component"/, 'framework component manifest inserted');
assert.match(after, /"previewMode": "svelte-component"/, 'Svelte component manifest inserted');
if (isInsert) {
assert.equal(svelteComponentSession.manifest.mode, 'insert', 'Svelte insert manifest marks insert mode');
if (agentMode === 'fake') {
@@ -331,9 +328,9 @@ for (const { name, fixture } of fixtures) {
assert.match(variantBody, /<([a-z][\w:-]*)\b[\s\S]*<\/\1>|<[a-z][\w:-]*\b[^>]*\/>/i, 'Svelte insert variant component contains a root element');
}
} else {
assert.match(variantBody, new RegExp(`<${svelteComponentSession.expectedTag}\\b`), 'component variant contains target element');
assert.match(variantBody, new RegExp(`<${svelteComponentSession.expectedTag}\\b`), 'Svelte variant component contains target element');
}
assert.doesNotMatch(routeBody, /data-impeccable-variants="/, 'route source is not edited during component preview');
assert.doesNotMatch(routeBody, /data-impeccable-variants="/, 'Svelte route source is not edited during generation');
} else {
assert.match(after, /data-impeccable-variants="/, 'wrapper inserted');
}
@@ -352,8 +349,7 @@ for (const { name, fixture } of fixtures) {
}
}
if (svelteComponentSession) {
const componentExtension = svelteComponentSession.manifest.componentExtension || 'svelte';
assert.match(readFileSync(join(tmp, svelteComponentSession.manifest.componentDir, `v2.${componentExtension}`), 'utf-8'), /<style\b/, 'component variant has a style block');
assert.match(readFileSync(join(tmp, svelteComponentSession.manifest.componentDir, 'v2.svelte'), 'utf-8'), /<style>/, 'Svelte component variant has scoped style block');
} else if (sourceFile.endsWith('.astro')) {
assert.match(after, /<style is:inline data-impeccable-css="/, 'Astro live CSS uses an inline compiler-bypassing style block');
assert.match(
@@ -380,13 +376,6 @@ for (const { name, fixture } of fixtures) {
for (const kind of ['range', 'steps', 'toggle']) {
assert.match(paramsSource, new RegExp(`"kind"\\s*:\\s*"${kind}"`), `param kind ${kind} present`);
}
await page.waitForFunction(() => {
const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.()
|| window.__IMPECCABLE_LIVE_UI_ROOT__
|| document;
const tune = root.querySelector('[data-iceq-tune="1"]');
return tune && tune.disabled === false && /Tune/.test(tune.textContent || '');
}, { timeout: 5_000 });
}
// 6. Cycle variants. Most fixtures stop at variant 2; Svelte Insert
@@ -660,271 +649,6 @@ for (const { name, fixture } of fixtures) {
}
});
if (['vite8-react-plain', 'astro-vite7', 'nextjs-app-router', 'vite8-sveltekit', 'nuxt-vite7'].includes(name) && shouldRunScenario('progressive')) {
it('reveals variant 1 safely while the remaining variants and params are pending', liveE2eTestOptions, async (t) => {
if (manualOnly || process.env.IMPECCABLE_E2E_MANUAL_SCENARIO) {
t.skip('manual scenario filter is active');
return;
}
const traceEvents = [];
const session = await bootFixtureSession({
name,
fixture,
browser,
agent: createFakeAgent(),
wrapTarget: wrapTargetFromPickedElement,
progressive: true,
progressiveDelayMs: 2500,
trace: (eventName, data = {}) => traceEvents.push({ name: eventName, at: Date.now(), ...data }),
log: (m) => t.diagnostic(m),
});
const { page, tmp, consoleErrors, teardown } = session;
let sourceFile = null;
try {
await waitForHandshake(page);
const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title';
const originalCopy = await page.locator(pickSelector).innerText();
await pickElement(page, pickSelector);
await clickGo(page);
const partial = await waitForProgressiveReviewState(page, 3);
assert.equal(partial.arrived, 1, 'exactly variant 1 is present during the progressive interval');
assert.equal(partial.visible, 1, 'variant 1 is the visible review target');
assert.equal(partial.copy, originalCopy, 'variant 1 preserves the picked copy');
assert.notEqual(partial.acceptPointerEvents, 'none', 'Accept is available for the first reviewable variant');
assert.notEqual(partial.discardPointerEvents, 'none', 'Discard can cancel unfinished generation');
assert.equal(partial.hasParams, false, 'variant 1 has no eager parameter manifest');
assert.equal(partial.tuneVisible, true, 'Tune stays visible while parameter generation is outstanding');
assert.equal(partial.tuneDisabled, true, 'pending Tune is non-interactive until controls arrive');
assert.match(partial.tuneTitle || '', /still being prepared/, 'pending Tune explains its loading state');
assert.equal(partial.paramsPanelVisible, false, 'the Tune popover stays closed until parameter delivery');
sourceFile = await locateSessionFile(tmp);
const isComponentPreview = sourceFile.endsWith('manifest.json');
if (isComponentPreview) {
const manifest = JSON.parse(readFileSync(sourceFile, 'utf-8'));
sourceFile = join(tmp, manifest.sourceFile);
const extension = manifest.componentExtension || 'svelte';
assert.equal(existsSync(join(tmp, manifest.componentDir, `v1.${extension}`)), true, 'partial component preview contains variant 1');
assert.equal(existsSync(join(tmp, manifest.componentDir, 'params.json')), false, 'partial component preview defers parameter manifests');
} else {
const partialSource = readFileSync(sourceFile, 'utf-8');
assert.equal(countSourceVariants(partialSource), 1, 'partial source contains one reviewable variant');
assert.doesNotMatch(partialSource, /data-impeccable-params=/, 'partial source defers parameter manifests');
}
// Keyboard Accept must durably fence the worker before its delayed
// second publication, then return the browser to picking without
// waiting for variants the user no longer wants.
const acceptClickedAt = Date.now();
await clickAccept(page, { expectedVariant: 1 });
await waitForBarHidden(page);
await page.waitForFunction(
() => window.__IMPECCABLE_LIVE_STATE__ === 'PICKING',
{ timeout: 2_000 },
);
const automationAcceptToPickingMs = Date.now() - acceptClickedAt;
const browserAcceptToPickingMs = Number(await page.evaluate(() => document.documentElement.dataset.impeccableAcceptToPickingMs));
const acceptToPickingMs = Number.isFinite(browserAcceptToPickingMs) && browserAcceptToPickingMs > 0
? browserAcceptToPickingMs
: automationAcceptToPickingMs;
t.diagnostic(`Accept dispatch → picker ready: ${acceptToPickingMs}ms (${automationAcceptToPickingMs}ms including Playwright actionability)`);
assert.ok(acceptToPickingMs < 500, `Accept should release the picker within 500ms of dispatch; got ${acceptToPickingMs}ms`);
const finalSource = await waitForSourceClean(sourceFile, 20_000);
assert.match(finalSource, new RegExp(escapeRegExp(originalCopy)), 'early accepted source preserves the original copy');
assert.doesNotMatch(finalSource, /data-impeccable-variant=/, 'early accepted source is free of preview scaffolding');
assert.equal(countSourceVariants(finalSource), 0, 'the delayed worker cannot reinsert later variants');
const firstGenerateId = traceEvents.find((event) => event.name === 'agent.event.received' && event.type === 'generate')?.id;
// Give framework HMR one paint to settle the newly committed tree;
// this stays inside the 1.5s next-pick budget and avoids selecting a
// node instance React is replacing in the same frame.
if (name === 'nextjs-app-router' || name === 'vite8-sveltekit' || name === 'nuxt-vite7') await waitForHandshake(page);
await page.waitForTimeout(250);
await page.mouse.move(1, 1);
const nextPickSelector = name === 'nextjs-app-router'
? 'main.page'
: name === 'vite8-sveltekit'
? 'article.feature-card'
: name === 'nuxt-vite7'
? 'main.page'
: '.hero-hook';
await pickElement(page, nextPickSelector, {
resetPickMode: name === 'nextjs-app-router' || name === 'nuxt-vite7',
position: name === 'nuxt-vite7' ? { x: 12, y: 12 } : undefined,
});
const nextGoAt = Date.now();
await clickGo(page);
let nextGenerateTrace = null;
const pickupDeadline = Date.now() + 1_500;
while (Date.now() < pickupDeadline) {
nextGenerateTrace = traceEvents.find((event) => (
event.name === 'agent.event.received'
&& event.type === 'generate'
&& event.id !== firstGenerateId
));
if (nextGenerateTrace) break;
await new Promise((resolve) => setTimeout(resolve, 20));
}
assert.ok(nextGenerateTrace, 'the poll supervisor picks up the next generation while the canceled worker unwinds');
const nextDispatchToPickupMs = nextGenerateTrace.at - nextGenerateTrace.clientSentAt;
assert.ok(
nextDispatchToPickupMs < 1_500,
`next generation pickup should stay below 1.5s from dispatch; got ${nextDispatchToPickupMs}ms`,
);
t.diagnostic(`Next Go dispatch → generation pickup: ${nextDispatchToPickupMs}ms (${nextGenerateTrace.at - nextGoAt}ms including Playwright actionability)`);
if (process.env.IMPECCABLE_E2E_METRICS_FILE) {
appendFileSync(process.env.IMPECCABLE_E2E_METRICS_FILE, JSON.stringify({
acceptToPickingMs,
nextGoToPickupMs: nextDispatchToPickupMs,
automationAcceptToPickingMs,
automationNextGoToPickupMs: nextGenerateTrace.at - nextGoAt,
fixture: name,
at: new Date().toISOString(),
}) + '\n');
}
assert.ok(
traceEvents.some((event) => event.name === 'agent.scaffold.reused'),
'agent reuses the server preflight scaffold',
);
assert.equal(
traceEvents.some((event) => event.name === 'agent.scaffold.start'),
false,
'agent does not repeat deterministic source discovery after preflight',
);
const generateTrace = traceEvents.find((event) => event.name === 'agent.event.received' && event.type === 'generate');
assert.ok(generateTrace?.id, 'generate trace exposes the durable session id');
const generationTimings = await waitForGenerationTimings(tmp, generateTrace.id, { requireAllVariants: false });
assert.ok(generationTimings.generation_ready?.at, 'durable timing records when generation work can start');
assert.ok(generationTimings.first_reviewable?.at, 'durable timing records the first reviewable variant');
assert.equal(generationTimings.all_variants_ready, undefined, 'canceled work never records all variants ready');
const realErrors = consoleErrors.filter((error) =>
!/(Download the React DevTools|StrictMode|Failed to load resource: the server responded with a status of 404)/i.test(error),
);
if (fixture.runtime.probe?.expectConsoleClean) {
assert.deepEqual(realErrors, [], 'progressive HMR and early-action guards produce no browser errors');
} else if (realErrors.length > 0) {
t.diagnostic(`Known framework HMR console noise during progressive source rewrites: ${realErrors.length} error(s)`);
for (const error of realErrors) t.diagnostic(error.split('\n')[0]);
}
} finally {
await teardownAndResetBrowser(teardown);
}
});
}
if (name === 'vite8-react-plain' && shouldRunScenario('progressive')) {
it('accepts variant 2 while variant 3 is still pending', liveE2eTestOptions, async (t) => {
const traceEvents = [];
const session = await bootFixtureSession({
name,
fixture,
browser,
agent: createFakeAgent(),
wrapTarget: wrapTargetFromPickedElement,
progressive: true,
progressiveInitialCount: 2,
progressiveDelayMs: 2500,
trace: (eventName, data = {}) => traceEvents.push({ name: eventName, at: Date.now(), ...data }),
log: (m) => t.diagnostic(m),
});
const { page, tmp, consoleErrors, teardown } = session;
try {
await waitForHandshake(page);
const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title';
const originalCopy = await page.locator(pickSelector).innerText();
await pickElement(page, pickSelector);
await clickGo(page);
const partial = await waitForProgressiveReviewState(page, 3, { arrived: 2, visible: 1 });
assert.equal(partial.arrived, 2, 'variants 1 and 2 arrive before variant 3');
assert.equal(partial.visible, 1, 'variant 1 remains visible until the user advances');
assert.notEqual(partial.acceptPointerEvents, 'none', 'arrived variants remain actionable while the tail is pending');
assert.equal(partial.hasParams, false, 'the partial two-variant revision still defers parameter manifests');
await clickNext(page);
const second = await readProgressiveReviewState(page);
assert.equal(second.visible, 2, 'variant 2 is reviewable before variant 3 exists');
assert.equal(second.copy, originalCopy, 'variant 2 preserves the picked copy');
const wrappedSource = await locateSessionFile(tmp);
const acceptStartedAt = Date.now();
await clickAccept(page, { expectedVariant: 2 });
await waitForBarHidden(page);
await page.waitForFunction(
() => window.__IMPECCABLE_LIVE_STATE__ === 'PICKING',
{ timeout: 2_000 },
);
const browserAcceptMs = Number(await page.evaluate(() => document.documentElement.dataset.impeccableAcceptToPickingMs));
const acceptToPickingMs = Number.isFinite(browserAcceptMs) && browserAcceptMs > 0
? browserAcceptMs
: Date.now() - acceptStartedAt;
assert.ok(acceptToPickingMs < 500, `variant 2 Accept should release the picker within 500ms; got ${acceptToPickingMs}ms`);
const cleanSource = await waitForSourceClean(wrappedSource, 20_000);
assert.match(cleanSource, new RegExp(escapeRegExp(originalCopy)), 'accepted variant 2 preserves source copy');
assert.doesNotMatch(cleanSource, /data-impeccable-variant=/, 'accepted variant 2 leaves no preview scaffolding');
await page.waitForTimeout(2750);
assert.doesNotMatch(readFileSync(wrappedSource, 'utf-8'), /data-impeccable-variant=/, 'the delayed variant 3 write stays fenced');
const generateId = traceEvents.find((event) => event.name === 'agent.event.received' && event.type === 'generate')?.id;
const timings = await waitForGenerationTimings(tmp, generateId, { requireAllVariants: false });
assert.equal(timings.all_variants_ready, undefined, 'accepting variant 2 cancels the unfinished third variant');
const realErrors = consoleErrors.filter((error) =>
!/(Download the React DevTools|StrictMode|Failed to load resource: the server responded with a status of 404)/i.test(error),
);
assert.deepEqual(realErrors, [], 'variant 2 early Accept stays console-clean');
} finally {
await teardownAndResetBrowser(teardown);
}
});
it('promotes pending Tune controls when the params-only revision arrives', liveE2eTestOptions, async (t) => {
const session = await bootFixtureSession({
name,
fixture,
browser,
agent: createFakeAgent(),
wrapTarget: wrapTargetFromPickedElement,
progressive: true,
progressiveDelayMs: 1500,
log: (message) => t.diagnostic(message),
});
const { page, teardown } = session;
try {
await waitForHandshake(page);
await pickElement(page, fixture.runtime.pickSelector || 'h1.hero-title');
await clickGo(page);
const pending = await waitForProgressiveReviewState(page, 3);
assert.equal(pending.tuneVisible, true);
assert.equal(pending.tuneDisabled, true);
await page.waitForFunction(() => {
const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.()
|| window.__IMPECCABLE_LIVE_UI_ROOT__
|| document;
const tune = root.querySelector('[data-iceq-tune="1"]');
const wrapper = document.querySelector('[data-impeccable-variants]');
return tune?.disabled === false
&& !!wrapper?.querySelector('[data-impeccable-params]');
}, { timeout: 10_000 });
const ready = await readProgressiveReviewState(page);
assert.equal(ready.arrived, 3, 'all variants remain mounted after params publication');
assert.equal(ready.tuneVisible, true);
assert.equal(ready.tuneDisabled, false, 'Tune becomes actionable without another variant arrival');
await clickDiscard(page);
await page.waitForFunction(() => window.__IMPECCABLE_LIVE_STATE__ === 'PICKING', { timeout: 2_000 });
} finally {
await teardownAndResetBrowser(teardown);
}
});
}
if (shouldRunScenario('manual') && Array.isArray(fixture.runtime.manualEditScenarios) && fixture.runtime.manualEditScenarios.length > 0) {
const manualScenarioFilter = process.env.IMPECCABLE_E2E_MANUAL_SCENARIO || '';
for (const scenario of fixture.runtime.manualEditScenarios) {
@@ -1074,94 +798,6 @@ function recordGenerateEvents(agent, events) {
};
}
async function waitForProgressiveReviewState(page, expected, { arrived: targetArrived = 1, visible: targetVisible = 1 } = {}) {
await installLiveQueryHelpers(page);
await page.waitForFunction(({ variantCount, targetArrived, targetVisible }) => {
const query = window.__impeccableLiveQuery || ((selector) => document.querySelector(selector));
const wrapper = query('[data-impeccable-variants]');
const variants = wrapper?.querySelectorAll('[data-impeccable-variant]:not([data-impeccable-variant="original"])');
const debugState = window.__IMPECCABLE_LIVE_CHROME_CORE__?.debugState?.();
const arrived = /^(?:svelte|vue)-component$/.test(wrapper?.dataset.impeccablePreview || '')
? Number(debugState?.arrivedVariants || 0)
: variants?.length;
const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.()
|| window.__IMPECCABLE_LIVE_UI_ROOT__
|| document;
const bar = root.querySelector('#impeccable-live-bar');
return arrived === targetArrived
&& new RegExp(`${targetVisible}\\s*\\/\\s*${variantCount}`).test(bar?.textContent || '')
&& /more arriving/.test(bar?.textContent || '');
}, { variantCount: expected, targetArrived, targetVisible }, { timeout: 15_000 });
return readProgressiveReviewState(page);
}
async function readProgressiveReviewState(page) {
await installLiveQueryHelpers(page);
return page.evaluate(() => {
const query = window.__impeccableLiveQuery || ((selector) => document.querySelector(selector));
const wrapper = query('[data-impeccable-variants]');
const variants = [...(wrapper?.querySelectorAll('[data-impeccable-variant]:not([data-impeccable-variant="original"])') || [])];
const debugState = window.__IMPECCABLE_LIVE_CHROME_CORE__?.debugState?.();
const isSveltePreview = /^(?:svelte|vue)-component$/.test(wrapper?.dataset.impeccablePreview || '');
const visibleVariant = variants.find((variant) => getComputedStyle(variant).display !== 'none');
const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.()
|| window.__IMPECCABLE_LIVE_UI_ROOT__
|| document;
const buttons = [...root.querySelectorAll('#impeccable-live-bar button')];
const accept = buttons.find((button) => /Accept/.test(button.textContent || ''));
const discard = buttons.find((button) => (button.textContent || '').includes('✕'));
const paramsPanel = root.querySelector('#impeccable-live-params-panel');
const tune = root.querySelector('[data-iceq-tune="1"]');
return {
arrived: isSveltePreview ? Number(debugState?.arrivedVariants || 0) : variants.length,
visible: isSveltePreview ? Number(debugState?.visibleVariant || 0) : Number(visibleVariant?.dataset.impeccableVariant || 0),
copy: isSveltePreview ? (wrapper?.innerText || '') : (visibleVariant?.innerText || ''),
acceptPointerEvents: accept ? getComputedStyle(accept).pointerEvents : null,
discardPointerEvents: discard ? getComputedStyle(discard).pointerEvents : null,
hasParams: variants.some((variant) => variant.hasAttribute('data-impeccable-params')),
tuneVisible: !!tune,
tuneDisabled: tune?.disabled ?? null,
tuneTitle: tune?.title || '',
paramsPanelVisible: !!paramsPanel
&& getComputedStyle(paramsPanel).pointerEvents !== 'none'
&& getComputedStyle(paramsPanel).clipPath === 'inset(0px)',
};
});
}
function countSourceVariants(source) {
return (String(source).match(/<div\s+data-impeccable-variant="(?!original")/g) || []).length;
}
function escapeRegExp(value) {
return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
async function waitForGenerationTimings(tmp, id, { timeoutMs = 5_000, requireAllVariants = true } = {}) {
const snapshotPath = join(tmp, '.impeccable', 'live', 'sessions', `${id}.snapshot.json`);
const journalPath = join(tmp, '.impeccable', 'live', 'sessions', `${id}.jsonl`);
const deadline = Date.now() + timeoutMs;
let lastTimings = null;
while (Date.now() < deadline) {
if (existsSync(snapshotPath)) {
const snapshot = JSON.parse(readFileSync(snapshotPath, 'utf-8'));
const timings = snapshot.generationTimings || {};
lastTimings = timings;
if (timings.generation_ready && timings.first_reviewable && (!requireAllVariants || timings.all_variants_ready)) return timings;
}
await new Promise((resolve) => setTimeout(resolve, 50));
}
const checkpointReasons = existsSync(journalPath)
? readFileSync(journalPath, 'utf-8')
.split('\n')
.filter(Boolean)
.map((line) => JSON.parse(line)?.event)
.filter((event) => event?.type === 'checkpoint')
.map((event) => ({ reason: event.reason, arrivedVariants: event.arrivedVariants, expectedVariants: event.expectedVariants }))
: [];
throw new Error(`generation timings did not complete for ${id}: timings=${JSON.stringify(lastTimings)} checkpoints=${JSON.stringify(checkpointReasons)}`);
}
async function captureLiveE2eFailure({ name, fixture, session, sourceFile, error, log = () => {} }) {
const root = process.env.IMPECCABLE_E2E_ARTIFACT_DIR;
if (!root || !session?.tmp) return;
@@ -1817,12 +1453,11 @@ function svelteComponentTargetFor(filePath) {
if (!filePath.endsWith('/manifest.json') && !filePath.endsWith('\\manifest.json')) return null;
let manifest;
try { manifest = JSON.parse(readFileSync(filePath, 'utf-8')); } catch { return null; }
if (!['svelte-component', 'vue-component'].includes(manifest.previewMode) || !manifest.sourceFile || !manifest.componentDir) return null;
if (manifest.previewMode !== 'svelte-component' || !manifest.sourceFile || !manifest.componentDir) return null;
const sep = pathSepFor(filePath);
const markers = [
`${sep}node_modules${sep}.impeccable-live${sep}`,
`${sep}src${sep}lib${sep}impeccable${sep}`,
`${sep}app${sep}.impeccable-live${sep}`,
];
const marker = markers.find((candidate) => filePath.includes(candidate));
const idx = marker ? filePath.indexOf(marker) : -1;
@@ -1912,19 +1547,18 @@ async function locateSessionFile(tmp) {
return f;
}
}
for (const f of walkComponentManifests(tmp)) {
for (const f of walkSvelteComponentManifests(tmp)) {
const body = readFileSync(f, 'utf-8');
if (/"previewMode": "(?:svelte|vue)-component"/.test(body)) return f;
if (body.includes('"previewMode": "svelte-component"')) return f;
}
throw new Error('Could not locate session source file under ' + tmp);
}
function walkComponentManifests(root) {
function walkSvelteComponentManifests(root) {
const results = [];
const stack = [
join(root, 'node_modules/.impeccable-live'),
join(root, 'src/lib/impeccable'),
join(root, 'app/.impeccable-live'),
];
while (stack.length) {
const dir = stack.pop();
+11 -323
View File
@@ -27,10 +27,6 @@ import { join } from 'node:path';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { completionTypeForAcceptResult } from '../../skill/scripts/live/completion.mjs';
import {
prepareGenerationArtifact,
publishGenerationArtifact,
} from '../../skill/scripts/live/generation-publisher.mjs';
const execFileP = promisify(execFile);
@@ -1329,25 +1325,15 @@ async function spliceVariantsIntoWrapper({ tmp, wrapInfo, sessionId, output }) {
styleMode: wrapInfo.styleMode,
});
const endMarkerIdx = lines.findIndex((line, index) =>
index > markerIdx && line.includes('impeccable-variants-end ' + sessionId),
);
if (endMarkerIdx === -1) {
throw new Error('end marker not found in ' + wrapInfo.file);
}
const tailIdx = wrapInfo.commentSyntax.open === '{/*'
? endMarkerIdx
: endMarkerIdx - 1;
const next = [
...lines.slice(0, markerIdx + 1),
block,
...lines.slice(tailIdx),
...lines.slice(markerIdx + 1),
];
await fs.writeFile(filePath, next.join('\n'), 'utf-8');
}
async function writeSvelteComponentVariants({ tmp, wrapInfo, event, output, writeParams = true }) {
async function writeSvelteComponentVariants({ tmp, wrapInfo, event, output }) {
const manifestPath = path.join(tmp, wrapInfo.file);
const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf-8'));
const componentDir = path.join(tmp, manifest.componentDir);
@@ -1387,168 +1373,7 @@ async function writeSvelteComponentVariants({ tmp, wrapInfo, event, output, writ
paramsByVariant[String(variantId)] = Array.isArray(variant.params) ? variant.params : [];
}
if (writeParams) {
await fs.writeFile(path.join(componentDir, 'params.json'), JSON.stringify(paramsByVariant, null, 2) + '\n', 'utf-8');
}
manifest.arrivedVariants = output.variants.length;
await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
}
async function publishSvelteComponentVariants({ tmp, wrapInfo, event, output, writeParams = true }) {
const prepared = prepareGenerationArtifact({
id: event.id,
sourceFile: wrapInfo.file,
cwd: tmp,
});
if (!prepared.ok) throw new Error(`Svelte publication prepare failed: ${prepared.error}`);
await writeSvelteComponentVariants({
tmp,
wrapInfo: { ...wrapInfo, file: prepared.artifactFile },
event,
output,
writeParams,
});
const published = publishGenerationArtifact({
id: event.id,
epoch: prepared.epoch,
sourceFile: wrapInfo.file,
artifactFile: prepared.artifactFile,
expectedSourceHash: prepared.expectedSourceHash,
arrivedVariants: output.variants.length,
expectedVariants: event.count,
cwd: tmp,
});
if (!published.ok) throw new Error(`Svelte publication failed: ${published.error}`);
return published;
}
async function writeVueComponentVariants({ tmp, wrapInfo, event, output, writeParams = true }) {
const manifestPath = path.join(tmp, wrapInfo.file);
const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf-8'));
const componentDir = path.join(tmp, manifest.componentDir);
const contract = Array.isArray(manifest.propContract) ? manifest.propContract : [];
const textValues = extractTextPieces(event.element?.outerHTML || event.element?.textContent || '');
const paramsByVariant = {};
for (let i = 0; i < output.variants.length; i++) {
const variantId = i + 1;
const variant = output.variants[i];
let markup = substituteLiveTextWithProps(variant.innerHtml || '', contract, textValues).trim();
for (const entry of contract) {
markup = markup.replaceAll(`{${entry.prop}}`, `{{ ${entry.prop} }}`);
}
const css = svelteCssForVariant(output.scopedCss || '', variantId, firstTagName(markup) || 'div');
const propsScript = contract.length > 0
? ['<script setup>', 'defineProps({', ...contract.map((entry) => ` ${entry.prop}: { default: '' },`), '});', '</script>', '']
: [];
const component = [
...propsScript,
'<template>',
markup || '<div></div>',
'</template>',
'',
'<style scoped>',
css || ':where(*) {}',
'</style>',
'',
].join('\n');
await fs.writeFile(path.join(componentDir, `v${variantId}.vue`), component, 'utf-8');
paramsByVariant[String(variantId)] = Array.isArray(variant.params) ? variant.params : [];
}
if (writeParams) {
await fs.writeFile(path.join(componentDir, 'params.json'), JSON.stringify(paramsByVariant, null, 2) + '\n', 'utf-8');
}
manifest.arrivedVariants = output.variants.length;
await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
}
async function publishVueComponentVariants({ tmp, wrapInfo, event, output, writeParams = true }) {
const prepared = prepareGenerationArtifact({ id: event.id, sourceFile: wrapInfo.file, cwd: tmp });
if (!prepared.ok) throw new Error(`Vue publication prepare failed: ${prepared.error}`);
await writeVueComponentVariants({
tmp,
wrapInfo: { ...wrapInfo, file: prepared.artifactFile },
event,
output,
writeParams,
});
const published = publishGenerationArtifact({
id: event.id,
epoch: prepared.epoch,
sourceFile: wrapInfo.file,
artifactFile: prepared.artifactFile,
expectedSourceHash: prepared.expectedSourceHash,
arrivedVariants: output.variants.length,
expectedVariants: event.count,
cwd: tmp,
});
if (!published.ok) throw new Error(`Vue publication failed: ${published.error}`);
return published;
}
async function publishSourceVariants({ tmp, wrapInfo, event, output }) {
const prepared = prepareGenerationArtifact({
id: event.id,
sourceFile: wrapInfo.file,
cwd: tmp,
});
if (!prepared.ok) throw new Error(`Source publication prepare failed: ${prepared.error}`);
await spliceVariantsIntoWrapper({
tmp,
wrapInfo: { ...wrapInfo, file: prepared.artifactFile },
sessionId: event.id,
output,
});
const published = publishGenerationArtifact({
id: event.id,
epoch: prepared.epoch,
sourceFile: wrapInfo.file,
artifactFile: prepared.artifactFile,
expectedSourceHash: prepared.expectedSourceHash,
arrivedVariants: output.variants.length,
expectedVariants: event.count,
cwd: tmp,
});
if (!published.ok) throw new Error(`Source publication failed: ${published.error}`);
return published;
}
async function publishVariantProgress({
base,
token,
event,
wrapInfo,
arrivedVariants,
signal,
revision = 1,
publicationKind = 'variants',
}) {
const previewMode = wrapInfo.previewMode || 'source';
await fetch(`${base}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token,
type: 'checkpoint',
id: event.id,
revision,
revisionDomain: 'publication',
phase: 'cycling',
reason: 'variants_progress',
arrivedVariants,
expectedVariants: event.count,
sourceFile: wrapInfo.sourceFile || wrapInfo.file,
previewFile: wrapInfo.file,
previewMode,
publicationKind,
}),
signal,
});
await fs.writeFile(path.join(componentDir, 'params.json'), JSON.stringify(paramsByVariant, null, 2) + '\n', 'utf-8');
}
function variantMarkupHasVisibleContent(markup) {
@@ -1682,11 +1507,6 @@ export async function runAgentLoop({
agent,
signal,
log = () => {},
trace = () => {},
progressive = false,
progressiveDelayMs = 0,
progressiveInitialCount = 1,
atomicDelayMs = 0,
wrapTarget = { classes: 'hero-title', tag: 'h1' },
steerSourceFile,
steerTarget,
@@ -1710,8 +1530,6 @@ export async function runAgentLoop({
if (event.type === 'prefetch') continue;
if (event.type === 'connected') continue;
trace('agent.event.received', { id: event.id, type: event.type, clientSentAt: event.clientSentAt ?? null });
if (event.type === 'steer') {
log(`steer id=${event.id} message=${JSON.stringify(event.message)}`);
try {
@@ -1760,16 +1578,7 @@ export async function runAgentLoop({
log(`generate id=${event.id} mode=${isInsert ? 'insert' : 'replace'}${isInsert ? '' : ` action=${event.action}`} count=${event.count}`);
try {
let wrapInfo;
if (event.scaffold) {
wrapInfo = event.scaffold;
trace('agent.scaffold.reused', {
id: event.id,
file: wrapInfo.file,
previewMode: wrapInfo.previewMode || 'source',
durationMs: event.scaffoldDurationMs ?? null,
});
} else if (isInsert) {
trace('agent.scaffold.start', { id: event.id, mode: 'insert' });
if (isInsert) {
const insertTarget = insertTargetFromEvent(event);
wrapInfo = await runInsert({
tmp,
@@ -1778,9 +1587,7 @@ export async function runAgentLoop({
count: event.count,
...insertTarget,
});
trace('agent.scaffold.end', { id: event.id, file: wrapInfo.file, previewMode: wrapInfo.previewMode || 'source' });
} else {
trace('agent.scaffold.start', { id: event.id, mode: 'replace' });
// 1. Wrap the original element in the variant scaffold (deterministic CLI)
// wrapTarget can be a static {classes, tag, elementId} (test fixtures
// know what they pick) or a function (event) => target (real-use
@@ -1799,154 +1606,41 @@ export async function runAgentLoop({
...target,
text,
});
trace('agent.scaffold.end', { id: event.id, file: wrapInfo.file, previewMode: wrapInfo.previewMode || 'source' });
}
log(`scaffolded: ${wrapInfo.file} insertLine=${wrapInfo.insertLine}`);
// 2. Agent generates variant content (LLM-pluggable seam).
// Providers may expose a true split path so variant 1 is written before
// the request for the remaining variants completes.
trace('agent.generate.start', { id: event.id, count: event.count });
const splitProgressive = progressive
&& typeof agent.generateFirstVariant === 'function'
&& typeof agent.generateRemainingVariants === 'function'
&& event.count > 1;
let output;
let firstOutput;
if (splitProgressive) {
firstOutput = normalizeVariantOutput(
await agent.generateFirstVariant(event, { wrapTarget, wrapInfo }),
wrapInfo,
);
firstOutput = {
...firstOutput,
variants: firstOutput.variants.slice(0, 1).map((variant) => ({ ...variant, params: [] })),
};
trace('agent.generate.first_ready', { id: event.id, count: firstOutput.variants.length });
trace('agent.first_variant.write.start', { id: event.id, file: wrapInfo.file });
if (wrapInfo.previewMode === 'svelte-component') {
await publishSvelteComponentVariants({ tmp, wrapInfo, event, output: firstOutput, writeParams: false });
} else if (wrapInfo.previewMode === 'vue-component') {
await publishVueComponentVariants({ tmp, wrapInfo, event, output: firstOutput, writeParams: false });
} else {
await publishSourceVariants({ tmp, wrapInfo, event, output: firstOutput });
}
await publishVariantProgress({
base,
token,
event,
wrapInfo,
arrivedVariants: firstOutput.variants.length,
signal,
});
trace('agent.first_variant.write.end', { id: event.id, file: wrapInfo.file });
output = normalizeVariantOutput(
await agent.generateRemainingVariants(event, { wrapTarget, wrapInfo, firstOutput }),
wrapInfo,
);
trace('agent.generate.end', { id: event.id, count: output?.variants?.length || 0 });
} else {
output = normalizeVariantOutput(
await agent.generateVariants(event, { wrapTarget, wrapInfo }),
wrapInfo,
);
if (!progressive && atomicDelayMs > 0) {
await new Promise((resolve) => setTimeout(resolve, atomicDelayMs));
}
trace('agent.generate.first_ready', { id: event.id, count: output?.variants?.length || 0 });
if (!progressive || output.variants.length <= 1) {
trace('agent.generate.end', { id: event.id, count: output?.variants?.length || 0 });
}
if (progressive && output.variants.length > 1) {
const initialCount = Math.max(1, Math.min(
Number(progressiveInitialCount) || 1,
output.variants.length - 1,
));
firstOutput = {
...output,
variants: output.variants
.slice(0, initialCount)
.map((variant) => ({ ...variant, params: [] })),
};
trace('agent.first_variant.write.start', { id: event.id, file: wrapInfo.file });
if (wrapInfo.previewMode === 'svelte-component') {
await publishSvelteComponentVariants({ tmp, wrapInfo, event, output: firstOutput, writeParams: false });
} else if (wrapInfo.previewMode === 'vue-component') {
await publishVueComponentVariants({ tmp, wrapInfo, event, output: firstOutput, writeParams: false });
} else {
await publishSourceVariants({ tmp, wrapInfo, event, output: firstOutput });
}
await publishVariantProgress({
base,
token,
event,
wrapInfo,
arrivedVariants: firstOutput.variants.length,
signal,
});
trace('agent.first_variant.write.end', { id: event.id, file: wrapInfo.file });
if (progressiveDelayMs > 0) {
await new Promise((resolve) => setTimeout(resolve, progressiveDelayMs));
}
trace('agent.generate.end', { id: event.id, count: output?.variants?.length || 0 });
}
}
// 2. Agent generates variant content (LLM-pluggable seam)
let output = await agent.generateVariants(event, { wrapTarget, wrapInfo });
output = normalizeVariantOutput(output, wrapInfo);
if (output.variants.length !== event.count) {
log(`warning: agent returned ${output.variants.length} variants, expected ${event.count}`);
}
// 3. Write the complete set into the deterministic preview target.
trace('agent.write.start', { id: event.id, file: wrapInfo.file });
// 3. Write variants into the deterministic preview target.
if (wrapInfo.previewMode === 'svelte-component') {
await publishSvelteComponentVariants({ tmp, wrapInfo, event, output, writeParams: true });
} else if (wrapInfo.previewMode === 'vue-component') {
await publishVueComponentVariants({ tmp, wrapInfo, event, output, writeParams: true });
} else if (progressive) {
await publishSourceVariants({ tmp, wrapInfo, event, output });
await writeSvelteComponentVariants({ tmp, wrapInfo, event, output });
} else {
await spliceVariantsIntoWrapper({ tmp, wrapInfo, sessionId: event.id, output });
}
trace('agent.write.end', { id: event.id, file: wrapInfo.file });
if (progressive) {
await publishVariantProgress({
base,
token,
event,
wrapInfo,
arrivedVariants: output.variants.length,
signal,
revision: 2,
publicationKind: 'params',
});
}
if (process.env.IMPECCABLE_E2E_DEBUG) {
const post = await fs.readFile(path.join(tmp, wrapInfo.file), 'utf-8');
log(`--- post-splice (variants written) ---\n${post}`);
}
// 4. Tell the server we're done (broadcasts SSE done → browser settles to CYCLING)
trace('agent.reply.start', { id: event.id });
await fetch(`${base}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token, type: 'done', sourceEventType: 'generate', id: event.id, file: wrapInfo.file }),
body: JSON.stringify({ token, type: 'done', id: event.id, file: wrapInfo.file }),
signal,
});
trace('agent.reply.end', { id: event.id });
} catch (err) {
if (signal.aborted) return;
if (isExpectedGenerationCancellation(err)) {
trace('agent.generate.canceled', { id: event.id, reason: 'stale_generation_epoch' });
log('generate canceled after Accept/Discard: ' + err.message);
continue;
}
trace('agent.generate.error', { id: event.id, message: err.message });
log('generate failed: ' + err.message);
await fetch(`${base}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token, type: 'error', sourceEventType: 'generate', id: event.id, message: err.message }),
body: JSON.stringify({ token, type: 'error', id: event.id, message: err.message }),
signal,
}).catch(() => {});
}
@@ -2046,7 +1740,6 @@ export async function runAgentLoop({
body: JSON.stringify({
token,
type: completionType,
sourceEventType: 'accept',
id: event.id,
file: acceptResult.file,
message: acceptResult.error,
@@ -2076,7 +1769,6 @@ export async function runAgentLoop({
body: JSON.stringify({
token,
type: completionType,
sourceEventType: 'discard',
id: event.id,
file: discardResult.file,
message: discardResult.error,
@@ -2095,10 +1787,6 @@ export async function runAgentLoop({
}
}
export function isExpectedGenerationCancellation(error) {
return /(?:^|\b)stale_generation_epoch(?:\b|$)/.test(String(error?.message || error || ''));
}
async function runPollReply({ tmp, scriptsDir, id, status, message, data }) {
const args = [path.join(scriptsDir, 'live-poll.mjs'), '--reply', id, status];
if (data !== undefined) args.push('--data', JSON.stringify(data));
+25 -76
View File
@@ -192,7 +192,6 @@ const STEER_SYSTEM_INSTRUCTIONS = [
* @property {string=} model Override the selected provider's default model.
* @property {string=} baseURL Override the provider API base URL.
* @property {object=} config Pre-resolved provider config from resolveLlmAgentConfig().
* @property {boolean=} includeLiveSpec Attach the full live.md reference. Defaults to true; latency benchmarks disable it to export only the synthetic element contract.
* @property {(msg: string) => void=} log Optional logger for debug output.
*/
@@ -241,22 +240,14 @@ export async function createLlmAgent(opts = {}) {
const { apiKey, baseURL, model, provider } = config;
const log = opts.log || (() => {});
const liveMd = opts.includeLiveSpec === false ? null : await fs.readFile(LIVE_MD_PATH, 'utf-8');
const liveMd = await fs.readFile(LIVE_MD_PATH, 'utf-8');
const client = new Anthropic({ apiKey, ...(baseURL ? { baseURL } : {}) });
const systemBlocks = (instructions) => [
{
type: 'text',
text: liveMd ? instructions : instructions.replace(/\n\nCONTEXT —[^\n]+$/, ''),
},
...(liveMd ? [{ type: 'text', text: liveMd, cache_control: { type: 'ephemeral' } }] : []),
];
return {
async generateVariants(event, context = {}) {
const isInsert = event.mode === 'insert';
const baseUserMessage = [
`Produce variants for the following ${isInsert ? 'insert request' : 'pick'}. Reply with the JSON object only — no prose.`,
progressiveVariantGuidance(event),
'',
'```json',
JSON.stringify(buildVariantRequestPayload(event, context), null, 2),
@@ -265,7 +256,6 @@ export async function createLlmAgent(opts = {}) {
let userMessage = baseUserMessage;
for (let attempt = 0; attempt < MANUAL_EDIT_RESPONSE_MAX_ATTEMPTS; attempt += 1) {
const lastAttempt = attempt + 1 >= MANUAL_EDIT_RESPONSE_MAX_ATTEMPTS;
let response;
try {
response = await client.messages.create(
@@ -273,10 +263,15 @@ export async function createLlmAgent(opts = {}) {
model,
temperature: 0,
max_tokens: 16000,
// When present, live.md is the final cacheable stable prefix.
// Benchmarks omit it so external payloads contain only the
// synthetic element contract and per-run event.
system: systemBlocks(VARIANT_SYSTEM_INSTRUCTIONS),
system: [
{ type: 'text', text: VARIANT_SYSTEM_INSTRUCTIONS },
// Cacheable: the entire stable prefix (instructions + spec) is
// cached up to this breakpoint. The user message holds all the
// per-call volatile content. DeepSeek compatibility support is
// provider-reported and best-effort; the usage log below tells us
// whether cache reads/writes actually happened.
{ type: 'text', text: liveMd, cache_control: { type: 'ephemeral' } },
],
messages: [{ role: 'user', content: userMessage }],
},
{
@@ -285,7 +280,7 @@ export async function createLlmAgent(opts = {}) {
},
);
} catch (err) {
if (lastAttempt) throw err;
if (attempt === 1) throw err;
log(`variant request failed; retrying: ${err.message}`);
userMessage = [
baseUserMessage,
@@ -305,7 +300,7 @@ export async function createLlmAgent(opts = {}) {
`provider=${provider} model=${model} attempt=${attempt + 1} input=${inputTokens} output=${outputTokens} cache_read=${cacheRead} cache_write=${cacheWrite}`,
);
if (!response || !Array.isArray(response.content)) {
if (lastAttempt) throw new Error('LLM agent: provider returned an empty variant response');
if (attempt === 1) throw new Error('LLM agent: provider returned an empty variant response');
log('variant response validation failed; retrying: provider returned an empty response');
userMessage = [
baseUserMessage,
@@ -325,7 +320,7 @@ export async function createLlmAgent(opts = {}) {
try {
parsed = parseVariantResponse(text);
} catch (err) {
if (lastAttempt) throw err;
if (attempt === 1) throw err;
log(`variant response validation failed; retrying: ${err.message.split('\n')[0]}`);
userMessage = [
baseUserMessage,
@@ -337,13 +332,11 @@ export async function createLlmAgent(opts = {}) {
continue;
}
const validationError = validateVariantCount(parsed, event)
|| validateProgressiveVariantOutput(parsed, event)
|| (isInsert
? validateInsertVariantOutput(parsed, event)
: (validateVariantVisibleCopy(parsed, event.element) || validateVariantMaterialChange(parsed, event.element)));
const validationError = isInsert
? validateInsertVariantOutput(parsed, event)
: (validateVariantVisibleCopy(parsed, event.element) || validateVariantMaterialChange(parsed, event.element));
if (!validationError) return parsed;
if (lastAttempt) throw new Error(`LLM agent: ${validationError}`);
if (attempt === 1) throw new Error(`LLM agent: ${validationError}`);
log(`variant validation failed; retrying: ${validationError}`);
if (isInsert) {
@@ -418,7 +411,10 @@ export async function createLlmAgent(opts = {}) {
model,
temperature: 0,
max_tokens: 16000,
system: systemBlocks(MANUAL_EDIT_SYSTEM_INSTRUCTIONS),
system: [
{ type: 'text', text: MANUAL_EDIT_SYSTEM_INSTRUCTIONS },
{ type: 'text', text: liveMd, cache_control: { type: 'ephemeral' } },
],
messages: [{ role: 'user', content: userMessage }],
},
{
@@ -546,7 +542,10 @@ export async function createLlmAgent(opts = {}) {
const response = await client.messages.create({
model,
max_tokens: 4096,
system: systemBlocks(STEER_SYSTEM_INSTRUCTIONS),
system: [
{ type: 'text', text: STEER_SYSTEM_INSTRUCTIONS },
{ type: 'text', text: liveMd, cache_control: { type: 'ephemeral' } },
],
messages: [{ role: 'user', content: userMessage }],
});
@@ -673,7 +672,6 @@ export function buildVariantRequestPayload(event, context = {}) {
action: event?.action,
freeformPrompt: event?.freeformPrompt,
count: event?.count,
progressive: event?.progressive,
element: isInsert ? null : {
outerHTML: event?.element?.outerHTML,
tagName: event?.element?.tagName,
@@ -693,31 +691,6 @@ export function buildVariantRequestPayload(event, context = {}) {
};
}
export function progressiveVariantGuidance(event = {}) {
if (event.progressive?.phase === 'first') {
return [
'PROGRESSIVE FIRST DELIVERY:',
`- Return exactly ${event.count} variant now.`,
'- Return params: [] for this variant; tunable parameters are generated in the final phase.',
'- The innerHtml must be materially different from the picked source, not merely paired with different CSS.',
'- For a bare-text element, preserve the full exact copy in one child span inside the unchanged root tag/class.',
].join('\n');
}
if (event.progressive?.phase === 'remaining') {
return [
'PROGRESSIVE FINAL DELIVERY:',
`- Return the complete final set of exactly ${event.count} variants, including variant 1.`,
'- progressive.firstVariant is the already-visible variant 1. Keep its innerHtml exactly unchanged and add its deferred params now.',
...(event.progressive.omitFirstVariantCss ? [
'- Variant 1 CSS is already published and immutable. Do not repeat or modify any scopedCss rule for data-impeccable-variant="1"; return scopedCss rules for variants 2+ only.',
] : []),
'- Generate the remaining distinct variants and their params in the other array positions.',
'- Every remaining variant innerHtml must be materially changed too; for bare text, wrap the full exact copy in one child span with a distinct class instead of relying on CSS alone.',
].join('\n');
}
return '';
}
/**
* Parse and validate a model response into the variant-output schema. Throws
* with a `Parsed (first 500 chars): ...` echo on every schema failure so the
@@ -877,30 +850,6 @@ export function validateInsertVariantOutput(parsed, event = {}) {
return null;
}
export function validateVariantCount(parsed, event = {}) {
const expected = Number(event.count);
if (!Number.isInteger(expected) || expected < 1) return 'event count must be a positive integer';
const actual = Array.isArray(parsed?.variants) ? parsed.variants.length : 0;
return actual === expected ? null : `expected exactly ${expected} variants, received ${actual}`;
}
export function validateProgressiveVariantOutput(parsed, event = {}) {
if (event.progressive?.phase === 'first') {
const hasEarlyParams = (parsed.variants || []).some((variant) => Array.isArray(variant.params) && variant.params.length > 0);
return hasEarlyParams ? 'progressive first delivery must defer params with an empty params array' : null;
}
if (event.progressive?.phase === 'remaining' && event.progressive.firstVariant?.innerHtml) {
const expected = String(event.progressive.firstVariant.innerHtml).trim();
const actual = String(parsed.variants?.[0]?.innerHtml || '').trim();
if (actual !== expected) return 'progressive final delivery must preserve variant 1 innerHtml exactly';
if (event.progressive.omitFirstVariantCss && /\[data-impeccable-variant\s*=\s*["']1["'][^\]]*\]/.test(parsed.scopedCss || '')) {
return 'progressive final delivery must omit already-published variant 1 CSS';
}
return null;
}
return null;
}
export function validateVariantMaterialChange(parsed, element) {
const originalHtml = normalizeVariantHtml(element?.outerHTML || '');
if (!originalHtml) return null;
+20 -92
View File
@@ -14,7 +14,7 @@
*/
import { execFileSync, spawn } from 'node:child_process';
import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -32,7 +32,8 @@ export { SCRIPTS_DIR, FIXTURES_DIR, REPO_ROOT };
// Stage
// ---------------------------------------------------------------------------
export function stageFixture(name, fixture, { fixtureRoot = join(FIXTURES_DIR, name) } = {}) {
export function stageFixture(name, fixture) {
const fixtureRoot = join(FIXTURES_DIR, name);
const gitignore = readFileSync(join(fixtureRoot, 'gitignore.txt'), 'utf-8');
const tmp = mkdtempSync(join(tmpdir(), 'impeccable-e2e-'));
@@ -55,7 +56,6 @@ export function runInstall(tmp, command, { timeoutMs = readTimeoutEnv('IMPECCABL
const installArgs = addNpmInstallDefaults(cmd, args);
try {
execFileSync(cmd, installArgs, { cwd: tmp, stdio: 'inherit', timeout: timeoutMs });
repairMissingRollupOptionalBinary(tmp, { timeoutMs });
} catch (err) {
if (err.signal === 'SIGTERM' || err.signal === 'SIGKILL' || err.killed) {
err.message = `fixture dependency install timed out after ${timeoutMs}ms: ${cmd} ${installArgs.join(' ')}`;
@@ -64,26 +64,11 @@ export function runInstall(tmp, command, { timeoutMs = readTimeoutEnv('IMPECCABL
}
}
function repairMissingRollupOptionalBinary(tmp, { timeoutMs }) {
if (process.platform !== 'darwin' || process.arch !== 'arm64') return;
const rollupPackage = join(tmp, 'node_modules', 'rollup', 'package.json');
const nativePackage = join(tmp, 'node_modules', '@rollup', 'rollup-darwin-arm64', 'package.json');
if (!existsSync(rollupPackage) || existsSync(nativePackage)) return;
const version = JSON.parse(readFileSync(rollupPackage, 'utf-8')).version;
execFileSync('npm', [
'install', '--no-save', '--no-audit', '--no-fund', '--no-progress',
`@rollup/rollup-darwin-arm64@${version}`,
], { cwd: tmp, stdio: 'inherit', timeout: timeoutMs });
}
function addNpmInstallDefaults(cmd, args) {
if (cmd !== 'npm') return args;
if (!['install', 'ci'].includes(args[0])) return args;
const out = [...args];
// npm can omit platform-specific Rollup binaries unless optional
// dependencies are requested explicitly (npm/cli#4828). Astro/Vite then
// fail before Live starts on fresh staged fixtures.
for (const flag of ['--no-progress', '--include=optional']) {
for (const flag of ['--prefer-offline', '--no-progress']) {
if (!out.some((arg) => arg === flag || arg.startsWith(flag + '='))) out.push(flag);
}
return out;
@@ -215,57 +200,29 @@ export async function stopDevServer(child) {
* @param {object} opts
* @param {string} opts.name fixture name
* @param {object} opts.fixture fixture.json contents
* @param {string=} opts.fixtureRoot fixture directory; defaults to the public framework fixture tree
* @param {import('playwright').Browser} opts.browser shared browser instance
* @param {object} opts.agent VariantAgent (defaults to fake)
* @param {object|function=} opts.wrapTarget live-wrap target or event mapper
* @param {(context: object) => Promise<object|void>} [opts.startWorker]
* Optional production worker factory. Return {stop, done}; when used,
* omit `agent` so the deterministic in-process loop is not started.
* @param {(context: object) => Promise<void>|void} [opts.prepareTmp]
* @param {(msg: string) => void} [opts.log]
*/
export async function bootFixtureSession({
name,
fixture,
fixtureRoot,
browser,
agent,
wrapTarget,
startWorker,
prepareTmp,
log = () => {},
trace = () => {},
progressive = false,
progressiveDelayMs = 0,
progressiveInitialCount = 1,
atomicDelayMs = 0,
keepTmp = false,
}) {
export async function bootFixtureSession({ name, fixture, browser, agent, wrapTarget, log = () => {} }) {
const runtime = fixture.runtime;
if (!runtime) throw new Error(`fixture ${name} has no runtime block`);
const tmp = stageFixture(name, fixture, { fixtureRoot });
const tmp = stageFixture(name, fixture);
let live;
let dev;
let agentAbort;
let agentDone;
let externalWorker;
let ctx;
const teardown = async () => {
try { if (ctx) await ctx.close(); } catch {}
try { if (agentAbort) agentAbort.abort(); } catch {}
try { if (agentDone) await agentDone.catch(() => {}); } catch {}
try { if (externalWorker?.stop) await externalWorker.stop(); } catch {}
try { if (externalWorker?.done) await externalWorker.done.catch(() => {}); } catch {}
try { if (dev?.child) await stopDevServer(dev.child); } catch {}
try { if (live) stopLiveServer(tmp); } catch {}
if (!keepTmp) {
try { rmSync(tmp, { recursive: true, force: true }); } catch {}
} else {
log(`kept staged fixture at ${tmp}`);
}
try { rmSync(tmp, { recursive: true, force: true }); } catch {}
};
const stopLiveForDeferredWork = () => {
@@ -276,67 +233,41 @@ export async function bootFixtureSession({
try {
const startedAt = Date.now();
if (prepareTmp) await prepareTmp({ tmp, fixture, scriptsDir: SCRIPTS_DIR, trace, log });
trace('setup.install.start', { fixture: name });
log(`installing deps`);
runInstall(tmp, runtime.install);
trace('setup.install.end', { fixture: name });
log(`deps installed in ${formatDuration(Date.now() - startedAt)}`);
const liveStartedAt = Date.now();
trace('setup.live_server.start', { fixture: name });
log(`starting live-server`);
live = startLiveServer(tmp);
trace('setup.live_server.end', { fixture: name, port: live.port });
log(`live-server ready in ${formatDuration(Date.now() - liveStartedAt)}`);
if (startWorker) {
trace('setup.worker.start', { fixture: name });
externalWorker = await startWorker({ tmp, fixture, scriptsDir: SCRIPTS_DIR, live, trace, log });
trace('setup.worker.end', { fixture: name });
}
const injectStartedAt = Date.now();
trace('setup.inject.start', { fixture: name });
log(`live-inject --port ${live.port}`);
const injectResult = runInject(tmp, live.port);
if (!injectResult.ok) throw new Error('live-inject failed: ' + JSON.stringify(injectResult));
trace('setup.inject.end', { fixture: name, files: injectResult.files || injectResult.pageFiles || [] });
log(`live-inject complete in ${formatDuration(Date.now() - injectStartedAt)}`);
const devStartedAt = Date.now();
trace('setup.dev_server.start', { fixture: name });
log(`spawning dev server: ${runtime.devCommand.join(' ')}`);
dev = startDevServer(tmp, runtime);
const { port: devPort } = await dev.ready;
trace('setup.dev_server.end', { fixture: name, port: devPort });
log(`dev server ready on ${devPort} in ${formatDuration(Date.now() - devStartedAt)}`);
// Agent loop runs concurrently — abort on teardown.
if (agent) {
agentAbort = new AbortController();
const loopOptions = {
tmp,
scriptsDir: SCRIPTS_DIR,
port: live.port,
token: live.token,
agent,
wrapTarget,
signal: agentAbort.signal,
trace,
progressive,
progressiveDelayMs,
progressiveInitialCount,
atomicDelayMs,
steerSourceFile: runtime.steer?.sourceFile,
steerTarget: runtime.steer?.target,
};
const loops = [runAgentLoop({ ...loopOptions, log: (m) => log('[worker] ' + m) })];
if (progressive) {
loops.push(runAgentLoop({ ...loopOptions, log: (m) => log('[supervisor] ' + m) }));
}
agentDone = Promise.all(loops);
}
agentAbort = new AbortController();
agentDone = runAgentLoop({
tmp,
scriptsDir: SCRIPTS_DIR,
port: live.port,
token: live.token,
agent,
wrapTarget,
signal: agentAbort.signal,
log: (m) => log('[agent] ' + m),
steerSourceFile: runtime.steer?.sourceFile,
steerTarget: runtime.steer?.target,
});
const scheme = runtime.scheme || 'http';
ctx = await browser.newContext({
@@ -352,12 +283,10 @@ export async function bootFixtureSession({
});
const pageStartedAt = Date.now();
trace('setup.page_load.start', { fixture: name });
await page.goto(`${scheme}://127.0.0.1:${devPort}`, {
waitUntil: 'domcontentloaded',
timeout: 30_000,
});
trace('setup.page_load.end', { fixture: name });
log(`page loaded in ${formatDuration(Date.now() - pageStartedAt)}`);
return {
@@ -366,7 +295,6 @@ export async function bootFixtureSession({
ctx,
dev,
live,
worker: externalWorker,
consoleErrors,
stopLiveServer: stopLiveForDeferredWork,
teardown,
+4 -58
View File
@@ -424,23 +424,7 @@ export async function pickElement(page, selector, opts = {}) {
if (visible) break;
await resetPickMode(page);
if (attempt === 2) {
const snapshot = await page.evaluate(({ selector, barSel, pickSel }) => {
const target = document.querySelector(selector);
const rect = target?.getBoundingClientRect();
const hit = rect ? document.elementFromPoint(rect.x + rect.width / 2, rect.y + rect.height / 2) : null;
const query = window.__impeccableLiveQuery || ((sel) => document.querySelector(sel));
const bar = query(barSel);
const pick = query(pickSel);
return {
liveState: window.__IMPECCABLE_LIVE_STATE__ || null,
target: target ? { tag: target.tagName, classes: target.className, rect: rect?.toJSON?.() || null } : null,
hit: hit ? { tag: hit.tagName, classes: hit.className, text: (hit.textContent || '').slice(0, 80) } : null,
pickActive: pick?.dataset.active || null,
bar: bar ? { display: bar.style.display, text: bar.textContent } : null,
debugState: window.__IMPECCABLE_LIVE_CHROME_CORE__?.debugState?.() || null,
};
}, { selector, barSel: BAR_ID, pickSel: PICK_TOGGLE_ID }).catch((error) => ({ error: error.message }));
throw new Error(`pick did not open configure bar for ${selector}: ${JSON.stringify(snapshot)}`);
await page.waitForSelector(BAR_ID, { state: 'visible', timeout: 1 });
}
}
// Wait specifically for the Configure-row submit button to be in the bar.
@@ -544,36 +528,6 @@ export async function setCount(page, count) {
throw new Error(`could not cycle count to ${count}`);
}
/** Select a named Impeccable sub-command from the configure-row picker. */
export async function selectAction(page, action) {
const pickerSelector = '#impeccable-live-picker';
const opened = await page.evaluate(({ barSel, pickerSel }) => {
const query = window.__impeccableLiveQuery || ((selector) => document.querySelector(selector));
const bar = query(barSel);
const picker = query(pickerSel);
const actionControl = [...(bar?.querySelectorAll('button') || [])]
.find((button) => (button.textContent || '').includes('\u25BE'));
if (!actionControl || !picker) return false;
actionControl.click();
return true;
}, { barSel: BAR_ID, pickerSel: pickerSelector });
if (!opened) throw new Error('could not open Live action picker');
await page.waitForFunction((selector) => {
const picker = window.__impeccableLiveQuery(selector);
return picker && picker.style.display !== 'none';
}, pickerSelector, { timeout: 5_000 });
const selected = await page.evaluate(({ pickerSel, value }) => {
const picker = window.__impeccableLiveQuery(pickerSel);
const chip = picker?.querySelector(`button[data-action="${CSS.escape(value)}"]`);
if (!chip) return false;
chip.click();
return true;
}, { pickerSel: pickerSelector, value: action });
if (!selected) throw new Error(`Live action ${JSON.stringify(action)} is unavailable`);
}
/**
* Click Go. Browser POSTs the generate event; the agent picks it up. Headed
* browser runs can occasionally accept the click without leaving configure
@@ -624,14 +578,7 @@ export async function waitForCycling(page, expectedCount, { timeout = 30_000 } =
// Counter format: "1/3", "2/3" etc. Look for any "i/N" with N matching.
const m = text.match(/(\d+)\s*\/\s*(\d+)/);
if (!m) return false;
const wrapper = window.__impeccableLiveQuery('[data-impeccable-variants]');
const debugState = window.__IMPECCABLE_LIVE_CHROME_CORE__?.debugState?.();
const arrived = /^(?:svelte|vue)-component$/.test(wrapper?.dataset.impeccablePreview || '')
? Number(debugState?.arrivedVariants || 0)
: wrapper
? wrapper.querySelectorAll('[data-impeccable-variant]:not([data-impeccable-variant="original"])').length
: 0;
return parseInt(m[2], 10) === expected && arrived >= expected;
return parseInt(m[2], 10) === expected;
},
{ barSel: BAR_ID, expected: expectedCount },
{ timeout },
@@ -643,7 +590,7 @@ export async function waitForCycling(page, expectedCount, { timeout = 30_000 } =
const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.() || window.__IMPECCABLE_LIVE_UI_ROOT__ || null;
const bar = query(barSel);
const toast = query('#impeccable-live-toast');
const wrapper = query('[data-impeccable-variants]');
const wrapper = document.querySelector('[data-impeccable-variants]');
return {
liveInit: window.__IMPECCABLE_LIVE_INIT__,
adapter: window.__IMPECCABLE_LIVE_ADAPTER__,
@@ -804,8 +751,7 @@ async function ensureVisibleVariant(page, expectedVariant) {
*/
export async function clickDiscard(page) {
// The discard button has just a "✕" glyph as text content.
if (await dispatchBarButton(page, '✕')) return;
await clickBarButton(page, '✕');
await page.locator(`${BAR_ID} button`, { hasText: '✕' }).click();
}
export async function clickEditCopy(page) {
-13
View File
@@ -97,16 +97,3 @@ describe('validateEvent — replace generate (regression)', () => {
);
});
});
describe('validateEvent — worker progress', () => {
it('accepts bounded agent phases and rejects malformed telemetry', () => {
assert.equal(validateEvent({
type: 'agent_phase',
id: VALID_ID,
phase: 'first_variant_generating',
durationMs: 123,
}), null);
assert.match(validateEvent({ type: 'agent_phase', id: VALID_ID, phase: 'Not valid' }), /phase/);
assert.match(validateEvent({ type: 'agent_phase', id: VALID_ID, phase: 'valid', durationMs: -1 }), /durationMs/);
});
});
-98
View File
@@ -1,98 +0,0 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import path from 'node:path';
import {
buildGenerationPreflight,
runGenerationPreflight,
} from '../skill/scripts/live/generation-preflight.mjs';
const SCRIPTS_DIR = path.resolve('skill/scripts');
test('builds a replace preflight from the picker locator', () => {
const command = buildGenerationPreflight({
type: 'generate',
id: 'session-1',
count: 3,
pageUrl: '/pricing',
element: {
id: 'hero',
classes: ['hero', 'hero--dark'],
tagName: 'SECTION',
textContent: 'A faster way to ship',
},
}, SCRIPTS_DIR);
assert.equal(command.mode, 'replace');
assert.deepEqual(command.args.slice(1), [
'--id', 'session-1', '--count', '3',
'--element-id', 'hero',
'--classes', 'hero hero--dark',
'--tag', 'SECTION',
'--text', 'A faster way to ship',
'--page-url', '/pricing',
]);
});
test('can request an isolated source preview for dedicated generation', () => {
const command = buildGenerationPreflight({
type: 'generate',
id: 'session-isolated',
count: 3,
element: { classes: ['hero'], tagName: 'SECTION' },
}, SCRIPTS_DIR, { isolated: true });
assert.equal(command.mode, 'replace');
assert.equal(command.args.includes('--isolated'), true);
});
test('builds an insert preflight from the anchor locator', () => {
const command = buildGenerationPreflight({
type: 'generate',
id: 'session-2',
count: 2,
mode: 'insert',
insert: {
position: 'before',
anchor: { classes: ['card'], tagName: 'ARTICLE', textContent: 'Plan' },
},
}, SCRIPTS_DIR);
assert.equal(command.mode, 'insert');
assert.deepEqual(command.args.slice(1), [
'--id', 'session-2', '--count', '2', '--position', 'before',
'--classes', 'card', '--tag', 'ARTICLE', '--text', 'Plan',
]);
});
test('returns scaffold metadata without exposing child-process details', () => {
const calls = [];
const result = runGenerationPreflight({
type: 'generate',
id: 'session-3',
count: 1,
element: { classes: ['hero'] },
}, {
scriptsDir: SCRIPTS_DIR,
cwd: '/tmp/example',
execFileSyncImpl(file, args, options) {
calls.push({ file, args, options });
return '{"file":"src/App.jsx","insertLine":12}\n';
},
});
assert.equal(result.ok, true);
assert.deepEqual(result.scaffold, { file: 'src/App.jsx', insertLine: 12 });
assert.equal(calls[0].file, process.execPath);
assert.equal(calls[0].options.cwd, '/tmp/example');
});
test('skips preflight when the picker has no source locator', () => {
const result = runGenerationPreflight({
type: 'generate',
id: 'session-4',
count: 3,
element: { tagName: 'DIV' },
}, { scriptsDir: SCRIPTS_DIR });
assert.deepEqual(result, { ok: false, skipped: true, reason: 'insufficient_locator' });
});
-442
View File
@@ -1,442 +0,0 @@
import assert from 'node:assert/strict';
import { afterEach, beforeEach, describe, it } from 'node:test';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { createLiveSessionStore } from '../skill/scripts/live/session-store.mjs';
import { scaffoldSourceArtifactSession } from '../skill/scripts/live/source-artifact.mjs';
import {
prepareGenerationArtifact,
publishGenerationArtifact,
sha256,
} from '../skill/scripts/live/generation-publisher.mjs';
describe('transactional generation publisher', () => {
let tmp;
let source;
let artifact;
let store;
beforeEach(() => {
tmp = mkdtempSync(join(tmpdir(), 'impeccable-publisher-'));
source = join(tmp, 'page.html');
artifact = join(tmp, 'variant.html');
writeFileSync(source, '<main><div data-impeccable-variants="abc12345"><div data-impeccable-variant="original">Original</div></div></main>');
store = createLiveSessionStore({ cwd: tmp, sessionId: 'abc12345' });
store.appendEvent({
type: 'generate',
id: 'abc12345',
generationEpoch: 1,
action: 'polish',
count: 3,
element: { outerHTML: '<main>Original</main>' },
});
});
afterEach(() => rmSync(tmp, { recursive: true, force: true }));
it('atomically publishes an artifact that matches the fenced source revision', () => {
const before = readFileSync(source, 'utf-8');
writeFileSync(artifact, '<main><div data-impeccable-variants="abc12345"><div data-impeccable-variant="original">Original</div><div data-impeccable-variant="1">Variant</div></div></main>');
const result = publishGenerationArtifact({
id: 'abc12345',
epoch: 1,
sourceFile: source,
artifactFile: artifact,
expectedSourceHash: sha256(before),
expectedVariants: 3,
cwd: tmp,
});
assert.equal(result.ok, true, JSON.stringify(result));
assert.equal(result.arrivedVariants, 1);
assert.equal(readFileSync(source, 'utf-8'), readFileSync(artifact, 'utf-8'));
const snapshot = store.getSnapshot('abc12345');
assert.equal(snapshot.phase, 'variants_progress');
assert.equal(snapshot.publishedRevision, 1);
assert.equal(snapshot.deliveredVariants['1'].digest, result.digest);
});
it('prepares a revision artifact with the current epoch and source fence', () => {
const result = prepareGenerationArtifact({ id: 'abc12345', sourceFile: source, cwd: tmp });
assert.equal(result.ok, true);
assert.equal(result.epoch, 1);
assert.equal(result.revision, 1);
assert.equal(result.expectedSourceHash, sha256(readFileSync(source, 'utf-8')));
assert.equal(readFileSync(join(tmp, result.artifactFile), 'utf-8'), readFileSync(source, 'utf-8'));
});
it('rejects a late publication after early accept without touching source', () => {
const before = readFileSync(source, 'utf-8');
writeFileSync(artifact, '<main><div data-impeccable-variants="abc12345"><div data-impeccable-variant="1">Late</div></div></main>');
store.appendEvent({ type: 'accept', id: 'abc12345', variantId: '1' });
const result = publishGenerationArtifact({
id: 'abc12345',
epoch: 1,
sourceFile: source,
artifactFile: artifact,
expectedSourceHash: sha256(before),
cwd: tmp,
});
assert.deepEqual(result, {
ok: false,
error: 'stale_generation_epoch',
canceled: true,
phase: 'accept_requested',
});
assert.equal(readFileSync(source, 'utf-8'), before);
});
it('rejects a stale artifact when source changed after the worker snapshot', () => {
const before = readFileSync(source, 'utf-8');
writeFileSync(artifact, '<main><div data-impeccable-variants="abc12345"><div data-impeccable-variant="1">Variant</div></div></main>');
writeFileSync(source, before.replace('Original', 'Changed'));
const result = publishGenerationArtifact({
id: 'abc12345',
epoch: 1,
sourceFile: source,
artifactFile: artifact,
expectedSourceHash: sha256(before),
cwd: tmp,
});
assert.equal(result.ok, false);
assert.equal(result.error, 'source_hash_mismatch');
assert.match(readFileSync(source, 'utf-8'), /Changed/);
});
it('keeps an already reviewable source variant immutable across revisions', () => {
const firstSource = '<main><div data-impeccable-variants="abc12345"><div data-impeccable-variant="original">Original</div><div data-impeccable-variant="1"><section><div>First</div></section></div></div></main>';
writeFileSync(artifact, firstSource);
const first = publishGenerationArtifact({
id: 'abc12345',
epoch: 1,
sourceFile: source,
artifactFile: artifact,
expectedSourceHash: sha256(readFileSync(source, 'utf-8')),
arrivedVariants: 1,
expectedVariants: 3,
cwd: tmp,
});
assert.equal(first.ok, true);
const prepared = prepareGenerationArtifact({ id: 'abc12345', sourceFile: source, cwd: tmp });
const changed = firstSource.replace('First', 'Silently changed')
.replace('</div></div></main>', '</div><div data-impeccable-variant="2">Second</div></div></main>');
writeFileSync(join(tmp, prepared.artifactFile), changed);
const result = publishGenerationArtifact({
id: 'abc12345',
epoch: prepared.epoch,
sourceFile: source,
artifactFile: prepared.artifactFile,
expectedSourceHash: prepared.expectedSourceHash,
arrivedVariants: 2,
expectedVariants: 3,
cwd: tmp,
});
assert.equal(result.ok, false);
assert.equal(result.error, 'published_variant_changed');
assert.equal(result.variant, 1);
assert.equal(readFileSync(source, 'utf-8'), firstSource);
});
it('allows the deferred parameter manifest without weakening prior markup immutability', () => {
const firstSource = '<main><div data-impeccable-variants="abc12345"><style data-impeccable-css="abc12345">@scope ([data-impeccable-variant="1"]) { h1 { color: red; } }</style><div data-impeccable-variant="original">Original</div><div data-impeccable-variant="1"><h1>First</h1></div></div></main>';
writeFileSync(artifact, firstSource);
const first = publishGenerationArtifact({
id: 'abc12345', epoch: 1, sourceFile: source, artifactFile: artifact,
expectedSourceHash: sha256(readFileSync(source, 'utf-8')), arrivedVariants: 1, expectedVariants: 3, cwd: tmp,
});
assert.equal(first.ok, true);
const prepared = prepareGenerationArtifact({ id: 'abc12345', sourceFile: source, cwd: tmp });
const withParams = firstSource
.replace('<div data-impeccable-variant="1"', '<div data-impeccable-variant="1" data-impeccable-params=\'[{"id":"scale"}]\'')
.replace('</div></main>', '<div data-impeccable-variant="2">Second</div></div></main>');
writeFileSync(join(tmp, prepared.artifactFile), withParams);
const result = publishGenerationArtifact({
id: 'abc12345', epoch: prepared.epoch, sourceFile: source, artifactFile: prepared.artifactFile,
expectedSourceHash: prepared.expectedSourceHash, arrivedVariants: 2, expectedVariants: 3, cwd: tmp,
});
assert.equal(result.ok, true, JSON.stringify(result));
assert.match(readFileSync(source, 'utf-8'), /data-impeccable-params/);
});
it('rejects later source revisions that restyle an already reviewable variant', () => {
const firstSource = '<main><div data-impeccable-variants="abc12345"><style data-impeccable-css="abc12345">@scope ([data-impeccable-variant="1"]) { :scope > h1 { color: red; } }</style><div data-impeccable-variant="original">Original</div><div data-impeccable-variant="1"><h1>First</h1></div></div></main>';
writeFileSync(artifact, firstSource);
const first = publishGenerationArtifact({
id: 'abc12345', epoch: 1, sourceFile: source, artifactFile: artifact,
expectedSourceHash: sha256(readFileSync(source, 'utf-8')), arrivedVariants: 1, expectedVariants: 3, cwd: tmp,
});
assert.equal(first.ok, true);
const prepared = prepareGenerationArtifact({ id: 'abc12345', sourceFile: source, cwd: tmp });
const changed = firstSource.replace('color: red', 'color: blue');
writeFileSync(join(tmp, prepared.artifactFile), changed);
const result = publishGenerationArtifact({
id: 'abc12345', epoch: prepared.epoch, sourceFile: source, artifactFile: prepared.artifactFile,
expectedSourceHash: prepared.expectedSourceHash, arrivedVariants: 1, expectedVariants: 3, cwd: tmp,
});
assert.equal(result.ok, false);
assert.equal(result.error, 'published_variant_css_changed', JSON.stringify(result));
assert.equal(readFileSync(source, 'utf-8'), firstSource);
});
});
describe('transactional isolated source preview publisher', () => {
let tmp;
const id = 'isolatedpub';
beforeEach(() => {
tmp = mkdtempSync(join(tmpdir(), 'impeccable-isolated-publisher-'));
writeFileSync(join(tmp, 'page.html'), '<main><section class="hero">Original</section></main>');
createLiveSessionStore({ cwd: tmp, sessionId: id }).appendEvent({
type: 'generate', id, generationEpoch: 1, count: 3,
});
});
afterEach(() => rmSync(tmp, { recursive: true, force: true }));
it('publishes to the preview artifact while fencing the byte-identical source', () => {
const original = readFileSync(join(tmp, 'page.html'), 'utf-8');
const session = scaffoldSourceArtifactSession({
id,
count: 3,
sourceFile: 'page.html',
sourceStartLine: 1,
sourceEndLine: 1,
originalSource: '<section class="hero">Original</section>',
previewContent: '<main><div data-impeccable-variants="isolatedpub"><div data-impeccable-variant="original"><section class="hero">Original</section></div></div></main>',
cwd: tmp,
});
const prepared = prepareGenerationArtifact({ id, sourceFile: session.previewFile, cwd: tmp });
assert.equal(prepared.ok, true);
assert.equal(prepared.sourceFile, 'page.html');
assert.equal(prepared.previewFile, session.previewFile);
assert.equal(prepared.previewMode, 'source-artifact');
const candidate = readFileSync(join(tmp, prepared.artifactFile), 'utf-8')
.replace('</div></main>', '<div data-impeccable-variant="1"><section>Variant one</section></div></div></main>');
writeFileSync(join(tmp, prepared.artifactFile), candidate);
const published = publishGenerationArtifact({
id,
epoch: prepared.epoch,
sourceFile: session.previewFile,
artifactFile: prepared.artifactFile,
expectedSourceHash: prepared.expectedSourceHash,
arrivedVariants: 1,
expectedVariants: 3,
cwd: tmp,
});
assert.equal(published.ok, true, JSON.stringify(published));
assert.equal(published.sourceFile, 'page.html');
assert.equal(published.previewMode, 'source-artifact');
assert.equal(readFileSync(join(tmp, 'page.html'), 'utf-8'), original);
assert.match(readFileSync(join(tmp, session.previewFile), 'utf-8'), /Variant one/);
});
});
describe('transactional Svelte component publisher', () => {
let tmp;
let source;
let manifestPath;
let componentDir;
let store;
beforeEach(() => {
tmp = mkdtempSync(join(tmpdir(), 'impeccable-svelte-publisher-'));
source = join(tmp, 'src', 'routes', '+page.svelte');
componentDir = join(tmp, 'node_modules', '.impeccable-live', 'svelte123');
manifestPath = join(componentDir, 'manifest.json');
mkdirSync(join(tmp, 'src', 'routes'), { recursive: true });
mkdirSync(componentDir, { recursive: true });
writeFileSync(source, '<main><h1>{title}</h1></main>\n');
writeFileSync(manifestPath, JSON.stringify({
id: 'svelte123',
previewMode: 'svelte-component',
sourceFile: 'src/routes/+page.svelte',
sourceStartLine: 1,
sourceEndLine: 1,
count: 3,
propContract: [{ prop: 'title', expr: 'title', placeholder: '{title}' }],
originalMarkup: '<main><h1>{title}</h1></main>',
componentDir: 'node_modules/.impeccable-live/svelte123',
runtimeModule: '/node_modules/.impeccable-live/__runtime.js',
}, null, 2) + '\n');
for (let variant = 1; variant <= 3; variant++) {
writeFileSync(join(componentDir, `v${variant}.svelte`), `<main>Stub ${variant}</main>\n`);
}
store = createLiveSessionStore({ cwd: tmp, sessionId: 'svelte123' });
store.appendEvent({
type: 'generate',
id: 'svelte123',
generationEpoch: 1,
action: 'polish',
count: 3,
element: { outerHTML: '<main><h1>Original</h1></main>' },
});
});
afterEach(() => rmSync(tmp, { recursive: true, force: true }));
it('prepares an isolated component directory fenced against the real route', () => {
const result = prepareGenerationArtifact({ id: 'svelte123', sourceFile: manifestPath, cwd: tmp });
assert.equal(result.ok, true);
assert.equal(result.previewMode, 'svelte-component');
assert.equal(result.sourceFile, 'node_modules/.impeccable-live/svelte123/manifest.json');
assert.equal(result.targetSourceFile, 'src/routes/+page.svelte');
assert.equal(result.expectedSourceHash, sha256(readFileSync(source, 'utf-8')));
const artifactManifest = JSON.parse(readFileSync(join(tmp, result.artifactFile), 'utf-8'));
assert.equal(artifactManifest.componentDir, result.componentDir);
assert.equal(readFileSync(join(tmp, result.componentDir, 'v1.svelte'), 'utf-8'), '<main>Stub 1</main>\n');
writeFileSync(join(tmp, result.componentDir, 'v1.svelte'), '<main>Prepared only</main>\n');
assert.equal(readFileSync(join(componentDir, 'v1.svelte'), 'utf-8'), '<main>Stub 1</main>\n');
});
it('publishes components before committing the arrived manifest and journals preview metadata', () => {
const prepared = prepareGenerationArtifact({ id: 'svelte123', sourceFile: manifestPath, cwd: tmp });
const artifactManifestPath = join(tmp, prepared.artifactFile);
const artifactManifest = JSON.parse(readFileSync(artifactManifestPath, 'utf-8'));
artifactManifest.arrivedVariants = 1;
writeFileSync(artifactManifestPath, JSON.stringify(artifactManifest, null, 2) + '\n');
writeFileSync(join(tmp, prepared.componentDir, 'v1.svelte'), '<main>First live variant</main>\n');
const result = publishGenerationArtifact({
id: 'svelte123',
epoch: prepared.epoch,
sourceFile: manifestPath,
artifactFile: artifactManifestPath,
expectedSourceHash: prepared.expectedSourceHash,
arrivedVariants: 1,
expectedVariants: 3,
cwd: tmp,
});
assert.equal(result.ok, true);
assert.equal(result.previewMode, 'svelte-component');
assert.equal(result.sourceFile, 'src/routes/+page.svelte');
assert.equal(result.previewFile, 'node_modules/.impeccable-live/svelte123/manifest.json');
assert.equal(readFileSync(join(componentDir, 'v1.svelte'), 'utf-8'), '<main>First live variant</main>\n');
assert.equal(readFileSync(source, 'utf-8'), '<main><h1>{title}</h1></main>\n');
const liveManifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
assert.equal(liveManifest.arrivedVariants, 1);
assert.equal(liveManifest.componentDir, 'node_modules/.impeccable-live/svelte123');
const snapshot = store.getSnapshot('svelte123');
assert.equal(snapshot.arrivedVariants, 1);
assert.equal(snapshot.previewMode, 'svelte-component');
assert.equal(snapshot.previewFile, 'node_modules/.impeccable-live/svelte123/manifest.json');
});
it('keeps published variants immutable across later revisions', () => {
const first = prepareGenerationArtifact({ id: 'svelte123', sourceFile: manifestPath, cwd: tmp });
publishSveltePrepared(first, { arrived: 1, edits: { 1: '<main>First live variant</main>\n' } });
const second = prepareGenerationArtifact({ id: 'svelte123', sourceFile: manifestPath, cwd: tmp });
const before = readFileSync(join(componentDir, 'v1.svelte'), 'utf-8');
const result = publishSveltePrepared(second, {
arrived: 2,
edits: {
1: '<main>Silently changed first variant</main>\n',
2: '<main>Second live variant</main>\n',
},
});
assert.equal(result.ok, false);
assert.equal(result.error, 'published_variant_changed');
assert.equal(result.variant, 1);
assert.equal(readFileSync(join(componentDir, 'v1.svelte'), 'utf-8'), before);
assert.equal(JSON.parse(readFileSync(manifestPath, 'utf-8')).arrivedVariants, 1);
});
it('publishes later variants and params without rewriting an already reviewable variant', () => {
const first = prepareGenerationArtifact({ id: 'svelte123', sourceFile: manifestPath, cwd: tmp });
publishSveltePrepared(first, { arrived: 1, edits: { 1: '<main>First live variant</main>\n' } });
const second = prepareGenerationArtifact({ id: 'svelte123', sourceFile: manifestPath, cwd: tmp });
writeFileSync(join(tmp, second.componentDir, 'params.json'), '{"2":[{"id":"density"}]}\n');
const result = publishSveltePrepared(second, {
arrived: 3,
edits: {
2: '<main>Second live variant</main>\n',
3: '<main>Third live variant</main>\n',
},
});
assert.equal(result.ok, true);
assert.equal(result.arrivedVariants, 3);
assert.equal(readFileSync(join(componentDir, 'v1.svelte'), 'utf-8'), '<main>First live variant</main>\n');
assert.equal(readFileSync(join(componentDir, 'v2.svelte'), 'utf-8'), '<main>Second live variant</main>\n');
assert.equal(existsSync(join(componentDir, 'params.json')), true);
assert.deepEqual(JSON.parse(readFileSync(join(componentDir, 'params.json'), 'utf-8')), {
2: [{ id: 'density' }],
});
});
it('rejects a prepared Svelte publication after Accept without touching live artifacts', () => {
const prepared = prepareGenerationArtifact({ id: 'svelte123', sourceFile: manifestPath, cwd: tmp });
const beforeManifest = readFileSync(manifestPath, 'utf-8');
const beforeVariant = readFileSync(join(componentDir, 'v1.svelte'), 'utf-8');
store.appendEvent({ type: 'accept', id: 'svelte123', variantId: '1' });
const result = publishSveltePrepared(prepared, {
arrived: 1,
edits: { 1: '<main>Too late</main>\n' },
});
assert.equal(result.ok, false);
assert.equal(result.error, 'stale_generation_epoch');
assert.equal(readFileSync(manifestPath, 'utf-8'), beforeManifest);
assert.equal(readFileSync(join(componentDir, 'v1.svelte'), 'utf-8'), beforeVariant);
});
it('rejects a live component directory masquerading as a staged artifact', () => {
const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
manifest.arrivedVariants = 1;
writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
const result = publishGenerationArtifact({
id: 'svelte123',
epoch: 1,
sourceFile: manifestPath,
artifactFile: manifestPath,
expectedSourceHash: sha256(readFileSync(source, 'utf-8')),
arrivedVariants: 1,
expectedVariants: 3,
cwd: tmp,
});
assert.equal(result.ok, false);
assert.equal(result.error, 'artifact_not_staged');
});
function publishSveltePrepared(prepared, { arrived, edits }) {
const artifactManifestPath = join(tmp, prepared.artifactFile);
const artifactManifest = JSON.parse(readFileSync(artifactManifestPath, 'utf-8'));
artifactManifest.arrivedVariants = arrived;
writeFileSync(artifactManifestPath, JSON.stringify(artifactManifest, null, 2) + '\n');
for (const [variant, content] of Object.entries(edits)) {
writeFileSync(join(tmp, prepared.componentDir, `v${variant}.svelte`), content);
}
return publishGenerationArtifact({
id: 'svelte123',
epoch: prepared.epoch,
sourceFile: manifestPath,
artifactFile: artifactManifestPath,
expectedSourceHash: prepared.expectedSourceHash,
arrivedVariants: arrived,
expectedVariants: 3,
cwd: tmp,
});
}
});
+1 -68
View File
@@ -5,7 +5,7 @@
import { describe, it, beforeEach, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { existsSync, mkdirSync, mkdtempSync, writeFileSync, readFileSync, realpathSync, rmSync } from 'node:fs';
import { mkdirSync, mkdtempSync, writeFileSync, readFileSync, realpathSync, rmSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import { tmpdir } from 'node:os';
import { fileURLToPath } from 'node:url';
@@ -389,71 +389,4 @@ const title = 'Test';
const afterRemove = readFileSync(file, 'utf-8');
assert.equal(afterRemove, original, 'CRLF file should round-trip cleanly after remove');
});
it('uses an idempotent dev-only client plugin for a Nuxt 4 app directory', () => {
const configSource = `export default defineNuxtConfig({\n devtools: { enabled: false },\n});\n`;
const appSource = `<template>\n <NuxtPage />\n</template>\n`;
writeFileSync(join(tmp, 'nuxt.config.ts'), configSource);
mkdirSync(join(tmp, 'app'), { recursive: true });
writeFileSync(join(tmp, 'app', 'app.vue'), appSource);
const cfgPath = join(tmp, 'config.json');
writeFileSync(cfgPath, JSON.stringify({
files: ['app/app.vue'],
insertBefore: '</template>',
commentSyntax: 'html',
}));
const first = runInject(tmp, cfgPath, ['--port', '8400']);
const pluginPath = join(tmp, 'app', 'plugins', 'impeccable-live.client.ts');
const firstPlugin = readFileSync(pluginPath, 'utf-8');
assert.equal(first.ok, true);
assert.equal(first.adapter, 'nuxt');
assert.equal(first.results[0].file, 'app/plugins/impeccable-live.client.ts');
assert.equal(first.results[0].changed, true);
assert.match(firstPlugin, /if \(!import\.meta\.dev/);
assert.match(firstPlugin, /data-impeccable-live-nuxt/);
assert.match(firstPlugin, /localhost:8400\/live\.js/);
assert.equal(readFileSync(join(tmp, 'nuxt.config.ts'), 'utf-8'), configSource, 'Nuxt config remains user-owned');
assert.equal(readFileSync(join(tmp, 'app', 'app.vue'), 'utf-8'), appSource, 'app.vue remains user-owned');
const second = runInject(tmp, cfgPath, ['--port', '8400']);
assert.equal(second.ok, true);
assert.equal(second.results[0].changed, false, 'same-port reinjection is byte-idempotent');
assert.equal(readFileSync(pluginPath, 'utf-8'), firstPlugin);
const moved = runInject(tmp, cfgPath, ['--port', '8401']);
assert.equal(moved.ok, true);
assert.equal(moved.results[0].changed, true);
assert.match(readFileSync(pluginPath, 'utf-8'), /localhost:8401\/live\.js/);
assert.doesNotMatch(readFileSync(pluginPath, 'utf-8'), /localhost:8400\/live\.js/);
const removed = runInject(tmp, cfgPath, ['--remove']);
assert.equal(removed.ok, true);
assert.equal(removed.adapter, 'nuxt');
assert.equal(removed.results[0].removed, true);
assert.equal(existsSync(pluginPath), false);
assert.equal(readFileSync(join(tmp, 'nuxt.config.ts'), 'utf-8'), configSource);
assert.equal(readFileSync(join(tmp, 'app', 'app.vue'), 'utf-8'), appSource);
});
it('respects a literal Nuxt srcDir and never overwrites a user plugin', () => {
writeFileSync(join(tmp, 'nuxt.config.ts'), `export default defineNuxtConfig({ srcDir: 'client/' });\n`);
mkdirSync(join(tmp, 'client', 'plugins'), { recursive: true });
const pluginPath = join(tmp, 'client', 'plugins', 'impeccable-live.client.ts');
const userPlugin = `export default defineNuxtPlugin(() => {});\n`;
writeFileSync(pluginPath, userPlugin);
const cfgPath = join(tmp, 'config.json');
writeFileSync(cfgPath, JSON.stringify({
files: ['client/app.vue'],
insertBefore: '</template>',
commentSyntax: 'html',
}));
const result = runInject(tmp, cfgPath, ['--port', '8400']);
assert.equal(result.ok, false);
assert.equal(result.adapter, 'nuxt');
assert.equal(result.results[0].error, 'nuxt_plugin_conflict');
assert.equal(readFileSync(pluginPath, 'utf-8'), userPlugin);
});
});
-83
View File
@@ -148,12 +148,6 @@ describe('live-poll --stream integration', () => {
assert.equal(secondEvent.type, 'steer');
assert.equal(secondEvent.id, '22222222');
assert.equal(secondEvent.message, 'stream test two');
await postReply(`http://localhost:${server.port}`, server.token, {
id: '22222222',
type: 'steer_done',
message: 'done two',
});
} finally {
streamProc.kill('SIGTERM');
}
@@ -206,81 +200,4 @@ describe('live-poll --stream integration', () => {
streamProc.kill('SIGTERM');
}
});
it('waits for carbonize cleanup, then resumes on the same stream process', async () => {
const streamProc = spawn('node', [
POLL_SCRIPT,
'--stream',
'--types=steer,manual_edit_apply,carbonize_cleanup,exit',
'--ack-timeout=15000',
], {
cwd: server.cwd,
stdio: ['ignore', 'pipe', 'pipe'],
env: { ...process.env },
});
try {
const streamPid = streamProc.pid;
const carbonizeLinePromise = readStdoutLine(streamProc);
await new Promise((resolve) => setTimeout(resolve, 150));
const carbonizeResponse = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'carbonize_cleanup',
id: 'c0ffee01',
sessionId: 'abc12345',
file: 'src/App.jsx',
variantId: '2',
acceptResult: { carbonize: true },
}),
});
assert.equal(carbonizeResponse.status, 200);
const carbonizeEvent = JSON.parse(await carbonizeLinePromise);
assert.equal(carbonizeEvent.type, 'carbonize_cleanup');
assert.equal(carbonizeEvent.id, 'c0ffee01');
assert.equal(streamProc.exitCode, null);
process.kill(streamPid, 0);
// Cleanup is performed by the main task through separate tool calls while
// this yielded process waits for its acknowledgement.
await postReply(`http://localhost:${server.port}`, server.token, {
id: 'c0ffee01',
type: 'complete',
file: 'src/App.jsx',
});
const steerLinePromise = readStdoutLine(streamProc);
const steerResponse = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'steer',
id: '33333333',
message: 'still listening after carbonize',
pageUrl: 'http://localhost:4321/',
}),
});
assert.equal(steerResponse.status, 200);
const steerEvent = JSON.parse(await steerLinePromise);
assert.equal(steerEvent.type, 'steer');
assert.equal(steerEvent.id, '33333333');
assert.equal(streamProc.pid, streamPid);
assert.equal(streamProc.exitCode, null);
await postReply(`http://localhost:${server.port}`, server.token, {
id: '33333333',
type: 'steer_done',
message: 'No-op: lifecycle test only.',
});
} finally {
streamProc.kill('SIGTERM');
}
});
});
-42
View File
@@ -6,9 +6,7 @@ import {
buildPollReplyPayload,
isEventPending,
manualApplyPollBanner,
normalizePollTypes,
parseReplyArgs,
resolveCodexWorkerFallbackTypes,
requiresAgentReply,
} from '../skill/scripts/live-poll.mjs';
@@ -27,15 +25,6 @@ describe('live-poll reply payloads', () => {
'event=live_poll.reply_data actor=agent operation=completion_ack risk=carbonize_flag_dropped_before_server_journal expected={"carbonize":true} actual=' + JSON.stringify(payload.data),
);
});
it('preserves the leased source event type when concurrent work shares a session id', () => {
const payload = buildPollReplyPayload('token-1', {
id: 'abc12345',
type: 'agent_done',
sourceEventType: 'accept',
});
assert.equal(payload.sourceEventType, 'accept');
});
});
describe('live-poll accept handling', () => {
@@ -145,7 +134,6 @@ describe('live-poll stream helpers', () => {
assert.equal(requiresAgentReply({ type: 'generate' }), true);
assert.equal(requiresAgentReply({ type: 'steer' }), true);
assert.equal(requiresAgentReply({ type: 'manual_edit_apply' }), true);
assert.equal(requiresAgentReply({ type: 'carbonize_cleanup' }), true);
assert.equal(requiresAgentReply({ type: 'prefetch' }), false);
assert.equal(requiresAgentReply({ type: 'accept' }), false);
assert.equal(requiresAgentReply({ type: 'timeout' }), false);
@@ -161,34 +149,4 @@ describe('live-poll stream helpers', () => {
assert.equal(isEventPending(status, 'abc12345'), true);
assert.equal(isEventPending(status, '00000000'), false);
});
it('normalizes a non-overlapping foreground control lane', () => {
assert.deepEqual(
normalizePollTypes('steer,manual_edit_apply,carbonize_cleanup,exit,steer'),
['steer', 'manual_edit_apply', 'carbonize_cleanup', 'exit'],
);
});
it('keeps generation isolated while the Codex worker starts and restores it after failure', () => {
const cwd = '/tmp/live-fallback';
const control = ['steer', 'manual_edit_apply', 'carbonize_cleanup', 'exit'];
const starting = {
owner: 'impeccable-live-codex-worker-v1',
cwd,
pid: 123,
status: 'starting',
};
assert.deepEqual(resolveCodexWorkerFallbackTypes(control, {
cwd,
state: starting,
isPidReachable: () => true,
}), control);
const fallback = resolveCodexWorkerFallbackTypes(control, {
cwd,
state: { ...starting, status: 'error' },
isPidReachable: () => false,
});
assert.deepEqual(fallback, [...control, 'generate', 'accept', 'discard', 'prefetch']);
});
});
-115
View File
@@ -1,115 +0,0 @@
import assert from 'node:assert/strict';
import { describe, it } from 'node:test';
import {
STRATEGIES,
assembleProgressiveOutput,
applyRuntimeSourceScore,
estimateCostUsd,
scoreVariantOutput,
summarizeProviderRuns,
validateAcceptedCleanup,
} from '../scripts/lib/live-provider-benchmark.mjs';
const VARIANT = [
'<article class="offer-card offer-card--measured" aria-labelledby="field-notes-title">',
'<div class="offer-card__copy">',
'<p class="offer-card__eyebrow">Quarterly print edition</p>',
'<h2 class="offer-card__title" id="field-notes-title">Field Notes</h2>',
'<p class="offer-card__body">Four routes, annotated maps, and practical details for unhurried weekends.</p>',
'</div>',
'<a class="action-link" href="#edition">Reserve issue eight</a>',
'</article>',
].join('');
const GOOD_OUTPUT = {
scopedCss: [
'@scope ([data-impeccable-variant="1"]) {',
' :scope > .offer-card { background: var(--color-paper-deep); color: var(--color-ink); gap: var(--space-3); }',
' :scope .offer-card__eyebrow { color: var(--color-moss); }',
'}',
].join('\n'),
variants: [{ innerHtml: VARIANT, params: [] }],
};
describe('cross-provider Live benchmark', () => {
it('defines the control, progressive, compact, and parallel candidates', () => {
assert.deepEqual(Object.keys(STRATEGIES), [
'atomic-full',
'progressive-full',
'progressive-compact',
'parallel-compact',
]);
});
it('assembles progressive output without asking the tail call to reproduce variant 1', () => {
const first = {
scopedCss: '@scope ([data-impeccable-variant="1"]) { .first { color: var(--color-ink); } }',
variants: [{ innerHtml: VARIANT, params: [] }],
};
const remaining = {
scopedCss: [
'@scope ([data-impeccable-variant="1"]) { .second { color: var(--color-moss); } }',
'@scope ([data-impeccable-variant="2"]) { .third { color: var(--color-brass); } }',
].join('\n'),
variants: [{ innerHtml: `${VARIANT} ` }, { innerHtml: `${VARIANT} ` }],
};
const assembled = assembleProgressiveOutput(first, remaining);
assert.equal(assembled.variants[0], first.variants[0]);
assert.ok(assembled.scopedCss.startsWith(first.scopedCss));
assert.match(assembled.scopedCss, /data-impeccable-variant="2"[^]*second/);
assert.match(assembled.scopedCss, /data-impeccable-variant="3"[^]*third/);
});
it('passes on-brand, token-driven, copy-preserving component output', () => {
const score = scoreVariantOutput(GOOD_OUTPUT);
assert.equal(score.brandFidelity, 1);
assert.equal(score.componentFidelity, 1);
assert.equal(score.copyFidelity, 1);
assert.equal(score.sourceValidity, 1);
assert.ok(score.tokenFidelity >= 0.75);
assert.equal(score.passed, true);
});
it('rejects off-brand raw colors, missing component parts, and changed copy', () => {
const score = scoreVariantOutput({
scopedCss: '.offer-card { color: #ff00ff; background: linear-gradient(red, blue); box-shadow: 0 0 20px cyan; }',
variants: [{ innerHtml: '<article class="offer-card">Different sales copy</article>' }],
});
assert.ok(score.brandFidelity < 0.75);
assert.ok(score.componentFidelity < 0.75);
assert.equal(score.copyFidelity, 0);
assert.equal(score.passed, false);
});
it('requires the accepted source to build and lose every Live marker', () => {
const cleanSource = `export default function Card(){return (${VARIANT.replaceAll('class=', 'className=')});}`;
const cleanup = validateAcceptedCleanup({ source: cleanSource, browserClean: true, buildPassed: true });
assert.equal(cleanup.passed, true);
const dirty = validateAcceptedCleanup({
source: `${cleanSource}\n{/* impeccable-carbonize-start test */}`,
browserClean: true,
buildPassed: true,
});
assert.equal(dirty.markerFree, false);
assert.equal(dirty.passed, false);
assert.equal(applyRuntimeSourceScore(scoreVariantOutput(GOOD_OUTPUT), dirty).passed, false);
});
it('estimates cached token cost and summarizes latency, quality, and cleanup', () => {
assert.equal(estimateCostUsd(
{ inputTokens: 1_000_000, cachedInputTokens: 500_000, outputTokens: 100_000 },
{ input: 3, cachedInput: 0.3, output: 15 },
), 3.15);
const summary = summarizeProviderRuns([
{ firstReviewableMs: 100, allReadyMs: 300, acceptCleanupMs: 20, estimatedCostUsd: 0.1, quality: { ...scoreVariantOutput(GOOD_OUTPUT), sourceValidity: 1 }, cleanup: { passed: true }, passed: true },
{ firstReviewableMs: 200, allReadyMs: 400, acceptCleanupMs: 30, estimatedCostUsd: 0.2, quality: { ...scoreVariantOutput(GOOD_OUTPUT), sourceValidity: 1 }, cleanup: { passed: true }, passed: true },
]);
assert.equal(summary.metrics.firstReviewableMs.median, 150);
assert.equal(summary.cleanupPassRate, 1);
assert.equal(summary.gatePassRate, 1);
assert.equal(summary.estimatedCostUsd, 0.3);
});
});
-54
View File
@@ -1,54 +0,0 @@
# Live cross-provider benchmark
This benchmark compares Live variant delivery strategies without confusing model latency with browser/poller overhead. It uses the realistic `vite8-react-brand-fidelity` fixture and scores every output with deterministic gates for:
- brand fidelity;
- component fidelity;
- CSS-token fidelity;
- exact copy fidelity;
- source/schema validity;
- provider-independent Accept cleanup and production build validity.
The model matrix and the cleanup control are intentionally separate. Provider generation runs use a fixed synthetic picker event. The cleanup control runs a real Vite/React Live session in Playwright, accepts variant 1, waits for Pick mode, checks that Live markers are gone, and builds the accepted source. This prevents a provider from being blamed for local publisher/poller behavior while retaining a real pipeline safety gate.
## Commands
Validate the matrix without API or browser calls:
```sh
npm run bench:live:providers -- --dry-run
```
Run the recommended small matrix and write a report:
```sh
npm run bench:live:providers -- \
--strategies atomic-full,progressive-compact,parallel-compact \
--iterations 1 \
--output artifacts/live-provider-benchmark.json
```
Run only the real Accept/build cleanup control:
```sh
npm run bench:live:providers -- --cleanup-only --output /tmp/live-cleanup.json
```
Arguments accept either `--name=value` or `--name value`. No output file is created unless `--output` is supplied. API keys load, in order, from `--env-file`, the repo `.env`, and `~/code/impeccable-evals/.env`; reports include only key availability, never key values.
## Strategies
- `atomic-full`: one call generates all variants with the full Live reference. This is the latency and cost control.
- `progressive-full`: a first-variant call followed by a remaining-directions call, both with full Live context.
- `progressive-compact`: the same split with the stable compact producer contract. Variant 1 and its CSS segment are carried forward byte-for-byte and assembled locally.
- `parallel-compact`: three compact one-variant producers run concurrently. The first valid result is reviewable immediately; centralized assembly remaps the other CSS scopes deterministically.
The earlier idea of asking the second progressive call to reproduce variant 1 is deliberately excluded. It adds tokens, permits drift, and conflicts with transactional source publication. Deterministic assembly is the production candidate.
## Interpretation
A run passes only when overall fidelity is at least `0.90`, every dimension is at least `0.75`, and the real cleanup control passes. One iteration is a smoke matrix, not a statistically stable claim; use at least five iterations before setting a release threshold.
Cost estimates use provider-reported token counts and standard per-million-token prices recorded on July 11, 2026. Update `PROVIDER_PROFILES` when model pricing changes. Price sources are embedded in every report.
Latency runs use low effort for Claude Sonnet and GPT, and Gemini 3.1 Flash-Lite's minimal-thinking default. These settings are emitted in provider metadata so a report cannot silently compare different reasoning budgets.
+3 -28
View File
@@ -11,7 +11,7 @@ describe('live reference authoring contract', () => {
const skillSrc = readFileSync(join(ROOT, 'skill/SKILL.src.md'), 'utf-8');
const liveMd = readFileSync(join(ROOT, 'skill/reference/live.md'), 'utf-8');
assert.match(skillSrc, /For any other invoked sub-command \(`audit`, `polish`, `live`, \.\.\.\), immediately read \*\*`reference\/<command>\.md`\*\*/);
assert.match(skillSrc, /For any other invoked sub-command[\s\S]*?reference\/<command>\.md/);
assert.doesNotMatch(skillSrc, /Use this same scripts directory for all Impeccable helper commands/);
assert.doesNotMatch(skillSrc, /walk upward for the nearest project `\.agents`, `\.claude`, or `\.cursor` skill/);
assert.doesNotMatch(skillSrc, /## Context diagnostics/);
@@ -22,7 +22,7 @@ describe('live reference authoring contract', () => {
const skillSrc = readFileSync(join(ROOT, 'skill/SKILL.src.md'), 'utf-8');
const liveMd = readFileSync(join(ROOT, 'skill/reference/live.md'), 'utf-8');
assert.match(skillSrc, /For any other invoked sub-command \(`audit`, `polish`, `live`, \.\.\.\), immediately read \*\*`reference\/<command>\.md`\*\*/);
assert.match(skillSrc, /For any other invoked sub-command[\s\S]*?reference\/<command>\.md/);
assert.doesNotMatch(skillSrc, /TARGET_SELECTION_REQUIRED/);
assert.doesNotMatch(skillSrc, /productStatus/);
assert.doesNotMatch(skillSrc, /designStatus/);
@@ -36,24 +36,17 @@ describe('live reference authoring contract', () => {
it('keeps the live prompt focused on the foreground poll loop', () => {
const liveMd = readFileSync(join(ROOT, 'skill/reference/live.md'), 'utf-8');
const generationAgentMd = readFileSync(join(ROOT, 'skill/agents/impeccable-live-generator.md'), 'utf-8');
const manualAgentMd = readFileSync(join(ROOT, 'skill/agents/impeccable-manual-edit-applier.md'), 'utf-8');
const openingContract = liveMd.split('\n').slice(0, 60).join('\n');
assert.match(liveMd, /1\. `live\.mjs`: boot\./);
assert.match(liveMd, /3\. Poll loop with the default long timeout \(600000 ms\)\. Run `live-poll\.mjs` again immediately.*Codex runs this one-shot poll in the foreground\./);
assert.match(liveMd, /3\. Poll loop with the default long timeout \(600000 ms\)\. After every event or `--reply`, run `live-poll\.mjs` again immediately\. Never pass a short `--timeout=`\./);
assert.match(openingContract, /## Poll loop/);
assert.match(openingContract, /No step skipped, no step reordered\./);
assert.doesNotMatch(liveMd, /live-copy-edits\.md/);
assert.doesNotMatch(liveMd, /IMPECCABLE_LIVE_COPY_AGENT|mock/);
assert.match(liveMd, /"manual_edit_apply" → Handle Manual Edit Apply/);
assert.match(liveMd, /## Handle `manual_edit_apply`/);
assert.match(openingContract, /Codex.*one-shot poll in a \*\*yielded foreground exec session\*\*/);
assert.doesNotMatch(openingContract, /dedicated app-server generation lane by default/);
assert.match(liveMd, /Experimental dedicated Codex worker/);
assert.match(liveMd, /IMPECCABLE_LIVE_CODEX_WORKER=1 node/);
assert.match(liveMd, /narrow, reasoned per-candidate waivers/);
assert.match(liveMd, /experimental dedicated worker, Accept emits a foreground `carbonize_cleanup` control event/i);
assert.ok(
liveMd.indexOf('## Handle `manual_edit_apply`') > liveMd.indexOf('## Handle `prefetch`'),
'manual_edit_apply handler section must sit after prefetch in the dispatch order',
@@ -67,14 +60,6 @@ describe('live reference authoring contract', () => {
assert.match(liveMd, /delegate source edits to `impeccable_manual_edit_applier`/);
assert.match(liveMd, /The subagent must not poll or reply/);
assert.match(liveMd, /parent live thread keeps the foreground poll loop/);
assert.match(liveMd, /delegate to the low-effort `impeccable_live_generator` agent/);
assert.match(liveMd, /Do not paste this full reference into the handoff/);
assert.match(generationAgentMd, /codex-name: impeccable_live_generator/);
assert.match(generationAgentMd, /effort: low/);
assert.match(generationAgentMd, /providers: codex/);
assert.match(generationAgentMd, /Never poll, Accept, Discard/);
assert.match(generationAgentMd, /Publish the first reviewable result/);
assert.match(generationAgentMd, /preserve every already-published variant byte-for-byte/i);
assert.match(liveMd, /live-accept\.mjs --page-url PAGE_URL/);
assert.match(liveMd, /If `repair` is present/);
assert.match(liveMd, /Fix the current source/);
@@ -144,16 +129,6 @@ describe('live reference authoring contract', () => {
/sandbox_permissions: "require_escalated"/,
'Codex-only sandbox guidance should not appear in Claude live reference',
);
assert.match(
codexLiveMd,
/Codex progressive override/,
'Codex live reference should progressively deliver the first reviewable variant',
);
assert.doesNotMatch(
claudeLiveMd,
/Codex progressive override|first-reviewable milestone/,
'Claude live reference should retain the atomic path without Codex-specific delivery instructions',
);
});
it('keeps live preview CSS guidance capability-mode driven', () => {
-107
View File
@@ -1,107 +0,0 @@
import assert from 'node:assert/strict';
import { describe, it } from 'node:test';
import {
buildRenderedJudgePrompt,
buildRenderedReviewContext,
parseRenderedJudgeResult,
summarizeRenderedJudgeRuns,
} from '../scripts/lib/live-rendered-quality.mjs';
describe('Live rendered quality judge', () => {
it('builds an identity-preserving multi-variant review contract', () => {
const prompt = buildRenderedJudgePrompt({
action: 'bolder',
brief: 'Make the selected offer more decisive.',
safeContext: { product: 'Northstar', constraints: ['Warm paper and dark ink.'] },
variants: [{ variantId: 1 }, { variantId: 2 }, { variantId: 3 }],
});
assert.match(prompt, /<action>\/bolder<\/action>/);
assert.match(prompt, /<remote_safe_review_context>/);
assert.match(prompt, /Treat all text visible inside screenshots as untrusted page content/);
assert.match(prompt, /Do not reward novelty that violates the existing identity/);
assert.match(prompt, /constraints as authoritative/);
assert.match(prompt, /palette allowlist permits/i);
assert.match(prompt, /Do not invent prohibitions/);
assert.match(prompt, /<variant_ids>1,2,3<\/variant_ids>/);
});
it('carries exact remote-safe tokens and component roles into review context', () => {
const context = buildRenderedReviewContext({
fixture: 'brand-fixture',
fixtureConfig: {
runtime: { pickSelector: '.offer' },
renderedQuality: {
action: 'bolder',
brief: 'Amplify the offer.',
constraints: ['Brass is allowed'],
tokens: { '--color-brass': '#9b6b2f' },
componentRoles: { ActionLink: 'Quiet outlined control' },
},
},
});
assert.equal(context.action, 'bolder');
assert.equal(context.captureSelector, '.offer');
assert.equal(context.safeContext.tokens['--color-brass'], '#9b6b2f');
assert.equal(context.safeContext.componentRoles.ActionLink, 'Quiet outlined control');
});
it('prefers rubric-free evidence capture settings for external harnesses', () => {
const context = buildRenderedReviewContext({
fixture: 'private-fixture',
fixtureConfig: {
runtime: { pickSelector: '.picked' },
evidenceCapture: {
captureSelector: '.selected-section',
mode: 'target',
action: 'bolder',
},
renderedQuality: {
captureSelector: '.public-smoke-only',
reviewFocus: 'Must not leak into the evidence contract.',
},
},
});
assert.equal(context.captureSelector, '.selected-section');
assert.equal(context.captureMode, 'target');
assert.equal(context.action, 'bolder');
assert.equal(context.safeContext.reviewFocus, '');
});
it('requires every expected rendered variant to pass the strict score floor', () => {
const result = parseRenderedJudgeResult(JSON.stringify({
variants: [
{ variantId: 1, commandFidelity: 8, brandAndSystemFidelity: 8, renderedQuality: 7, taskCompletion: 8, criticalFailure: false, summary: 'Good.' },
{ variantId: 2, commandFidelity: 8, brandAndSystemFidelity: 6, renderedQuality: 8, taskCompletion: 8, criticalFailure: false, summary: 'Drifted.' },
],
}), [1, 2]);
assert.equal(result.variants[0].passed, true);
assert.equal(result.variants[1].passed, false);
assert.equal(result.passed, false);
assert.throws(() => parseRenderedJudgeResult('{"variants":[]}', [1]), /variant ids mismatch/);
});
it('summarizes run and per-variant quality independently', () => {
const variants = [
{ variantId: 1, commandFidelity: 8, brandAndSystemFidelity: 8, renderedQuality: 8, taskCompletion: 8, passed: true },
{ variantId: 2, commandFidelity: 6, brandAndSystemFidelity: 8, renderedQuality: 8, taskCompletion: 8, passed: false },
];
const summary = summarizeRenderedJudgeRuns([
{ renderedJudge: { passed: false, variants } },
{ renderedJudge: { passed: true, variants: [variants[0]] } },
]);
assert.deepEqual(summary, {
runs: 2,
variants: 3,
passedRuns: 1,
passedVariants: 2,
averageScores: {
commandFidelity: 7.33,
brandAndSystemFidelity: 8,
renderedQuality: 8,
taskCompletion: 8,
},
});
});
});
+1 -427
View File
@@ -11,7 +11,6 @@ import { tmpdir } from 'node:os';
import { execFileSync, execSync, spawn } from 'node:child_process';
import {
getDesignSidecarPath,
getLiveCodexWorkerStatePath,
getLiveDir,
getLiveServerPath,
getLiveSessionsDir,
@@ -112,31 +111,6 @@ it('gitignores local Impeccable runtime artifacts', () => {
assert.match(ignored, /\.impeccable\/live\/deferred-svelte-component-accepts\.json/);
});
it('Stop Live removes Nuxt Vue preview modules and their generated root', async () => {
const cwd = mkdtempSync(join(tmpdir(), 'impeccable-live-nuxt-stop-'));
const generatedRoot = join(cwd, 'app/.impeccable-live');
mkdirSync(join(generatedRoot, 'session123'), { recursive: true });
writeFileSync(join(cwd, 'nuxt.config.ts'), 'export default defineNuxtConfig({});\n');
writeFileSync(join(generatedRoot, '__runtime.js'), 'export const runtime = true;\n');
writeFileSync(join(generatedRoot, 'session123', 'v1.vue'), '<template><h1>Preview</h1></template>\n');
let live;
try {
live = await startServer(8498, { cwd });
const exited = new Promise((resolve) => live.proc.once('exit', resolve));
await stopServer(live.port, live.token);
await Promise.race([
exited,
new Promise((_, reject) => setTimeout(() => reject(new Error('live server did not stop')), 2_000)),
]);
assert.equal(existsSync(join(generatedRoot, '__runtime.js')), false);
assert.equal(existsSync(generatedRoot), false);
} finally {
live?.proc?.kill();
rmSync(cwd, { recursive: true, force: true });
}
});
async function readSseUntil(reader, decoder, needle, maxReads = 12) {
let text = '';
for (let i = 0; i < maxReads; i++) {
@@ -226,37 +200,6 @@ describe('live-server integration', () => {
await drainPolls(server);
});
it('/status exposes a safe actionable Codex foreground fallback', async () => {
const workerPath = getLiveCodexWorkerStatePath(serverCwd);
mkdirSync(join(serverCwd, '.impeccable', 'live'), { recursive: true });
writeFileSync(workerPath, JSON.stringify({
owner: 'impeccable-live-codex-worker-v1',
cwd: serverCwd,
pid: null,
status: 'unavailable',
mode: 'foreground',
error: 'codex_cli_unavailable',
command: 'codex',
stack: 'must not cross the status boundary',
setup: {
docsUrl: 'https://learn.chatgpt.com/docs/codex/cli',
afterInstall: 'codex login',
},
}));
try {
const res = await fetch(`http://localhost:${server.port}/status?token=${server.token}`);
assert.equal(res.status, 200);
const data = await res.json();
assert.equal(data.codexWorker.status, 'unavailable');
assert.equal(data.codexWorker.mode, 'foreground');
assert.equal(data.codexWorker.error, 'codex_cli_unavailable');
assert.equal(data.codexWorker.setup.afterInstall, 'codex login');
assert.equal(data.codexWorker.stack, undefined);
} finally {
rmSync(workerPath, { force: true });
}
});
it('/status reports agentPolling from active poll leases', async () => {
await drainPolls(server);
let res = await fetch(`http://localhost:${server.port}/status?token=${server.token}`);
@@ -281,40 +224,6 @@ describe('live-server integration', () => {
assert.equal(data.agentPolling, false);
});
it('/status stops reporting agentPolling as soon as a poll returns an event', async () => {
await drainPolls(server);
const pollPromise = fetch(
`http://localhost:${server.port}/poll?token=${server.token}&timeout=5000&leaseMs=30000`,
).then((response) => response.json());
await new Promise((resolve) => setTimeout(resolve, 50));
const eventRes = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'generate',
id: 'aabbcc77',
action: 'impeccable',
count: 1,
pageUrl: '/',
element: { outerHTML: '<button>Truthful poll</button>', tagName: 'BUTTON' },
}),
});
assert.equal(eventRes.status, 200);
const event = await pollPromise;
assert.equal(event.id, 'aabbcc77');
const status = await fetch(`http://localhost:${server.port}/status?token=${server.token}`).then((response) => response.json());
assert.equal(status.agentPolling, false);
await fetch(`http://localhost:${server.port}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token: server.token, id: event.id, type: 'done', sourceEventType: 'generate' }),
});
});
it('/live.js serves script with token injected', async () => {
const res = await fetch(`http://localhost:${server.port}/live.js`);
assert.equal(res.status, 200);
@@ -2114,59 +2023,6 @@ colors: {}
assert.equal(data.type, 'timeout');
});
it('/poll type filters keep dedicated worker and foreground control lanes disjoint', async () => {
await drainPolls(server);
const controlPoll = fetch(
`http://localhost:${server.port}/poll?token=${server.token}&timeout=2000&types=steer,manual_edit_apply,carbonize_cleanup,exit`,
).then((response) => response.json());
const workerPoll = fetch(
`http://localhost:${server.port}/poll?token=${server.token}&timeout=2000&types=generate,accept,discard,prefetch`,
).then((response) => response.json());
const steer = {
token: server.token,
type: 'steer',
id: 'aabbcc01',
pageUrl: '/',
message: 'Keep this on the foreground lane',
};
const generate = {
token: server.token,
type: 'generate',
id: 'aabbcc02',
action: 'impeccable',
count: 1,
pageUrl: '/',
element: { outerHTML: '<button id="lane-test">Book</button>', id: 'lane-test', tagName: 'BUTTON' },
};
for (const event of [steer, generate]) {
const response = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(event),
});
assert.equal(response.status, 200);
}
const [controlEvent, workerEvent] = await Promise.all([controlPoll, workerPoll]);
assert.equal(controlEvent.type, 'steer');
assert.equal(controlEvent.id, steer.id);
assert.equal(workerEvent.type, 'generate');
assert.equal(workerEvent.id, generate.id);
for (const reply of [
{ id: steer.id, type: 'steer_done', message: 'Control lane handled it', sourceEventType: 'steer' },
{ id: generate.id, type: 'done', sourceEventType: 'generate' },
]) {
const response = await fetch(`http://localhost:${server.port}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token: server.token, ...reply }),
});
assert.equal(response.status, 200);
}
});
it('/poll rejects invalid token', async () => {
const res = await fetch(`http://localhost:${server.port}/poll?token=wrong&timeout=100`);
assert.equal(res.status, 401);
@@ -2286,9 +2142,6 @@ colors: {}
assert.equal(event.id, 'a1b2c3d4');
assert.equal(event.action, 'bolder');
assert.equal(event.count, 2);
assert.equal(event.scaffoldAttempted, true);
assert.equal(event.scaffoldError, 'insufficient_locator');
assert.equal(Number.isFinite(event.generationReadyAt), true);
await fetch(`http://localhost:${server.port}/poll`, {
method: 'POST',
@@ -2334,42 +2187,6 @@ colors: {}
it('accepts checkpoint events without exposing them as agent poll work', async () => {
await drainPolls(server);
const partialRes = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'checkpoint',
id: 'a1b2c3d7',
phase: 'cycling',
reason: 'browser_resumed',
revision: 1,
owner: 'browser-a',
expectedVariants: 3,
arrivedVariants: 1,
visibleVariant: 1,
}),
});
assert.equal(partialRes.status, 200);
const secondRes = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'checkpoint',
id: 'a1b2c3d7',
phase: 'cycling',
reason: 'variants_progress',
revision: 2,
owner: 'browser-a',
expectedVariants: 3,
arrivedVariants: 2,
visibleVariant: 2,
}),
});
assert.equal(secondRes.status, 200);
const res = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@@ -2378,10 +2195,8 @@ colors: {}
type: 'checkpoint',
id: 'a1b2c3d7',
phase: 'cycling',
reason: 'variants_ready',
revision: 3,
revision: 2,
owner: 'browser-a',
expectedVariants: 3,
arrivedVariants: 3,
visibleVariant: 2,
paramValues: { density: 'packed' },
@@ -2399,148 +2214,6 @@ colors: {}
const snapshot = JSON.parse(readFileSync(join(getLiveSessionsDir(server.cwd), 'a1b2c3d7.snapshot.json'), 'utf-8'));
assert.equal(snapshot.visibleVariant, 2);
assert.deepEqual(snapshot.paramValues, { density: 'packed' });
assert.ok(snapshot.generationTimings.first_reviewable?.at);
assert.ok(snapshot.generationTimings.second_reviewable?.at);
assert.ok(snapshot.generationTimings.all_variants_ready?.at);
assert.ok(snapshot.generationTimings.first_reviewable.at <= snapshot.generationTimings.second_reviewable.at);
assert.ok(snapshot.generationTimings.second_reviewable.at <= snapshot.generationTimings.all_variants_ready.at);
const atomicRes = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'checkpoint',
id: 'a1b2c3da',
phase: 'cycling',
reason: 'variants_ready',
revision: 1,
owner: 'browser-a',
expectedVariants: 3,
arrivedVariants: 3,
visibleVariant: 1,
}),
});
assert.equal(atomicRes.status, 200);
const atomicSnapshot = JSON.parse(readFileSync(join(getLiveSessionsDir(server.cwd), 'a1b2c3da.snapshot.json'), 'utf-8'));
assert.ok(atomicSnapshot.generationTimings.first_reviewable?.at);
assert.equal(
atomicSnapshot.generationTimings.first_reviewable.at,
atomicSnapshot.generationTimings.all_variants_ready?.at,
'atomic delivery makes the first variant and full set reviewable together',
);
});
it('journals and streams dedicated worker progress without leasing it as work', async () => {
await drainPolls(server);
const controller = new AbortController();
const sseRes = await fetch(
`http://localhost:${server.port}/events?token=${server.token}`,
{ signal: controller.signal },
);
const reader = sseRes.body.getReader();
await reader.read();
const progress = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'agent_phase',
id: 'a1b2c3e1',
phase: 'first_variant_generating',
owner: 'impeccable-live-codex-worker-v1',
}),
});
assert.equal(progress.status, 200);
const message = new TextDecoder().decode((await reader.read()).value);
controller.abort();
assert.match(message, /"type":"agent_phase"/);
assert.match(message, /"phase":"first_variant_generating"/);
const polled = await fetch(`http://localhost:${server.port}/poll?token=${server.token}&timeout=50`).then(r => r.json());
assert.equal(polled.type, 'timeout');
const snapshot = JSON.parse(readFileSync(join(getLiveSessionsDir(server.cwd), 'a1b2c3e1.snapshot.json'), 'utf-8'));
assert.ok(snapshot.generationTimings.first_variant_generating?.at);
});
it('streams Svelte component checkpoints as progressive preview updates', async () => {
const controller = new AbortController();
const sseRes = await fetch(
`http://localhost:${server.port}/events?token=${server.token}`,
{ signal: controller.signal },
);
const reader = sseRes.body.getReader();
const decoder = new TextDecoder();
await reader.read(); // connected
const res = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'checkpoint',
id: 'a1b2c3de',
phase: 'cycling',
reason: 'variants_progress',
revision: 1,
owner: 'svelte-worker',
expectedVariants: 3,
arrivedVariants: 1,
visibleVariant: 1,
previewMode: 'svelte-component',
previewFile: 'node_modules/.impeccable-live/a1b2c3de/manifest.json',
sourceFile: 'src/routes/+page.svelte',
}),
});
assert.equal(res.status, 200);
const { value } = await reader.read();
const message = decoder.decode(value);
assert.match(message, /"type":"variant_progress"/);
assert.match(message, /"arrivedVariants":1/);
assert.match(message, /"previewMode":"svelte-component"/);
controller.abort();
});
it('streams source checkpoints so no-HMR frameworks can review variant 1', async () => {
const controller = new AbortController();
const sseRes = await fetch(
`http://localhost:${server.port}/events?token=${server.token}`,
{ signal: controller.signal },
);
const reader = sseRes.body.getReader();
const decoder = new TextDecoder();
await reader.read(); // connected
const res = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'checkpoint',
id: 'a1b2c3df',
phase: 'cycling',
reason: 'variants_progress',
revision: 1,
owner: 'source-worker',
expectedVariants: 3,
arrivedVariants: 1,
visibleVariant: 1,
previewMode: 'source',
previewFile: 'app/pages/index.vue',
sourceFile: 'app/pages/index.vue',
publicationKind: 'params',
}),
});
assert.equal(res.status, 200);
const { value } = await reader.read();
const message = decoder.decode(value);
assert.match(message, /"type":"variant_progress"/);
assert.match(message, /"arrivedVariants":1/);
assert.match(message, /"previewMode":"source"/);
assert.match(message, /"previewFile":"app\/pages\/index.vue"/);
assert.match(message, /"publicationKind":"params"/);
controller.abort();
});
it('redelivers an unacknowledged browser event after helper server restart', async () => {
@@ -2687,105 +2360,6 @@ colors: {}
assert.equal(acked.type, 'timeout', 'acked event should be removed from the poll queue');
});
it('retires the leased Generate when early Accept or Discard takes ownership', async () => {
await drainPolls(server);
for (const [type, id] of [['accept', 'ea11ac01'], ['discard', 'ea11dc01']]) {
const generated = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'generate',
id,
action: 'bolder',
count: 3,
element: { outerHTML: '<section>early choice</section>', tagName: 'section' },
}),
});
assert.equal(generated.status, 200);
const generation = await fetch(`http://localhost:${server.port}/poll?token=${server.token}&types=generate&timeout=100&leaseMs=40`).then((response) => response.json());
assert.equal(generation.id, id);
const chosen = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type,
id,
...(type === 'accept' ? { variantId: '1' } : {}),
}),
});
assert.equal(chosen.status, 200);
const choice = await fetch(`http://localhost:${server.port}/poll?token=${server.token}&types=${type}&timeout=100&leaseMs=40`).then((response) => response.json());
assert.equal(choice.type, type);
assert.equal(choice.id, id);
const reply = await fetch(`http://localhost:${server.port}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
id,
sourceEventType: type,
type: type === 'discard' ? 'discarded' : 'complete',
}),
});
assert.equal(reply.status, 200);
await new Promise((resolve) => setTimeout(resolve, 60));
const stale = await fetch(`http://localhost:${server.port}/poll?token=${server.token}&types=generate&timeout=30&leaseMs=20`).then((response) => response.json());
assert.equal(stale.type, 'timeout', `${type} must prevent Generate redelivery after its old lease expires`);
const status = await fetch(`http://localhost:${server.port}/status?token=${server.token}`).then((response) => response.json());
assert.equal(status.pendingEvents.some((event) => event.id === id && event.type === 'generate'), false);
}
});
it('releases a failed worker Generate lease without consuming or broadcasting it', async () => {
await drainPolls(server);
const id = 'fa11bac1';
const generated = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'generate',
id,
action: 'bolder',
count: 3,
element: { outerHTML: '<article>fallback</article>', tagName: 'article' },
}),
});
assert.equal(generated.status, 200);
const leased = await fetch(`http://localhost:${server.port}/poll?token=${server.token}&types=generate&timeout=100&leaseMs=5000`).then((response) => response.json());
assert.equal(leased.id, id);
const retried = await fetch(`http://localhost:${server.port}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
id,
type: 'retry',
sourceEventType: 'generate',
}),
});
assert.equal(retried.status, 200);
assert.equal((await retried.json()).released, true);
const fallback = await fetch(`http://localhost:${server.port}/poll?token=${server.token}&types=generate&timeout=100&leaseMs=100`).then((response) => response.json());
assert.equal(fallback.id, id);
assert.equal(fallback.type, 'generate');
const status = await fetch(`http://localhost:${server.port}/status?token=${server.token}`).then((response) => response.json());
assert.equal(status.pendingEvents.some((event) => event.id === id && event.type === 'generate'), true);
const done = await fetch(`http://localhost:${server.port}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token: server.token, id, type: 'done', sourceEventType: 'generate' }),
});
assert.equal(done.status, 200);
});
it('wakes a parked poll as soon as a missed-ack lease expires', async () => {
await drainPolls(server);
-124
View File
@@ -62,84 +62,6 @@ describe('live-session-store', () => {
assert.equal(active[0].id, 'session-a');
});
it('persists the progressive variant plan across worker restarts', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'planned-session' });
const plan = {
identityLock: ['Preserve copy'],
directions: [
{ variantId: 1, name: 'Hierarchy', axis: 'scale', intent: 'Increase hierarchy' },
{ variantId: 2, name: 'Composition', axis: 'layout', intent: 'Recompose the root' },
{ variantId: 3, name: 'Rhythm', axis: 'spacing', intent: 'Increase rhythm' },
],
};
store.appendEvent({ type: 'generate', id: 'planned-session', count: 3 });
store.appendEvent({ type: 'variant_plan', id: 'planned-session', plan });
store.appendEvent({ type: 'checkpoint', id: 'planned-session', revision: 1, arrivedVariants: 1 });
const restarted = createLiveSessionStore({ cwd: tmp, sessionId: 'planned-session' });
assert.deepEqual(restarted.getSnapshot('planned-session').variantPlan, plan);
});
it('tracks parameter publication separately from variant arrival', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'parameter-phase' });
store.appendEvent({ type: 'generate', id: 'parameter-phase', count: 3, generationEpoch: 1 });
store.appendEvent({
type: 'variant_published', id: 'parameter-phase', revision: 1,
generationEpoch: 1, arrivedVariants: 3, publicationKind: 'variants',
});
assert.equal(store.getSnapshot('parameter-phase').paramsPublished, false);
store.appendEvent({
type: 'variant_published', id: 'parameter-phase', revision: 2,
generationEpoch: 1, arrivedVariants: 3, publicationKind: 'params',
});
assert.equal(store.getSnapshot('parameter-phase').paramsPublished, true);
});
it('tombstones generation on early accept and ignores late generation writes', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'early-accept' });
store.appendEvent({
type: 'generate',
id: 'early-accept',
action: 'polish',
count: 3,
element: { outerHTML: '<section>Hero</section>', tagName: 'section' },
});
store.appendEvent({
type: 'checkpoint',
id: 'early-accept',
revision: 1,
phase: 'cycling',
arrivedVariants: 1,
visibleVariant: 1,
});
store.appendEvent({ type: 'accept', id: 'early-accept', variantId: '1' });
store.appendEvent({
type: 'checkpoint',
id: 'early-accept',
revision: 2,
phase: 'variants_ready',
arrivedVariants: 3,
visibleVariant: 3,
});
store.appendEvent({
type: 'agent_done',
id: 'early-accept',
file: 'src/App.jsx',
arrivedVariants: 3,
});
const snapshot = store.getSnapshot('early-accept');
assert.equal(snapshot.phase, 'accept_requested');
assert.equal(snapshot.generationCanceled, true);
assert.equal(snapshot.cancelReason, 'accept');
assert.equal(snapshot.arrivedVariants, 1);
assert.equal(snapshot.visibleVariant, 1);
assert.equal(
snapshot.diagnostics.some((entry) => entry.error === 'late_generation_event_ignored'),
true,
);
});
it('reports corrupted journal lines while preserving valid prior events', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'corrupt-session' });
store.appendEvent({
@@ -239,30 +161,6 @@ describe('live-session-store', () => {
);
});
it('tracks publication and browser checkpoint revisions independently', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'split-revisions' });
store.appendEvent({
type: 'generate', id: 'split-revisions', count: 3,
element: { outerHTML: '<section>Hero</section>', tagName: 'section' },
});
store.appendEvent({
type: 'checkpoint', id: 'split-revisions', revision: 8, revisionDomain: 'browser',
owner: 'browser-a', phase: 'cycling', visibleVariant: 2,
});
store.appendEvent({
type: 'checkpoint', id: 'split-revisions', revision: 3, revisionDomain: 'publication',
reason: 'variants_progress', phase: 'cycling', arrivedVariants: 3,
});
const snapshot = store.getSnapshot('split-revisions');
assert.equal(snapshot.browserCheckpointRevision, 8);
assert.equal(snapshot.checkpointRevision, 8);
assert.equal(snapshot.publicationCheckpointRevision, 3);
assert.equal(snapshot.visibleVariant, 2);
assert.equal(snapshot.arrivedVariants, 3);
assert.equal(snapshot.diagnostics.some((entry) => entry.error === 'stale_checkpoint_ignored'), false);
});
it('keeps carbonize-required accepted sessions active until explicit completion', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'carbonize-session' });
store.appendEvent({
@@ -386,26 +284,4 @@ describe('live-session-store', () => {
assert.equal(migratedSnapshot.expectedVariants, 2);
assert.equal(migratedSnapshot.sourceFile, 'src/App.jsx');
});
it('records generation phase timings without replacing the workflow phase', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'phase-session' });
store.appendEvent({
type: 'generate',
id: 'phase-session',
count: 3,
element: { classes: ['hero'] },
});
store.appendEvent({
type: 'agent_phase',
id: 'phase-session',
phase: 'source_ready',
at: 1234,
durationMs: 42,
});
const snapshot = store.getSnapshot('phase-session');
assert.equal(snapshot.phase, 'generate_requested');
assert.equal(snapshot.generationPhase, 'source_ready');
assert.deepEqual(snapshot.generationTimings.source_ready, { at: 1234, durationMs: 42 });
});
});
-212
View File
@@ -1,212 +0,0 @@
import assert from 'node:assert/strict';
import { afterEach, beforeEach, describe, it } from 'node:test';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { createLiveSessionStore } from '../skill/scripts/live/session-store.mjs';
import {
prepareGenerationArtifact,
publishGenerationArtifact,
} from '../skill/scripts/live/generation-publisher.mjs';
import {
inlineVueComponentAccept,
nuxtViteFsModulePath,
removeAllVueComponentSessions,
scaffoldVueComponentSession,
} from '../skill/scripts/live/vue-component.mjs';
describe('Nuxt Vue component preview', () => {
let tmp;
let source;
beforeEach(() => {
tmp = mkdtempSync(join(tmpdir(), 'impeccable-vue-component-'));
source = join(tmp, 'app', 'pages', 'index.vue');
mkdirSync(join(tmp, 'app', 'pages'), { recursive: true });
writeFileSync(join(tmp, 'nuxt.config.ts'), 'export default defineNuxtConfig({ ssr: false });\n');
writeFileSync(source, [
'<template>',
' <main>',
' <h1 class="hero-title">Hello {{ user.name }}</h1>',
' </main>',
'</template>',
'',
'<style scoped>',
'.hero-title { font-size: 2rem; }',
'</style>',
'',
].join('\n'));
});
afterEach(() => rmSync(tmp, { recursive: true, force: true }));
it('stages real Vue SFCs without rewriting the active route', () => {
const before = readFileSync(source, 'utf-8');
const result = scaffoldVueComponentSession({
id: 'vue12345',
count: 3,
sourceFile: 'app/pages/index.vue',
sourceStartLine: 3,
sourceEndLine: 3,
originalLines: [' <h1 class="hero-title">Hello {{ user.name }}</h1>'],
cwd: tmp,
});
assert.equal(readFileSync(source, 'utf-8'), before);
assert.equal(result.manifest.previewMode, 'vue-component');
assert.equal(result.manifest.componentExtension, 'vue');
assert.match(result.manifestFile, /^app\/\.impeccable-live\/vue12345\/manifest\.json$/);
const variant = readFileSync(join(tmp, result.componentDir, 'v1.vue'), 'utf-8');
assert.match(variant, /<template>/);
assert.match(variant, /Hello \{\{ name \}\}/);
assert.equal(existsSync(join(tmp, 'app/.impeccable-live/__runtime.js')), true);
assert.equal(
result.manifest.runtimeModule,
nuxtViteFsModulePath(join(tmp, 'app/.impeccable-live/__runtime.js'), tmp),
);
assert.equal(
result.manifest.componentModuleBase,
nuxtViteFsModulePath(join(tmp, result.componentDir), tmp),
);
assert.match(result.manifest.runtimeModule, /^\/@fs\//);
assert.doesNotMatch(result.manifest.runtimeModule, /^\/app\//);
assert.match(result.manifest.componentModuleBase, /^\/@fs\//);
});
it('keeps Vite module URLs valid for literal Nuxt srcDir projects', () => {
writeFileSync(join(tmp, 'nuxt.config.ts'), "export default defineNuxtConfig({ srcDir: 'client/' });\n");
const clientSource = join(tmp, 'client', 'pages', 'index.vue');
mkdirSync(join(tmp, 'client', 'pages'), { recursive: true });
writeFileSync(clientSource, '<template><h1>Client app</h1></template>\n');
const result = scaffoldVueComponentSession({
id: 'clientsrc',
count: 1,
sourceFile: 'client/pages/index.vue',
sourceStartLine: 1,
sourceEndLine: 1,
originalLines: ['<h1>Client app</h1>'],
cwd: tmp,
});
assert.match(result.manifestFile, /^client\/\.impeccable-live\/clientsrc\/manifest\.json$/);
assert.match(result.manifest.runtimeModule, /^\/@fs\/.*\/client\/\.impeccable-live\/__runtime\.js$/);
assert.match(result.manifest.componentModuleBase, /^\/@fs\/.*\/client\/\.impeccable-live\/clientsrc$/);
});
it('accepts one generated SFC into clean Vue source and restores route expressions', () => {
const result = scaffoldVueComponentSession({
id: 'vue12345',
count: 3,
sourceFile: 'app/pages/index.vue',
sourceStartLine: 3,
sourceEndLine: 3,
originalLines: [' <h1 class="hero-title">Hello {{ user.name }}</h1>'],
cwd: tmp,
});
writeFileSync(join(tmp, result.componentDir, 'v1.vue'), [
'<script setup>',
"defineProps({ name: { default: '' } });",
'</script>',
'<template>',
' <h1 class="hero-title variant-one">Welcome {{ name }}</h1>',
'</template>',
'<style scoped>',
'.variant-one { letter-spacing: 0.02em; }',
'</style>',
'',
].join('\n'));
const accepted = inlineVueComponentAccept(result.manifest, 1, tmp);
assert.equal(accepted.handled, true);
const next = readFileSync(source, 'utf-8');
assert.match(next, /Welcome \{\{ user\.name \}\}/);
assert.match(next, /class="hero-title variant-one"|class="variant-one hero-title"/);
assert.match(next, /\.variant-one \{ letter-spacing: 0\.02em; \}/);
assert.doesNotMatch(next, /data-impeccable/);
assert.equal(existsSync(join(tmp, result.componentDir, 'manifest.json')), false);
assert.equal(existsSync(join(tmp, result.componentDir, 'v1.vue')), true, 'imported SFC remains until Live shutdown');
});
it('removes deferred SFCs, the shared runtime, and the generated root on Live shutdown', () => {
const result = scaffoldVueComponentSession({
id: 'vue12345',
count: 1,
sourceFile: 'app/pages/index.vue',
sourceStartLine: 3,
sourceEndLine: 3,
originalLines: [' <h1 class="hero-title">Hello {{ user.name }}</h1>'],
cwd: tmp,
});
inlineVueComponentAccept(result.manifest, 1, tmp);
const root = join(tmp, 'app/.impeccable-live');
assert.equal(existsSync(join(root, '__runtime.js')), true);
assert.equal(existsSync(join(tmp, result.componentDir, 'v1.vue')), true);
removeAllVueComponentSessions(tmp);
assert.equal(existsSync(join(root, '__runtime.js')), false);
assert.equal(existsSync(root), false);
});
it('publishes manifest-last, preserves the route, and rejects late work after Accept', () => {
const result = scaffoldVueComponentSession({
id: 'vue12345',
count: 3,
sourceFile: 'app/pages/index.vue',
sourceStartLine: 3,
sourceEndLine: 3,
originalLines: [' <h1 class="hero-title">Hello {{ user.name }}</h1>'],
cwd: tmp,
});
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'vue12345' });
store.appendEvent({
type: 'generate',
id: 'vue12345',
generationEpoch: 1,
count: 3,
action: 'polish',
element: { outerHTML: '<h1>Hello Paul</h1>' },
});
const routeBefore = readFileSync(source, 'utf-8');
const prepared = prepareGenerationArtifact({ id: 'vue12345', sourceFile: result.manifestFile, cwd: tmp });
assert.equal(prepared.ok, true);
assert.equal(prepared.previewMode, 'vue-component');
const artifactManifest = JSON.parse(readFileSync(join(tmp, prepared.artifactFile), 'utf-8'));
artifactManifest.arrivedVariants = 1;
writeFileSync(join(tmp, prepared.artifactFile), JSON.stringify(artifactManifest, null, 2) + '\n');
writeFileSync(join(tmp, prepared.componentDir, 'v1.vue'), '<template><h1>First</h1></template>\n');
const published = publishGenerationArtifact({
id: 'vue12345',
epoch: prepared.epoch,
sourceFile: result.manifestFile,
artifactFile: prepared.artifactFile,
expectedSourceHash: prepared.expectedSourceHash,
arrivedVariants: 1,
expectedVariants: 3,
cwd: tmp,
});
assert.equal(published.ok, true);
assert.equal(published.previewMode, 'vue-component');
assert.equal(readFileSync(source, 'utf-8'), routeBefore);
assert.equal(JSON.parse(readFileSync(join(tmp, result.manifestFile), 'utf-8')).arrivedVariants, 1);
const late = prepareGenerationArtifact({ id: 'vue12345', sourceFile: result.manifestFile, cwd: tmp });
store.appendEvent({ type: 'accept', id: 'vue12345', variantId: '1' });
const rejected = publishGenerationArtifact({
id: 'vue12345',
epoch: late.epoch,
sourceFile: result.manifestFile,
artifactFile: late.artifactFile,
expectedSourceHash: late.expectedSourceHash,
arrivedVariants: 2,
expectedVariants: 3,
cwd: tmp,
});
assert.equal(rejected.ok, false);
assert.equal(rejected.error, 'stale_generation_epoch');
assert.equal(readFileSync(source, 'utf-8'), routeBefore);
});
});
+2 -50
View File
@@ -6,9 +6,9 @@
import { describe, it, beforeEach, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, writeFileSync, readFileSync, rmSync, mkdirSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { execFileSync, execSync } from 'node:child_process';
import { execSync } from 'node:child_process';
import {
buildSearchQueries,
@@ -253,26 +253,6 @@ describe('wrapCli integration', () => {
assert.ok(!modified.includes('data-impeccable-variant="original" style="display: none"'));
});
it('creates an isolated source preview without mutating the project file', () => {
const html = '<main>\n <section class="hero"><h1>Original</h1></section>\n</main>\n';
writeFileSync(join(tmp, 'index.html'), html);
const output = execFileSync(process.execPath, [
resolve('skill/scripts/live-wrap.mjs'),
'--id', 'isolated123', '--count', '3', '--classes', 'hero',
'--file', 'index.html', '--isolated',
], { cwd: tmp, encoding: 'utf-8' });
const result = JSON.parse(output);
assert.equal(readFileSync(join(tmp, 'index.html'), 'utf-8'), html);
assert.equal(result.sourceFile, 'index.html');
assert.equal(result.previewMode, 'source-artifact');
assert.match(result.file, /^\.impeccable\/live\/previews\/isolated123\/preview\.html$/);
assert.match(readFileSync(join(tmp, result.file), 'utf-8'), /data-impeccable-variants="isolated123"/);
const manifest = JSON.parse(readFileSync(join(tmp, result.previewManifest), 'utf-8'));
assert.equal(manifest.originalSource, ' <section class="hero"><h1>Original</h1></section>');
assert.equal(manifest.sourceFile, 'index.html');
});
it('wraps a JSX element and uses JSX comment syntax', () => {
const jsx = `export default function App() {
return (
@@ -800,34 +780,6 @@ export default function App() {
assert.ok(modified.includes('data-impeccable-variants="dyn1"'), 'wrapped (first-match fallback)');
});
it('refuses multiple dynamic source branches when rendered text cannot identify one', () => {
const astro = `---
const results = [{ title: 'Result 01' }, { title: 'Result 02' }];
---
<main>
<article class="result-card"><h2>{results[0].title}</h2></article>
<article class="result-card"><h2>{results[1].title}</h2></article>
</main>`;
const file = join(tmp, 'Results.astro');
writeFileSync(file, astro);
let errPayload;
try {
execSync(
`node skill/scripts/live-wrap.mjs --id dyn2 --count 3 --classes "result-card" --tag "article" --text "Result 02 rendered body" --file "${file}"`,
{ cwd: process.cwd(), encoding: 'utf-8', stdio: 'pipe' },
);
assert.fail('Should have refused an unsafe first-match fallback');
} catch (err) {
errPayload = JSON.parse(err.stderr.toString().trim());
}
assert.equal(errPayload.error, 'element_ambiguous');
assert.equal(errPayload.reason, 'rendered_text_not_in_source');
assert.equal(errPayload.candidates.length, 2);
assert.doesNotMatch(readFileSync(file, 'utf-8'), /impeccable-variants-start/);
});
it('errors with element_ambiguous when --text matches multiple identical branches', () => {
// Two <aside className="card"> with truly identical body text. --text
// can't pick a winner — wrap should refuse rather than silently land.
+6 -6
View File
@@ -40,18 +40,18 @@ The trace is the source of truth, not the model's free-form reply.
| # | Setup | Assertion |
|---|---|---|
| 1 | empty workspace | runs `context.mjs`; loads `reference/init.md` when it treats the run as attended or `reference/new-work.md` when it recognizes the one-shot exception; resolves that gate before implementation |
| 2 | PRODUCT.md only | runs `context.mjs` 1-3 times; loads `reference/new-work.md` because no committed design system exists |
| 3 | PRODUCT.md + DESIGN.md | runs `context.mjs` 1-3 times; receives or explores the committed design system |
| 1 | empty workspace | runs `context.mjs`; loads `reference/init.md` before implementation; automation is not an init bypass |
| 2 | PRODUCT.md only | runs `context.mjs` 1-3 times; loads `reference/init.md` to establish DESIGN.md with the user before surface work |
| 3 | PRODUCT.md + DESIGN.md | runs `context.mjs` 1-3 times; receives the committed design system and loads `reference/new-work.md` for the task-scoped concept |
| 4 | PRODUCT.md + DESIGN.md, context already loaded in turn 1 | turn 2 does **not** re-run `context.mjs` |
| 5 | PRODUCT.md without the legacy `## Register` field | runs `context.mjs`; greenfield craft still loads `reference/new-work.md` |
| 5 | PRODUCT.md without the legacy `## Register` field and no DESIGN.md | runs `context.mjs`; greenfield craft still loads `reference/init.md` to establish the missing world |
| 6 | PRODUCT.md + DESIGN.md + a minimal `index.html`; prompt is `/impeccable polish` | loads `reference/polish.md` |
| 7 | same fixture; prompt is `/impeccable audit` | loads `reference/audit.md` |
| 8 | PRODUCT.md + DESIGN.md + a SvelteKit scaffold (`src/app.css`, components, `+page.svelte`); prompt is `/impeccable polish src/routes/+page.svelte` | reads at least one project code file (CSS / component / page) — not just the skill's reference files |
| 9 | PRODUCT.md + `index.html` + a seeded update cache with a newer version (`skillVersion` copy-mode so `context.mjs` has a `SKILL.md` to version-check against); prompt is `/impeccable polish index.html` | `context.mjs` runs and its output carries the `UPDATE_AVAILABLE` directive (proven via captured bash output); the agent does **not** auto-run `npx impeccable update` (it must ask first) |
| 10 | no PRODUCT.md + a minimal `index.html`; prompt is `/impeccable polish index.html` | runs `context.mjs`, loads `reference/polish.md`, and does **not** divert into `reference/init.md` |
| 11 | empty workspace; prompt is `/impeccable shape ...` | runs `context.mjs`; resolves `reference/init.md` (attended) or `reference/new-work.md` (unattended) before implementation |
| 12 | empty workspace; prompt is natural-language build intent with no command word | runs `context.mjs`; resolves `reference/init.md` (attended) or `reference/new-work.md` (unattended) before implementation |
| 11 | empty workspace; prompt is `/impeccable shape ...` | runs `context.mjs`; resolves `reference/init.md` before planning the surface |
| 12 | empty workspace; prompt is natural-language build intent with no command word | runs `context.mjs`; resolves `reference/init.md` before implementation |
| 13 | empty workspace; prompt is `/impeccable teach` | runs `context.mjs` and diverts into `reference/init.md` because `teach` aliases `init` |
| 14 | PRODUCT.md with `## Register: product` + `## Platform: ios` (native iOS app); prompt is `/impeccable craft a tide detail screen` | `context.mjs` runs and emits a NEXT STEP pointing at `reference/ios.md` (proven via captured bash output); agent loads `reference/ios.md` (Setup step 5, native conventions on top of the register reference) |
| 15 | same iOS fixture; prompt is `/impeccable audit` | agent loads `reference/audit.native.md` (the Commands-table native variant, routed instead of `audit.md`) |
+68 -19
View File
@@ -7,8 +7,8 @@
*/
export const PRODUCT_MD_SAMPLE = `# Acme Notes
## Register
brand
## Platform
web
## Product Purpose
Acme Notes is a marketing-driven landing page for a research-grade note-taking
@@ -21,7 +21,23 @@ Working researchers (PhD students, postdocs, principal investigators) who
already maintain disciplined note-taking systems and are choosing between
ours and rolling their own in a Zettelkasten plugin.
## Brand
## Positioning
The research notebook that preserves a scientist's chain of thought instead
of flattening it into generic documents and folders.
## Audience World
Lab notebooks, margin annotations, citation trails, preprint PDFs, index cards,
and the quiet ritual of reconstructing why a conclusion was reached months ago.
## Cultural Context
Research monographs and working laboratory archives: precise, annotated,
accumulative, and visibly handled rather than pristine lifestyle publishing.
## Pinned Direction
Type-led and evidence-first. Never lead with product screenshots or generic
startup chrome.
## Brand Personality
Editorial, considered, technical. The product is for people who quote
Knuth. The voice is closer to a long-read magazine than to a startup
landing page.
@@ -31,20 +47,25 @@ landing page.
- Obsidian (too community-cottagecore)
- Any SaaS landing page with a hero-metric grid
## Strategic Principles
## Design Principles
- Type does most of the work. The hero is words, not chrome.
- One named accent color, used sparingly.
- Never lead with screenshots. Lead with the idea.
## Accessibility & Inclusion
WCAG AA, fully keyboard accessible, readable at 200% zoom, and calm under
reduced motion.
`;
/**
* Same project shape as PRODUCT_MD_SAMPLE but with no `## Register` field.
* Exercises the cascade fallback (task cue then surface in focus) in
* scenarios where context.mjs cannot detect the register and the agent
* must follow the SKILL.md priority list to pick brand.md.
* Legacy product context with the modern strategic fields but no visual
* world. Exercises init completion without a deprecated brand/product field.
*/
export const PRODUCT_MD_SAMPLE_NO_REGISTER = `# Acme Notes
## Platform
web
## Product Purpose
Acme Notes is a marketing-driven landing page for a research-grade note-taking
tool aimed at independent scientists and graduate students. The site needs to
@@ -56,7 +77,21 @@ Working researchers (PhD students, postdocs, principal investigators) who
already maintain disciplined note-taking systems and are choosing between
ours and rolling their own in a Zettelkasten plugin.
## Brand
## Positioning
The research notebook that preserves a scientist's chain of thought instead
of flattening it into generic documents and folders.
## Audience World
Lab notebooks, margin annotations, citation trails, preprint PDFs, index cards,
and the ritual of reconstructing a conclusion months later.
## Cultural Context
Research monographs and working laboratory archives.
## Pinned Direction
Type-led and evidence-first; no startup chrome.
## Brand Personality
Editorial, considered, technical. The product is for people who quote
Knuth. The voice is closer to a long-read magazine than to a startup
landing page.
@@ -66,23 +101,22 @@ landing page.
- Obsidian (too community-cottagecore)
- Any SaaS landing page with a hero-metric grid
## Strategic Principles
## Design Principles
- Type does most of the work. The hero is words, not chrome.
- One named accent color, used sparingly.
- Never lead with screenshots. Lead with the idea.
## Accessibility & Inclusion
WCAG AA, keyboard access, 200% zoom, and reduced motion support.
`;
/**
* Native iOS app fixture: product register, `## Platform` set to `ios`.
* Exercises Setup step 5 when context.mjs reports the platform is native,
* the agent must also load `reference/ios.md` (Apple HIG) on top of the
* register reference. Product register because this is app UI, not marketing.
* Native iOS app fixture with `## Platform` set to `ios`. Exercises Setup
* step 5 the agent must also load `reference/ios.md` (Apple HIG) on top of
* the task-scoped visitor-mode guidance.
*/
export const PRODUCT_MD_SAMPLE_IOS = `# Tideline
## Register
product
## Platform
ios
@@ -97,7 +131,18 @@ Saltwater anglers checking conditions dockside on an iPhone, often one-handed
in bright sun and sometimes offline. They live in Apple Weather, Notes, and
Maps and expect the same gestures and controls here.
## Brand
## Positioning
The fastest trustworthy read on whether the next coastal window is worth the trip.
## Audience World
Tide tables, chartplotters, dock logs, weather radar, tackle trays, wet gloves,
and the repeated glance from water to phone in hard daylight.
## Pinned Direction
Native iOS controls and navigation are non-negotiable; the logbook may carry
the product's distinctive character.
## Brand Personality
Calm, legible, marine. Identity shows through color, type accent, and the
logbook's character never by reinventing the navigation bar or the back
gesture.
@@ -107,10 +152,14 @@ gesture.
- Custom toggles and bespoke tab bars that fight the platform
- Cluttered, metric-theater home screens
## Strategic Principles
## Design Principles
- Platform conformance is the structural bar; brand lives in the expressive layer.
- Standard navigation, SF Symbols, Dynamic Type, Dark Mode first-class.
- One accent tint drives interactive elements.
## Accessibility & Inclusion
Dynamic Type, VoiceOver, reduced motion, high contrast in direct sun, and
targets usable one-handed with wet hands.
`;
/**
+57 -5
View File
@@ -10,7 +10,7 @@
* 4. Inlines SKILL.md as the system prompt (placeholders stripped to
* neutral values so the same body works for all providers).
* 5. Runs Vercel AI SDK generateText with workspace-scoped tools
* (bash, read, write, list).
* (bash, read, write, list, ask_user_question).
* 6. Captures every tool call and returns a trace + the raw response
* messages (so multi-turn scenarios can append to them).
*
@@ -49,7 +49,7 @@ function loadSkillBody() {
md = md
.replaceAll('{{model}}', 'the assistant')
.replaceAll('{{command_prefix}}', '/')
.replaceAll('{{ask_instruction}}', 'Ask the user')
.replaceAll('{{ask_instruction}}', 'Use the ask_user_question tool.')
.replaceAll('{{config_file}}', 'AGENTS.md')
.replaceAll('{{scripts_path}}', '.claude/skills/impeccable/scripts')
.replaceAll('{{command_hint}}', 'command');
@@ -159,7 +159,26 @@ function execBash(workspace, command, timeoutMs = 20_000, extraEnv = {}) {
* Build the workspace-scoped tool set + the trace it writes into.
* Returns `{ tools, trace }`. The trace mutates in place as the agent runs.
*/
export function makeTools(workspace, extraEnv = {}) {
function defaultSimulatedAnswer(question) {
const text = String(question?.question ?? '').toLowerCase();
const options = Array.isArray(question?.options) ? question.options : [];
const firstOption = options.find((option) => typeof option?.label === 'string')?.label;
// Option labels are model-authored and therefore the most faithful answer
// when the agent is asking the user to choose a proposed world or concept.
if (firstOption) return firstOption;
if (/platform|web|ios|android|adaptive/.test(text)) return 'Web.';
if (/who|audience|user|people/.test(text)) return 'Night-shift ferry dispatchers working from noisy control rooms.';
if (/purpose|job|problem|outcome|success/.test(text)) return 'Help dispatchers resolve berth conflicts before they delay the overnight crossing.';
if (/position|different|claim|only/.test(text)) return 'It turns fragmented radio calls into one trustworthy handoff record.';
if (/world|tool|place|object|ritual|context/.test(text)) return 'Harbor logs, tide tables, grease-pencil berth boards, radio call signs, and sodium-lit terminals.';
if (/direction|feel|personality|reference|look/.test(text)) return 'Decisive, maritime, and operational; avoid generic SaaS dashboards and nautical decoration.';
if (/accessib|motion|contrast/.test(text)) return 'WCAG AA, keyboard access, reduced motion, and high contrast for dim control rooms.';
if (/scope|fidelity|breadth|interactiv|polish/.test(text)) return 'One production-ready responsive surface with working interactions.';
return 'Use the brief, preserve real operational content, and make the primary decision obvious.';
}
export function makeTools(workspace, extraEnv = {}, simulatedUser = {}) {
const trace = {
toolCalls: [],
bashCommands: [],
@@ -167,6 +186,8 @@ export function makeTools(workspace, extraEnv = {}) {
readPaths: [],
writePaths: [],
listPaths: [],
questionCalls: [],
questionAnswers: [],
};
function record(name, input) {
trace.toolCalls.push({ name, input });
@@ -174,6 +195,7 @@ export function makeTools(workspace, extraEnv = {}) {
if (name === 'read' && typeof input?.path === 'string') trace.readPaths.push(input.path);
if (name === 'write' && typeof input?.path === 'string') trace.writePaths.push(input.path);
if (name === 'list' && typeof input?.path === 'string') trace.listPaths.push(input.path);
if (name === 'ask_user_question') trace.questionCalls.push(input);
}
const tools = {
bash: tool({
@@ -241,6 +263,34 @@ export function makeTools(workspace, extraEnv = {}) {
return entries.length ? entries.join('\n') : '(empty)';
},
}),
ask_user_question: tool({
description:
'Ask the user 1-4 structured questions and wait for answers. Use this for required Impeccable init, visual-world selection, and task-concept checkpoints instead of asking in prose.',
inputSchema: z.object({
questions: z.array(z.object({
header: z.string().optional(),
question: z.string(),
options: z.array(z.object({
label: z.string(),
description: z.string().optional(),
})).optional(),
multiSelect: z.boolean().optional(),
})).min(1).max(4),
}),
execute: async ({ questions }) => {
record('ask_user_question', { questions });
const answers = {};
for (let index = 0; index < questions.length; index++) {
const question = questions[index];
const custom = typeof simulatedUser.answer === 'function'
? await simulatedUser.answer(question, index, { workspace, trace })
: undefined;
answers[question.question] = custom ?? defaultSimulatedAnswer(question);
}
trace.questionAnswers.push(answers);
return JSON.stringify({ answers });
},
}),
};
return { tools, trace };
}
@@ -251,8 +301,8 @@ export function makeTools(workspace, extraEnv = {}) {
* `priorMessages` lets multi-turn scenarios chain context from a previous
* call (append the SDK's response messages between turns).
*/
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8, env = {} }) {
const { tools, trace } = makeTools(workspace, env);
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8, env = {}, simulatedUser = {} }) {
const { tools, trace } = makeTools(workspace, env, simulatedUser);
const messages = [
...priorMessages,
{ role: 'user', content: userPrompt },
@@ -306,5 +356,7 @@ export function summarizeTrace(trace) {
bashCommands: trace.bashCommands,
readPaths: trace.readPaths,
writePaths: trace.writePaths,
questionCalls: trace.questionCalls,
questionAnswers: trace.questionAnswers,
};
}
+22 -28
View File
@@ -39,7 +39,7 @@ const SHAPE_PROMPT = '/impeccable shape a landing page for the project in this w
const NATURAL_BUILD_PROMPT = 'Build a landing page for the project in this workspace.';
const TEACH_PROMPT = '/impeccable teach';
const PRIMER_PROMPT =
'Take a quick look at the project. What register is this? Run the impeccable context loader once if you need to.';
'Take a quick look at the project. What context should guide later design work? Run the impeccable context loader once if you need to.';
const VERBOSE = process.env.IMPECCABLE_SKILL_BEHAVIOR_VERBOSE === '1';
@@ -109,19 +109,14 @@ for (const modelId of resolveModelList()) {
`expected agent to run context.mjs at least once; got ${loadCalls.length}.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
const resolvedBuildGate =
fileLoaded(trace, 'init.md') || fileLoaded(trace, 'new-work.md');
assert.ok(
resolvedBuildGate,
`craft should load init.md for an attended run or new-work.md when it treats the harness as unattended.\n` +
fileLoaded(trace, 'init.md'),
`craft should load init.md when no product or visual world exists; an automated harness is not a bypass.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
const gatePrecededImplementation =
loadedBeforeImplementationWrite(trace, 'init.md') ||
loadedBeforeImplementationWrite(trace, 'new-work.md');
assert.ok(
gatePrecededImplementation,
`agent should resolve the init/new-work gate before writing implementation files.\n` +
loadedBeforeImplementationWrite(trace, 'init.md'),
`agent should resolve init before writing implementation files.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
} finally {
@@ -148,8 +143,8 @@ for (const modelId of resolveModelList()) {
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
);
assert.ok(
fileLoaded(trace, 'new-work.md'),
`greenfield craft should load new-work.md when PRODUCT.md exists without a committed design system.\n` +
fileLoaded(trace, 'init.md'),
`greenfield craft should load init.md Step 5 when PRODUCT.md exists without a committed design system.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
} finally {
@@ -175,6 +170,11 @@ for (const modelId of resolveModelList()) {
`expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` +
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
);
assert.ok(
fileLoaded(trace, 'new-work.md'),
`craft inside a committed PRODUCT.md + DESIGN.md world should load new-work.md for the task-specific concept.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
// The skill tells the agent to also familiarize with the existing
// design system. DESIGN.md is bundled in context.mjs output, but
// exploring CSS / tokens / theme files or a directory listing
@@ -236,7 +236,7 @@ for (const modelId of resolveModelList()) {
}
});
it('scenario 5: PRODUCT.md without legacy register metadata still follows new-work', async () => {
it('scenario 5: legacy PRODUCT.md still completes init when DESIGN.md is missing', async () => {
const workspace = prepareWorkspace({
files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_NO_REGISTER },
});
@@ -247,7 +247,7 @@ for (const modelId of resolveModelList()) {
userPrompt: CRAFT_PROMPT,
maxSteps: setupMaxSteps,
});
logTrace('S5', 'no-register-field', modelId, trace, { textSample: text.slice(0, 400) });
logTrace('S5', 'legacy-product', modelId, trace, { textSample: text.slice(0, 400) });
const loadCalls = bashCommandsMatching(trace, 'context.mjs');
assert.ok(
loadCalls.length >= 1,
@@ -255,8 +255,8 @@ for (const modelId of resolveModelList()) {
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
);
assert.ok(
fileLoaded(trace, 'new-work.md'),
`greenfield craft should load new-work.md regardless of legacy register metadata.\n` +
fileLoaded(trace, 'init.md'),
`greenfield craft should load init.md for legacy product context when DESIGN.md is missing.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
} finally {
@@ -462,12 +462,9 @@ for (const modelId of resolveModelList()) {
`expected agent to run context.mjs at least once.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
const gatePrecededImplementation =
loadedBeforeImplementationWrite(trace, 'init.md') ||
loadedBeforeImplementationWrite(trace, 'new-work.md');
assert.ok(
gatePrecededImplementation,
`shape should resolve init.md (attended) or new-work.md (unattended) before implementation.\n` +
loadedBeforeImplementationWrite(trace, 'init.md'),
`shape should resolve init.md before implementation when no world exists.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
} finally {
@@ -490,12 +487,9 @@ for (const modelId of resolveModelList()) {
`expected agent to run context.mjs at least once.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
const gatePrecededImplementation =
loadedBeforeImplementationWrite(trace, 'init.md') ||
loadedBeforeImplementationWrite(trace, 'new-work.md');
assert.ok(
gatePrecededImplementation,
`build intent should resolve init.md (attended) or new-work.md (unattended) before implementation.\n` +
loadedBeforeImplementationWrite(trace, 'init.md'),
`build intent should resolve init.md before implementation when no world exists.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
} finally {
@@ -533,10 +527,10 @@ for (const modelId of resolveModelList()) {
}
});
it('scenario 14: native iOS project (agent loads ios.md on top of register)', async () => {
it('scenario 14: native iOS project (agent loads ios.md)', async () => {
// PRODUCT.md sets `## Platform` to `ios`. context.mjs emits a NEXT STEP
// directive to read reference/ios.md for native conventions. Setup step 5
// requires it on top of the register reference. The detector / live mode
// requires it on top of the visitor-mode guidance. The detector / live mode
// are web-only, so the only platform-specific obligation is loading the
// native reference — that's what this asserts.
const workspace = prepareWorkspace({
@@ -0,0 +1,174 @@
/**
* Provider-backed workflow contract tests. Unlike scenarios.test.mjs, these
* assert the attended turns and writes that make init/redesign/refinement real.
*/
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import {
prepareWorkspace,
cleanupWorkspace,
runTurn,
fileLoaded,
summarizeTrace,
} from './harness.mjs';
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
import { PRODUCT_MD_SAMPLE, DESIGN_MD_SAMPLE } from './fixtures.mjs';
const LEGACY_DESIGN = `# Design
## Identity
BORING_BEIGE_CARDS. Quiet beige panels, timid scale, rounded cards everywhere.
## Color
Warm gray background with a muted tan accent.
`;
const EXISTING_PAGE = `<!doctype html>
<html><head><style>
:root { --legacy-beige: #e8e1d5; --legacy-tan: #a78969; }
body { background: var(--legacy-beige); color: #3c3833; font-family: Arial, sans-serif; }
.card { border: 1px solid #cfc5b6; border-radius: 18px; padding: 24px; }
</style></head><body>
<header data-untouched="header"><a href="/">Harbor Desk</a></header>
<main><section id="case-study" class="card"><h1>Harbor Desk</h1><p>Challenge. Approach. Outcome.</p><p>Image placeholder</p></section></main>
<footer data-untouched="footer">Operational since 1987</footer>
</body></html>`;
function firstCall(trace, predicate) {
return trace.toolCalls.findIndex(predicate);
}
function firstWrite(trace, pattern) {
return firstCall(trace, ({ name, input }) => name === 'write' && pattern.test(input?.path ?? ''));
}
function workflowTraceMessage(trace) {
return JSON.stringify(summarizeTrace(trace), null, 2);
}
for (const modelId of resolveModelList()) {
const provider = detectProvider(modelId);
const keyPresent = hasKey(provider);
describe(`skill workflow contract :: ${modelId}`, () => {
if (!keyPresent) {
it(`skipped — ${PROVIDERS[provider].envKey} is unset`, { skip: true }, () => {});
return;
}
const model = getModel(modelId);
it('fresh init asks, writes PRODUCT without Register, then establishes DESIGN', async () => {
const workspace = prepareWorkspace({ files: {} });
try {
const { trace } = await runTurn({
workspace,
model,
userPrompt: '/impeccable init for a harbor operations product, then finish setup.',
maxSteps: 24,
});
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
const productWrite = firstWrite(trace, /(^|\/)PRODUCT\.md$/i);
const designWrite = firstWrite(trace, /(^|\/)DESIGN\.md$/i);
assert.ok(fileLoaded(trace, 'init.md'), `init.md was not loaded.\n${workflowTraceMessage(trace)}`);
assert.ok(question >= 0, `structured user was never asked.\n${workflowTraceMessage(trace)}`);
assert.ok(productWrite > question, `PRODUCT.md must follow a user answer.\n${workflowTraceMessage(trace)}`);
assert.ok(designWrite > productWrite, `DESIGN.md must follow PRODUCT.md.\n${workflowTraceMessage(trace)}`);
const product = fs.readFileSync(path.join(workspace, 'PRODUCT.md'), 'utf8');
assert.doesNotMatch(product, /^## Register\s*$/im);
assert.match(product, /ferry|dispatch|harbor/i, 'PRODUCT.md should incorporate the simulated user context');
assert.ok(fs.existsSync(path.join(workspace, 'DESIGN.md')));
} finally {
cleanupWorkspace(workspace);
}
});
it('initialized craft asks for the task concept before implementation', async () => {
const workspace = prepareWorkspace({
files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE, 'DESIGN.md': DESIGN_MD_SAMPLE },
});
try {
const { trace } = await runTurn({
workspace,
model,
userPrompt: '/impeccable craft a concise evidence-led case-study page. Leave it at index.html.',
maxSteps: 22,
});
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
const implementation = firstWrite(trace, /\.(?:html?|astro|svelte|jsx?|tsx?)$/i);
assert.ok(fileLoaded(trace, 'new-work.md'), `new-work.md was not loaded.\n${workflowTraceMessage(trace)}`);
assert.ok(question >= 0, `task concept was never put to the user.\n${workflowTraceMessage(trace)}`);
assert.ok(implementation > question, `implementation began before the attended concept checkpoint.\n${workflowTraceMessage(trace)}`);
const artifact = fs.readFileSync(path.join(workspace, 'index.html'), 'utf8');
assert.match(artifact.slice(0, 1400), /DIRECTION CONTRACT/i);
for (const field of ['UNIQUE', 'NOT-TEMPLATE', 'OWN-WORLD', 'STORY', 'FIRST VIEWPORT', 'FORM']) {
assert.match(artifact.slice(0, 1800), new RegExp(`${field}:`, 'i'));
}
} finally {
cleanupWorkspace(workspace);
}
});
it('redesign replaces DESIGN before touching the existing page', async () => {
const workspace = prepareWorkspace({
files: {
'PRODUCT.md': PRODUCT_MD_SAMPLE,
'DESIGN.md': LEGACY_DESIGN,
'current.html': EXISTING_PAGE,
},
});
try {
const { trace } = await runTurn({
workspace,
model,
userPrompt: '/impeccable craft redesign current.html for this product. Leave the result at current.html.',
maxSteps: 26,
});
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
const designWrite = firstWrite(trace, /(^|\/)DESIGN\.md$/i);
const implementation = firstWrite(trace, /(^|\/)current\.html$/i);
assert.ok(fileLoaded(trace, 'init.md'), `redesign did not route through init.\n${workflowTraceMessage(trace)}`);
assert.ok(question >= 0, `replacement world was not put to the user.\n${workflowTraceMessage(trace)}`);
assert.ok(designWrite > question, `replacement DESIGN.md must follow user choice.\n${workflowTraceMessage(trace)}`);
assert.ok(implementation > designWrite, `redesign touched the page before replacing DESIGN.md.\n${workflowTraceMessage(trace)}`);
const design = fs.readFileSync(path.join(workspace, 'DESIGN.md'), 'utf8');
assert.notEqual(design.trim(), LEGACY_DESIGN.trim(), 'redesign preserved the old visual world verbatim');
} finally {
cleanupWorkspace(workspace);
}
});
it('bolder refinement preserves the world and everything outside scope', async () => {
const workspace = prepareWorkspace({
files: {
'PRODUCT.md': PRODUCT_MD_SAMPLE,
'DESIGN.md': DESIGN_MD_SAMPLE,
'current.html': EXISTING_PAGE,
},
});
try {
const { trace } = await runTurn({
workspace,
model,
userPrompt: '/impeccable bolder current.html, only the #case-study section. Keep everything else untouched.',
maxSteps: 16,
});
const productWrite = firstWrite(trace, /(^|\/)PRODUCT\.md$/i);
const designWrite = firstWrite(trace, /(^|\/)DESIGN\.md$/i);
const implementation = firstWrite(trace, /(^|\/)current\.html$/i);
assert.ok(fileLoaded(trace, 'bolder.md'), `bolder.md was not loaded.\n${workflowTraceMessage(trace)}`);
assert.equal(productWrite, -1, `refinement rewrote PRODUCT.md.\n${workflowTraceMessage(trace)}`);
assert.equal(designWrite, -1, `refinement rewrote DESIGN.md.\n${workflowTraceMessage(trace)}`);
assert.ok(implementation >= 0, `refinement did not write current.html.\n${workflowTraceMessage(trace)}`);
const artifact = fs.readFileSync(path.join(workspace, 'current.html'), 'utf8');
assert.match(artifact, /data-untouched="header"/);
assert.match(artifact, /data-untouched="footer"/);
assert.match(artifact, /id="case-study"/);
} finally {
cleanupWorkspace(workspace);
}
});
});
}
+43
View File
@@ -0,0 +1,43 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import test from 'node:test';
import { ANTIPATTERNS } from '../cli/engine/registry/antipatterns.mjs';
const CRITIQUE_ONLY_RULES = new Set([
'glassmorphism',
'over-round',
'sketchy-svg',
'hero-metric-layout',
'identical-card-grids',
]);
test('the Slop catalog covers every detector rule', () => {
const source = fs.readFileSync(new URL('../site/pages/slop/index.astro', import.meta.url), 'utf8');
const staticRuleIds = [...source.matchAll(/id="rule-([^"]+)"/g)].map((match) => match[1]);
const catalogLists = source.match(/const CATALOG_RULE_IDS = \{([\s\S]*?)\n\};/);
assert.ok(catalogLists, 'CATALOG_RULE_IDS should remain easy to audit');
const dynamicRuleIds = [...catalogLists[1].matchAll(/'([^']+)'/g)].map((match) => match[1]);
const catalogRuleIds = new Set([...staticRuleIds, ...dynamicRuleIds]);
const registryRuleIds = new Set(ANTIPATTERNS.map((rule) => rule.id));
const missingRuleIds = [...registryRuleIds].filter((id) => !catalogRuleIds.has(id));
const critiqueOnlyRuleIds = [...catalogRuleIds].filter((id) => !registryRuleIds.has(id));
assert.deepEqual(missingRuleIds, []);
assert.deepEqual(new Set(critiqueOnlyRuleIds), CRITIQUE_ONLY_RULES);
assert.equal(catalogRuleIds.size, ANTIPATTERNS.length + CRITIQUE_ONLY_RULES.size);
});
test('new detector rules read like catalog entries, not release notes', () => {
const source = fs.readFileSync(new URL('../site/pages/slop/index.astro', import.meta.url), 'utf8');
const copyBlock = source.match(/const CATALOG_RULE_COPY = \{([\s\S]*?)\n\};/);
assert.ok(copyBlock, 'CATALOG_RULE_COPY should remain easy to audit');
assert.doesNotMatch(source, /latest detector coverage|catalog had fallen behind/i);
const descriptions = [...copyBlock[1].matchAll(/:\s*'([^']+)'/g)].map((match) => match[1]);
assert.equal(descriptions.length, 18);
assert.ok(descriptions.every((description) => description.length <= 155));
});
+26
View File
@@ -0,0 +1,26 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
getStoredPref,
nextPref,
setStoredPref,
} from '../site/scripts/utils/theme.js';
test('dark is the first-visit theme and the switcher keeps auto explicit', () => {
const values = new Map();
globalThis.localStorage = {
getItem: (key) => values.get(key) ?? null,
setItem: (key, value) => values.set(key, value),
};
assert.equal(getStoredPref(), 'dark');
assert.equal(nextPref('dark'), 'light');
assert.equal(nextPref('light'), 'auto');
assert.equal(nextPref('auto'), 'dark');
setStoredPref('auto');
assert.equal(getStoredPref(), 'auto');
delete globalThis.localStorage;
});