skills/pbakaus_impeccable
1
0
Fork 0
mirror of https://github.com/pbakaus/impeccable.git synced 2026-09-18 09:06:53 +03:00
Code Issues Packages Projects Releases Wiki Activity

Preserve experimental Live app-server workstream

Browse Source 
Snapshot the current app-server implementation, shared Live optimizations, generated harness output, and in-progress site work before restoring polling as the primary runtime path.

Prepared with Codex assistance under maintainer direction.
This commit is contained in:
Paul Bakaus
2026-07-15 16:07:34 -07:00
parent ed7a6fbe4e
commit ead6ddabe5
603 changed files with 148852 additions and 7584 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/framework-fixtures.test.mjs
+10
View File
@@ -117,16 +117,26 @@ for (const name of listFixtures()) {
const ignored = execFileSync('git', [
'check-ignore',
'.impeccable/live/server.json',
'.impeccable/live/codex-worker.json',
'.impeccable/live/codex-worker.log',
'.impeccable/live/sessions/example.jsonl',
'.impeccable/live/previews/example/v1.html',
'.impeccable/live/artifacts/example-r1.jsx',
'.impeccable/live/accept-receipts/example.json',
'.impeccable/live/locks/example.lock',
'.impeccable/live/deferred-svelte-component-accepts.json',
'src/lib/impeccable/ImpeccableLiveRoot.svelte',
'src/lib/impeccable/__runtime.js',
'src/lib/impeccable/a4ac4e74/v3.svelte',
], { cwd: tmp, encoding: 'utf-8' });
assert.match(ignored, /\.impeccable\/live\/server\.json/);
assert.match(ignored, /\.impeccable\/live\/codex-worker\.json/);
assert.match(ignored, /\.impeccable\/live\/codex-worker\.log/);
assert.match(ignored, /\.impeccable\/live\/sessions\/example\.jsonl/);
assert.match(ignored, /\.impeccable\/live\/previews\/example\/v1\.html/);
assert.match(ignored, /\.impeccable\/live\/artifacts\/example-r1\.jsx/);
assert.match(ignored, /\.impeccable\/live\/accept-receipts\/example\.json/);
assert.match(ignored, /\.impeccable\/live\/locks\/example\.lock/);
assert.match(ignored, /\.impeccable\/live\/deferred-svelte-component-accepts\.json/);
assert.match(ignored, /src\/lib\/impeccable\/ImpeccableLiveRoot\.svelte/);
assert.match(ignored, /src\/lib\/impeccable\/__runtime\.js/);
tests/live-accept.test.mjs
+51 -1
View File
@@ -5,11 +5,12 @@
import { describe, it, beforeEach, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
import { existsSync, mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import { tmpdir } from 'node:os';
import { fileURLToPath } from 'node:url';
import { execFileSync } from 'node:child_process';
import { scaffoldSourceArtifactSession } from '../skill/scripts/live/source-artifact.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const ACCEPT = resolve(__dirname, '..', 'skill/scripts/live-accept.mjs');
@@ -29,6 +30,55 @@ function runAccept(cwd, args) {
}
}
describe('live-accept — isolated source artifacts', () => {
let tmp;
beforeEach(() => { tmp = mkdtempSync(join(tmpdir(), 'impeccable-accept-isolated-')); });
afterEach(() => { rmSync(tmp, { recursive: true, force: true }); });
function scaffold(id) {
const original = '<main>\n <section class="hero"><h1>Original</h1></section>\n</main>\n';
writeFileSync(join(tmp, 'page.html'), original);
const session = scaffoldSourceArtifactSession({
id,
count: 2,
sourceFile: 'page.html',
sourceStartLine: 2,
sourceEndLine: 2,
originalSource: '<section class="hero"><h1>Original</h1></section>',
previewContent: `<main>
<!-- impeccable-variants-start ${id} -->
<div data-impeccable-variants="${id}" data-impeccable-variant-count="2" style="display: contents">
<div data-impeccable-variant="original"><section class="hero"><h1>Original</h1></section></div>
<div data-impeccable-variant="1"><section class="hero"><h1>Accepted one</h1></section></div>
<div data-impeccable-variant="2"><section class="hero"><h1>Accepted two</h1></section></div>
</div>
<!-- impeccable-variants-end ${id} -->
</main>
`,
cwd: tmp,
});
return { original, session };
}
it('accepts one preview into true source exactly once', () => {
const { session } = scaffold('isolatedaccept');
const result = runAccept(tmp, ['--id', 'isolatedaccept', '--variant', '2']);
assert.equal(result.handled, true, JSON.stringify(result));
const source = readFileSync(join(tmp, 'page.html'), 'utf-8');
assert.match(source, /Accepted two/);
assert.doesNotMatch(source, /Accepted one|data-impeccable-variant/);
assert.equal(existsSync(join(tmp, session.sessionDir)), false);
});
it('discards the preview instantly without touching true source', () => {
const { original, session } = scaffold('isolateddiscard');
const result = runAccept(tmp, ['--id', 'isolateddiscard', '--discard']);
assert.equal(result.handled, true, JSON.stringify(result));
assert.equal(readFileSync(join(tmp, 'page.html'), 'utf-8'), original);
assert.equal(existsSync(join(tmp, session.sessionDir)), false);
});
});
describe('live-accept — style-element edge cases', () => {
let tmp;
beforeEach(() => { tmp = mkdtempSync(join(tmpdir(), 'impeccable-accept-test-')); });
tests/live-browser-regression.test.mjs
+43 -1
View File
@@ -141,11 +141,37 @@ describe('live-browser.js regression guards', () => {
it('restores unsaved inline edit drafts before hideBar tears editing down', () => {
assert.match(
SOURCE,
/function hideBar\(\) \{[\s\S]{0,620}?if \(state === 'EDITING'\) restoreInlineEditDrafts\(\);[\s\S]{0,80}?disableInlineEdit\(\);/,
/function hideBar\(instant\) \{[\s\S]{0,720}?if \(state === 'EDITING'\) restoreInlineEditDrafts\(\);[\s\S]{0,80}?disableInlineEdit\(\);/,
'hideBar should not leave unsaved contenteditable drafts in the DOM when an external event hides the bar',
);
});
it('discards variants without hiding the original or animating stale chrome', () => {
assert.match(SOURCE, /function showOriginalDuringDiscard\(sessionId\)[\s\S]{0,900}?data-impeccable-variant="original"/);
assert.match(SOURCE, /function handleDiscard\(\)[\s\S]{0,420}?cleanup\(\{ restoreOriginal: true, instantChrome: true \}\)/);
assert.match(SOURCE, /if \(instant\) barEl\.style\.display = 'none'/);
assert.match(
SOURCE,
/if \(restoreOriginal\) showOriginalDuringDiscard\(cleanupSessionId\);\s*else wrapper\.style\.display = 'none';/,
'only non-discard cleanup may blank the wrapper while waiting for HMR',
);
});
it('stores live state off the document root and preserves the selected anchor top', () => {
assert.match(SOURCE, /window\.__IMPECCABLE_LIVE_STATE__ = next/);
assert.doesNotMatch(SOURCE, /document\.documentElement\.dataset\.impeccableLiveState/);
assert.match(SOURCE, /pickedAnchorViewportTop: Number\.isFinite\(pickedAnchorViewportTop\)/);
assert.match(SOURCE, /scrollLockAnchorTop = typeof initialAnchorTop === 'number' && isFinite\(initialAnchorTop\)/);
assert.match(SOURCE, /const anchorDelta = anchorTop - scrollLockAnchorTop/);
});
it('injects source-artifact previews immediately instead of waiting for HMR', () => {
assert.match(
SOURCE,
/else if \(isSourceArtifactPreviewMode\(msg\.previewMode\) && msg\.previewFile\) \{\s*injectVariantsFromSource\(msg\.previewFile/,
);
});
it('does not autofocus the steering chat while inline editing', () => {
assert.match(
SOURCE,
@@ -869,6 +895,22 @@ describe('live-browser.js regression guards', () => {
);
});
it('keeps deferred Tune controls visible and refreshes params-only publications', () => {
assert.match(
SOURCE,
/const paramsPending = !hasParams && \(parameterGenerationState === 'pending' \|\| parameterGenerationState === 'loading'\)/,
'the cycling bar must expose Tune while parameter generation is outstanding',
);
assert.match(SOURCE, /tune\.disabled = true/, 'pending Tune must be visibly loading but non-interactive');
assert.match(SOURCE, /Tune controls are ready\./, 'parameter arrival needs a clear ready indication');
assert.match(
SOURCE,
/msg\.publicationKind !== 'params' && arrivedVariants >= targetArrived/,
'a params-only publication must refresh even though the variant count is unchanged',
);
assert.match(SOURCE, /revisionDomain: 'browser'/, 'browser checkpoints must use their own revision domain');
});
it('promotes an early-accepted Svelte preview before releasing the picker', () => {
assert.match(
SOURCE,
tests/live-codex-worker-supervisor.test.mjs
+253 -27
View File
@@ -9,7 +9,9 @@ import {
CODEX_WORKER_EVENT_LEASE_MS,
CODEX_WORKER_EVENT_TYPES,
CodexLiveWorkerSupervisor,
buildDetectorRepairPrompt,
buildDeterministicScaffoldCommand,
resolveDetectorFindingWaivers,
} from '../skill/scripts/live/codex-worker-supervisor.mjs';
import { createLiveSessionStore } from '../skill/scripts/live/session-store.mjs';
import { selectAvailablePendingEvent } from '../skill/scripts/live/poll-lanes.mjs';
@@ -42,7 +44,7 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
}, '/scripts');
assert.equal(replace.script, '/scripts/live-wrap.mjs');
assert.deepEqual(replace.args, [
'--id', 'abc12345', '--count', '3', '--element-id', 'hero',
'--id', 'abc12345', '--count', '3', '--isolated', '--element-id', 'hero',
'--classes', 'hero,title', '--tag', 'h1', '--text', 'Exact hero copy',
]);
const insert = buildDeterministicScaffoldCommand({
@@ -82,6 +84,7 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
cwd,
threadId: 'live-worker-thread',
status: 'ready',
threadPrimed: true,
}));
const client = fakeClient();
const supervisor = createSupervisor({ cwd, statePath, client });
@@ -89,6 +92,8 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
assert.equal(client.calls.resumeDedicatedThread.length, 1);
assert.equal(client.calls.resumeDedicatedThread[0].threadId, 'live-worker-thread');
assert.equal(client.calls.startDedicatedThread.length, 0);
assert.equal(supervisor.threadPrimed, true, 'a resumed thread must not receive the skill attachment again');
assert.equal(JSON.parse(readFileSync(statePath, 'utf-8')).threadPrimed, true);
await supervisor.shutdown();
});
@@ -350,6 +355,208 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
assert.equal(attempts, 2);
});
it('repairs new detector findings on the same persistent thread before publication', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-detector-repair-'));
mkdirSync(path.join(cwd, 'src'), { recursive: true });
const sessionId = 'detectorrepair';
writeFileSync(path.join(cwd, 'src/App.jsx'), [
'<main>',
` <div data-impeccable-variants="${sessionId}" data-impeccable-variant-count="3">`,
` <style data-impeccable-css="${sessionId}"></style>`,
' <div data-impeccable-variant="original"><h1>Original</h1></div>',
` {/* impeccable-variants-end ${sessionId} */}`,
' </div>',
'</main>',
].join('\n'));
createLiveSessionStore({ cwd, sessionId }).appendEvent({
type: 'generate', id: sessionId, count: 3, generationEpoch: 1,
});
const plan = {
identityLock: ['Preserve identity'],
directions: [
{ variantId: 1, name: 'One', axis: 'hierarchy', intent: 'Strengthen hierarchy' },
{ variantId: 2, name: 'Two', axis: 'layout', intent: 'Recompose layout' },
{ variantId: 3, name: 'Three', axis: 'density', intent: 'Adjust density' },
],
};
const client = fakeClient();
const turnThreadIds = [];
const prompts = [];
const outputSchemas = [];
let turn = 0;
client.startTurn = async ({ threadId, input, outputSchema, onStarted }) => {
turn += 1;
turnThreadIds.push(threadId);
prompts.push(input.find((item) => item.type === 'text').text);
outputSchemas.push(outputSchema);
onStarted?.(`turn-${turn}`);
return { message: JSON.stringify({
sourceDelta: {
variantId: 1,
markup: `<h1>${turn === 1 ? 'Flagged' : 'Repaired'}</h1>`,
css: '@scope ([data-impeccable-variant="1"]) { h1 { color: currentColor; } }',
},
plan,
...(turn === 1 ? {} : { detectorWaivers: [] }),
}) };
};
let detectorCall = 0;
const supervisor = new CodexLiveWorkerSupervisor({
cwd,
base: 'http://localhost:1',
token: 'token',
client,
config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
statePath: path.join(cwd, '.impeccable/live/codex-worker.json'),
scriptsDir: path.resolve('skill/scripts'),
detectCandidate: () => {
detectorCall += 1;
return detectorCall === 2
? [{ antipattern: 'gradient-text', name: 'Gradient text', snippet: 'flagged candidate', file: 'App.jsx' }]
: [];
},
publishCheckpoint: async () => {},
publishPhase: async () => {},
});
supervisor.thread = { id: 'persistent-live-thread' };
supervisor.model = client.models[0];
await supervisor.runGenerationPhaseOnce({
type: 'generate',
id: sessionId,
count: 3,
scaffold: { file: 'src/App.jsx', styleMode: 'scoped' },
}, 'first', 1);
assert.deepEqual(turnThreadIds, ['persistent-live-thread', 'persistent-live-thread']);
assert.match(prompts[1], /new Impeccable detector findings/);
assert.equal(outputSchemas[0].properties.detectorWaivers, undefined);
assert.equal(outputSchemas[1].properties.detectorWaivers.type, 'array');
assert.match(readFileSync(path.join(cwd, 'src/App.jsx'), 'utf-8'), /Repaired/);
assert.doesNotMatch(readFileSync(path.join(cwd, 'src/App.jsx'), 'utf-8'), /Flagged/);
assert.equal(detectorCall, 3);
});
it('accepts only explicit narrow detector false-positive waivers', () => {
const findings = [
{
antipattern: 'gradient-text',
file: '/project/src/App.jsx',
snippet: 'intentional campaign wordmark',
ignoreValue: '',
},
{
antipattern: 'overused-font',
file: '/project/src/App.jsx',
snippet: 'font-family: Inter',
ignoreValue: 'Inter',
},
];
const resolved = resolveDetectorFindingWaivers(findings, [
{
rule: 'gradient-text',
file: 'App.jsx',
snippet: 'intentional campaign wordmark',
ignoreValue: '',
reason: 'The selected element is the established campaign wordmark.',
},
{
rule: 'overused-font',
file: 'App.jsx',
snippet: '',
ignoreValue: 'Roboto',
reason: 'Wrong value must not waive the finding.',
},
]);
assert.deepEqual(resolved.accepted.map(({ waiver }) => waiver.rule), ['gradient-text']);
assert.deepEqual(resolved.unresolved, [findings[1]]);
assert.match(buildDetectorRepairPrompt('first', findings), /Fix real defects/);
assert.match(buildDetectorRepairPrompt('first', findings), /contextually intentional or a detector false positive/);
assert.match(buildDetectorRepairPrompt('first', findings), /do not.*persist project detector config/i);
});
it('publishes a candidate when the repair turn explicitly waives the remaining false positive', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-detector-waiver-'));
mkdirSync(path.join(cwd, 'src'), { recursive: true });
const sessionId = 'detectorwaiver';
writeFileSync(path.join(cwd, 'src/App.jsx'), [
'<main>',
` <div data-impeccable-variants="${sessionId}" data-impeccable-variant-count="1">`,
` <style data-impeccable-css="${sessionId}"></style>`,
' <div data-impeccable-variant="original"><h1>Original</h1></div>',
` {/* impeccable-variants-end ${sessionId} */}`,
' </div>',
'</main>',
].join('\n'));
createLiveSessionStore({ cwd, sessionId }).appendEvent({
type: 'generate', id: sessionId, count: 1, generationEpoch: 1,
});
const finding = {
antipattern: 'gradient-text',
name: 'Gradient text',
snippet: 'intentional campaign wordmark',
ignoreValue: '',
file: 'App.jsx',
};
const client = fakeClient();
let turn = 0;
client.startTurn = async ({ onStarted }) => {
turn += 1;
onStarted?.(`turn-${turn}`);
return { message: JSON.stringify({
sourceDelta: {
variantId: 1,
markup: '<h1>Intentional wordmark</h1>',
css: '@scope ([data-impeccable-variant="1"]) { h1 { color: currentColor; } }',
},
...(turn === 1 ? {} : {
detectorWaivers: [{
rule: 'gradient-text',
file: 'App.jsx',
snippet: finding.snippet,
ignoreValue: '',
reason: 'This selected element is the established campaign wordmark.',
}],
}),
}) };
};
let detectorCall = 0;
const supervisor = new CodexLiveWorkerSupervisor({
cwd,
base: 'http://localhost:1',
token: 'token',
client,
config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
statePath: path.join(cwd, '.impeccable/live/codex-worker.json'),
scriptsDir: path.resolve('skill/scripts'),
detectCandidate: () => (++detectorCall === 1 ? [] : [finding]),
publishCheckpoint: async () => {},
publishPhase: async () => {},
});
supervisor.thread = { id: 'persistent-live-thread' };
supervisor.model = client.models[0];
await supervisor.runGenerationPhaseOnce({
type: 'generate',
id: sessionId,
count: 1,
scaffold: { file: 'src/App.jsx', styleMode: 'scoped' },
}, 'first', 1);
assert.equal(turn, 2);
assert.equal(detectorCall, 3);
assert.match(readFileSync(path.join(cwd, 'src/App.jsx'), 'utf-8'), /Intentional wordmark/);
const snapshot = createLiveSessionStore({ cwd, sessionId }).getSnapshot(sessionId, { includeCompleted: true });
assert.deepEqual(snapshot.detectorWaivers, [{
rule: 'gradient-text',
file: 'App.jsx',
snippet: finding.snippet,
ignoreValue: '',
reason: 'This selected element is the established campaign wordmark.',
}]);
});
it('resumes progressive delivery from durable variant checkpoints', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-checkpoint-resume-'));
const phases = [];
@@ -381,10 +588,7 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
generationEpoch: 1,
scaffold: { file: 'src/App.jsx' },
});
assert.deepEqual(phases, [
{ phase: 'second', arrivedVariants: 2 },
{ phase: 'final', arrivedVariants: 3 },
]);
assert.deepEqual(phases, [{ phase: 'remainder', arrivedVariants: 3 }]);
assert.equal(replies.at(-1).type, 'done');
phases.length = 0;
@@ -399,7 +603,7 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
generationEpoch: 1,
scaffold: { file: 'src/App.jsx' },
});
assert.deepEqual(phases, []);
assert.deepEqual(phases, [{ phase: 'params', arrivedVariants: 3 }]);
assert.equal(replies.at(-1).id, completeId);
assert.equal(replies.at(-1).type, 'done');
});
@@ -445,6 +649,10 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
it('publishes progressive source checkpoints only through the fenced publisher', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-publish-'));
mkdirSync(path.join(cwd, 'src'), { recursive: true });
mkdirSync(path.join(cwd, 'skill'), { recursive: true });
writeFileSync(path.join(cwd, 'PRODUCT.md'), '# Product\nStable product context');
writeFileSync(path.join(cwd, 'DESIGN.md'), '# Design\nStable design context');
writeFileSync(path.join(cwd, 'skill/SKILL.md'), '# Impeccable skill');
const sessionId = 'codexprogress';
const original = '<main><div data-impeccable-variants="codexprogress"><style data-impeccable-css="codexprogress"></style><div data-impeccable-variant="original"><h1>Original</h1></div></div></main>';
writeFileSync(path.join(cwd, 'src/App.jsx'), original);
@@ -457,6 +665,7 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
const client = fakeClient();
let turn = 0;
const prompts = [];
const turnInputs = [];
const plan = {
identityLock: ['Preserve copy and shared component roles'],
directions: [
@@ -467,29 +676,37 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
};
client.startTurn = async ({ input, onStarted, onAgentMessage }) => {
turn += 1;
turnInputs.push(input);
onStarted?.(`turn-${turn}`);
const prompt = input.find((item) => item.type === 'text').text;
prompts.push(prompt);
const message = turn <= 2
const message = turn === 1
? JSON.stringify({
sourceDelta: {
variantId: turn,
markup: turn === 1 ? '<h1>One</h1>' : '<h1>Two</h1>',
css: turn === 1
? '@scope ([data-impeccable-variant="1"]) { h1 { color: red; } }'
: '@scope ([data-impeccable-variant="2"]) { h1 { color: green; } }',
variantId: 1,
markup: '<h1>One</h1>',
css: '@scope ([data-impeccable-variant="1"]) { h1 { color: red; } }',
},
...(turn === 1 ? { plan } : {}),
plan,
})
: JSON.stringify({
sourceDelta: {
variantId: 3,
markup: '<h1>Three</h1>',
css: '@scope ([data-impeccable-variant="3"]) { h1 { color: blue; } }',
: turn === 2
? JSON.stringify({
sourceDeltas: [
{
variantId: 2,
markup: '<h1>Two</h1>',
css: '@scope ([data-impeccable-variant="2"]) { h1 { color: green; } }',
},
{
variantId: 3,
markup: '<h1>Three</h1>',
css: '@scope ([data-impeccable-variant="3"]) { h1 { color: blue; } }',
},
],
parameterCss: '',
paramsJson: '{"1":[],"2":[],"3":[]}',
},
});
})
: JSON.stringify({ parameterCss: '', paramsJson: '{"1":[],"2":[],"3":[]}' });
await Promise.all([
onAgentMessage?.(message),
onAgentMessage?.(message),
@@ -527,16 +744,14 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
scaffold: { file: 'src/App.jsx', styleMode: 'scoped' },
});
assert.equal(checkpoints.length, 3);
assert.deepEqual(checkpoints.map((item) => item.arrivedVariants), [1, 2, 3]);
assert.equal(checkpoints.length, 2);
assert.deepEqual(checkpoints.map((item) => item.arrivedVariants), [1, 3]);
assert.deepEqual(phases.map((item) => item.phase), [
'first_variant_generating',
'first_variant_validating',
'first_variant_validating',
'second_variant_generating',
'second_variant_validating',
'remaining_variants_generating',
'remaining_variants_validating',
'parameters_ready',
]);
assert.equal(replies.at(-1).type, 'done');
const publishedSource = readFileSync(path.join(cwd, 'src/App.jsx'), 'utf-8');
@@ -546,10 +761,21 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
assert.doesNotMatch(publishedSource, /Mutated/);
const snapshot = createLiveSessionStore({ cwd, sessionId }).getSnapshot(sessionId, { includeCompleted: true });
assert.equal(snapshot.arrivedVariants, 3);
assert.equal(snapshot.publishedRevision, 3);
assert.equal(snapshot.publishedRevision, 2);
assert.equal(snapshot.paramsPublished, true);
assert.deepEqual(snapshot.variantPlan, plan);
assert.equal(checkpointAttempts, 4, 'the durable first publication only retries its checkpoint');
assert.equal(checkpointAttempts, 3, 'the durable first publication retries only its checkpoint');
assert.match(prompts[1], /"name": "Composition"/);
assert.equal(turnInputs[0].some((item) => item.type === 'skill'), true);
assert.equal(turnInputs[1].some((item) => item.type === 'skill'), false);
assert.equal(JSON.parse(readFileSync(path.join(cwd, '.impeccable/live/codex-worker.json'), 'utf-8')).threadPrimed, true);
assert.equal(client.calls.startDedicatedThread.length, 0, 'both turns stay on the existing durable thread');
assert.equal(client.calls.archiveThread.length, 0);
assert.match(prompts[0], /Stable product context/);
assert.match(prompts[0], /Stable design context/);
assert.doesNotMatch(prompts[0], /<source_neighborhood>/);
assert.doesNotMatch(prompts[1], /Stable product context|<source_neighborhood>/);
assert.match(prompts[1], /parameterCss and paramsJson/);
});
});
tests/live-codex-worker.test.mjs
+133 -78
View File
@@ -11,6 +11,7 @@ import {
buildCodexWorkerInstructions,
buildCodexWorkerTurnInputs,
buildGenerationTurnInput,
codexWorkerDetectorRepairSchema,
codexWorkerOutputSchemaForPhase,
codexWorkerProcessStateIsOwned,
codexWorkerStateIsOwned,
@@ -303,7 +304,7 @@ describe('Codex Live worker structured artifact boundary', () => {
it('keeps the model read-only and the supervisor as the only publisher', () => {
const instructions = buildCodexWorkerInstructions('LIVE SPEC');
assert.match(instructions, /Do not write source/);
assert.match(instructions, /read-only tools only/);
assert.match(instructions, /read-only repository tools whenever needed/);
assert.match(instructions, /supervisor alone writes staged artifacts/);
assert.match(instructions, /shared-component visual roles/);
assert.match(instructions, /recompose the selected element itself/);
@@ -317,27 +318,28 @@ describe('Codex Live worker structured artifact boundary', () => {
it('requires a coherent variant plan before progressive or atomic multi-variant output', () => {
const firstSchema = codexWorkerOutputSchemaForPhase('first', 3);
const finalSchema = codexWorkerOutputSchemaForPhase('final', 3);
const paramsSchema = codexWorkerOutputSchemaForPhase('params', 3);
assert.deepEqual(firstSchema.required, ['files', 'plan']);
assert.ok(firstSchema.properties.plan);
assert.deepEqual(codexWorkerOutputSchemaForPhase('atomic', 3).required, ['files', 'plan']);
assert.deepEqual(finalSchema.required, ['files']);
assert.equal(finalSchema.properties.plan, undefined, 'strict schemas cannot expose optional properties');
assert.deepEqual(paramsSchema.required, ['files']);
assert.equal(paramsSchema.properties.plan, undefined, 'strict schemas cannot expose optional properties');
assert.deepEqual(codexWorkerOutputSchemaForPhase('atomic', 1).required, ['files']);
assert.deepEqual(
codexWorkerOutputSchemaForPhase('second', 3, { sourceDelta: true }).required,
['sourceDelta'],
codexWorkerOutputSchemaForPhase('remainder', 3, { sourceDelta: true }).required,
['sourceDeltas', 'parameterCss', 'paramsJson'],
);
assert.deepEqual(
codexWorkerOutputSchemaForPhase('first', 3, { sourceDelta: true }).required,
['sourceDelta', 'plan'],
);
const finalDelta = codexWorkerOutputSchemaForPhase('final', 3, { sourceDelta: true });
assert.deepEqual(finalDelta.required, ['sourceDelta']);
assert.deepEqual(finalDelta.properties.sourceDelta.required, [
'variantId', 'markup', 'css', 'parameterCss', 'paramsJson',
]);
assert.equal(finalDelta.properties.sourceDelta.properties.variantId.minimum, 3);
const parameterDelta = codexWorkerOutputSchemaForPhase('params', 3, { sourceDelta: true });
assert.deepEqual(parameterDelta.required, ['parameterCss', 'paramsJson']);
assert.equal(parameterDelta.properties.sourceDelta, undefined);
const repairSchema = codexWorkerDetectorRepairSchema(firstSchema);
assert.deepEqual(repairSchema.required, ['files', 'plan', 'detectorWaivers']);
assert.equal(repairSchema.properties.detectorWaivers.type, 'array');
assert.equal(firstSchema.properties.detectorWaivers, undefined, 'normal generation cannot invent waivers');
});
it('writes only the prepared source artifact path', () => {
@@ -419,7 +421,7 @@ describe('Codex Live worker structured artifact boundary', () => {
);
});
it('merges a fenced variant 2 delta without letting the model resend variant 1', () => {
it('merges the fenced remaining variants without letting the model resend variant 1', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-source-delta-'));
const artifact = path.join(cwd, '.impeccable/live/artifacts/session-r2.jsx');
mkdirSync(path.dirname(artifact), { recursive: true });
@@ -440,14 +442,23 @@ describe('Codex Live worker structured artifact boundary', () => {
applyCodexWorkerOutput({
output: {
sourceDelta: {
variantId: 2,
markup: '<article className="two"><h1>Two</h1></article>',
css: '@scope ([data-impeccable-variant="2"]) { :scope > .two { color: green; } }',
},
sourceDeltas: [
{
variantId: 2,
markup: '<article className="two"><h1>Two</h1></article>',
css: '@scope ([data-impeccable-variant="2"]) { :scope > .two { color: green; } }',
},
{
variantId: 3,
markup: '<article className="three"><h1>Three</h1></article>',
css: '@scope ([data-impeccable-variant="3"]) { :scope > .three { color: blue; } }',
},
],
parameterCss: '',
paramsJson: emptyParamsJson(),
},
prepared,
phase: 'second',
phase: 'remainder',
expectedVariants: 3,
sessionId: 'session',
scaffold: { styleMode: 'scoped' },
@@ -466,14 +477,23 @@ describe('Codex Live worker structured artifact boundary', () => {
assert.throws(() => applyCodexWorkerOutput({
output: {
sourceDelta: {
variantId: 2,
markup: '<article>Unsafe</article>',
css: '@scope ([data-impeccable-variant="1"]) { :scope { color: hotpink; } }',
},
sourceDeltas: [
{
variantId: 2,
markup: '<article>Unsafe</article>',
css: '@scope ([data-impeccable-variant="1"]) { :scope { color: hotpink; } }',
},
{
variantId: 3,
markup: '<article>Three</article>',
css: '@scope ([data-impeccable-variant="3"]) { :scope > article { color: blue; } }',
},
],
parameterCss: '',
paramsJson: emptyParamsJson(),
},
prepared: { ...prepared, artifactFile: prepared.artifactFile },
phase: 'second',
phase: 'remainder',
expectedVariants: 3,
sessionId: 'session',
scaffold: { styleMode: 'scoped' },
@@ -519,14 +539,23 @@ describe('Codex Live worker structured artifact boundary', () => {
});
applyCodexWorkerOutput({
output: {
sourceDelta: {
variantId: 2,
markup: '<article className="two">Two</article>',
css: '@scope ([data-impeccable-variant="2"]) { :scope > .two { color: green; } }',
},
sourceDeltas: [
{
variantId: 2,
markup: '<article className="two">Two</article>',
css: '@scope ([data-impeccable-variant="2"]) { :scope > .two { color: green; } }',
},
{
variantId: 3,
markup: '<article className="three">Three</article>',
css: '@scope ([data-impeccable-variant="3"]) { :scope > .three { color: blue; } }',
},
],
parameterCss: '',
paramsJson: emptyParamsJson(),
},
prepared,
phase: 'second',
phase: 'remainder',
expectedVariants: 3,
sessionId: 'session',
scaffold: { styleMode: 'scoped' },
@@ -566,14 +595,23 @@ describe('Codex Live worker structured artifact boundary', () => {
applyCodexWorkerOutput({
output: {
sourceDelta: {
variantId: 2,
markup: '<article class="two"><h1>Two</h1></article>',
css: '[data-impeccable-variant="2"] > .two { color: green; }',
},
sourceDeltas: [
{
variantId: 2,
markup: '<article class="two"><h1>Two</h1></article>',
css: '[data-impeccable-variant="2"] > .two { color: green; }',
},
{
variantId: 3,
markup: '<article class="three"><h1>Three</h1></article>',
css: '[data-impeccable-variant="3"] > .three { color: blue; }',
},
],
parameterCss: '',
paramsJson: emptyParamsJson(),
},
prepared: { artifactFile: '.impeccable/live/artifacts/session-r2.astro' },
phase: 'second',
phase: 'remainder',
expectedVariants: 3,
sessionId: 'session',
scaffold: { styleMode: 'astro-global-prefixed' },
@@ -582,13 +620,13 @@ describe('Codex Live worker structured artifact boundary', () => {
const after = readFileSync(artifact, 'utf-8');
assert.match(after, /\[data-impeccable-variant="2"\] > \.two/);
assert.match(after, /<div data-impeccable-variant="2">/);
assert.match(after, /<div data-impeccable-variant="2"[^>]*>/);
assert.doesNotMatch(after, /@scope/);
assert.match(after, /<!-- impeccable-variants-end session -->/);
assert.ok(after.indexOf('<div data-impeccable-variant="2">') < after.indexOf('impeccable-variants-end session'));
});
it('appends the final source variant and deferred parameter manifest without rewriting prior output', () => {
it('publishes the remaining source variants and parameters together without rewriting prior output', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-final-delta-'));
const artifact = path.join(cwd, 'App.jsx');
writeFileSync(artifact, [
@@ -596,11 +634,9 @@ describe('Codex Live worker structured artifact boundary', () => {
' <div data-impeccable-variants="session" data-impeccable-variant-count="3">',
' <style data-impeccable-css="session">{`',
'@scope ([data-impeccable-variant="1"]) { :scope > .one { color: red; } }',
'@scope ([data-impeccable-variant="2"]) { :scope > .two { color: green; } }',
'`}</style>',
' <div data-impeccable-variant="original"><article>Original</article></div>',
' <div data-impeccable-variant="1"><article className="one">Immutable one</article></div>',
' <div data-impeccable-variant="2"><article className="two">Immutable two</article></div>',
' {/* impeccable-variants-end session */}',
' </div>',
'</main>',
@@ -619,20 +655,27 @@ describe('Codex Live worker structured artifact boundary', () => {
applyCodexWorkerOutput({
output: {
sourceDelta: {
variantId: 3,
markup: '<article className="three">Three</article>',
css: '@scope ([data-impeccable-variant="3"]) { :scope > .three { color: blue; } }',
parameterCss: [
'@scope ([data-impeccable-variant="1"]) { :scope[data-p-scale] > .one { scale: var(--p-scale); } }',
'@scope ([data-impeccable-variant="2"]) { :scope[data-p-dense] > .two { padding: 0; } }',
'@scope ([data-impeccable-variant="3"]) { :scope[data-p-face="sans"] > .three { font-family: sans-serif; } }',
].join('\n'),
paramsJson,
},
sourceDeltas: [
{
variantId: 2,
markup: '<article className="two">Two</article>',
css: '@scope ([data-impeccable-variant="2"]) { :scope > .two { color: green; } }',
},
{
variantId: 3,
markup: '<article className="three">Three</article>',
css: '@scope ([data-impeccable-variant="3"]) { :scope > .three { color: blue; } }',
},
],
parameterCss: [
'@scope ([data-impeccable-variant="1"]) { :scope[data-p-scale] > .one { scale: var(--p-scale); } }',
'@scope ([data-impeccable-variant="2"]) { :scope[data-p-dense] > .two { padding: 0; } }',
'@scope ([data-impeccable-variant="3"]) { :scope[data-p-face="sans"] > .three { font-family: sans-serif; } }',
].join('\n'),
paramsJson,
},
prepared: { artifactFile: 'App.jsx' },
phase: 'final',
phase: 'remainder',
expectedVariants: 3,
sessionId: 'session',
scaffold: { styleMode: 'scoped' },
@@ -641,14 +684,14 @@ describe('Codex Live worker structured artifact boundary', () => {
const after = readFileSync(artifact, 'utf-8');
assert.match(after, /Immutable one/);
assert.match(after, /Immutable two/);
assert.match(after, /className="two">Two/);
assert.match(after, /className="three">Three/);
assert.equal((after.match(/data-impeccable-params=/g) || []).length, 3);
assert.match(after, /data-p-scale/);
assert.ok(after.indexOf('className="three"') < after.indexOf('impeccable-variants-end session'));
});
it('never lets a final component turn rewrite arrived variant 1', () => {
it('never lets a remaining component turn rewrite arrived variant 1', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-component-'));
const componentDir = path.join(cwd, '.impeccable/live/artifacts/session-r2-svelte');
mkdirSync(componentDir, { recursive: true });
@@ -669,23 +712,23 @@ describe('Codex Live worker structured artifact boundary', () => {
() => applyCodexWorkerOutput({
output: { files: [{ path: 'v1.svelte', content: '<h1>Changed</h1>' }] },
prepared,
phase: 'final',
phase: 'remainder',
expectedVariants: 3,
cwd,
}),
/published_variant_changed/,
);
writeFileSync(path.join(componentDir, 'v2.svelte'), '<h1>Two</h1>');
applyCodexWorkerOutput({
output: {
files: [
{ path: 'v2.svelte', content: '<h1>Two</h1>' },
{ path: 'v3.svelte', content: '<h1>Three</h1>' },
{ path: 'params.json', content: '{"1":[],"2":[],"3":[]}' },
{ path: 'params.json', content: emptyParamsJson() },
],
},
prepared,
phase: 'final',
phase: 'remainder',
expectedVariants: 3,
cwd,
});
@@ -693,7 +736,7 @@ describe('Codex Live worker structured artifact boundary', () => {
assert.equal(JSON.parse(readFileSync(path.join(componentDir, 'manifest.json'))).arrivedVariants, 3);
});
it('publishes component variant 2 without waiting for variant 3 or parameters', () => {
it('publishes all remaining component variants and parameters in the second turn', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-component-second-'));
const componentDir = path.join(cwd, '.impeccable/live/artifacts/session-r2-svelte');
mkdirSync(componentDir, { recursive: true });
@@ -710,17 +753,22 @@ describe('Codex Live worker structured artifact boundary', () => {
};
applyCodexWorkerOutput({
output: { files: [{ path: 'v2.svelte', content: '<h1>Two</h1>' }] },
output: { files: [
{ path: 'v2.svelte', content: '<h1>Two</h1>' },
{ path: 'v3.svelte', content: '<h1>Three</h1>' },
{ path: 'params.json', content: emptyParamsJson() },
] },
prepared,
phase: 'second',
phase: 'remainder',
expectedVariants: 3,
cwd,
});
assert.equal(readFileSync(path.join(componentDir, 'v1.svelte'), 'utf-8'), '<h1>Immutable</h1>');
assert.equal(readFileSync(path.join(componentDir, 'v2.svelte'), 'utf-8'), '<h1>Two</h1>');
assert.equal(JSON.parse(readFileSync(path.join(componentDir, 'manifest.json'))).arrivedVariants, 2);
assert.equal(existsSync(path.join(componentDir, 'params.json')), false);
assert.equal(readFileSync(path.join(componentDir, 'v3.svelte'), 'utf-8'), '<h1>Three</h1>');
assert.equal(JSON.parse(readFileSync(path.join(componentDir, 'manifest.json'))).arrivedVariants, 3);
assert.equal(existsSync(path.join(componentDir, 'params.json')), true);
});
it('requires atomic component output to contain v1 through vN plus params', () => {
@@ -750,7 +798,7 @@ describe('Codex Live worker structured artifact boundary', () => {
}), /worker_output_component_file_missing/);
});
it('does not let precreated stubs satisfy missing final component output', () => {
it('does not let precreated stubs satisfy missing remaining component output', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-component-final-'));
const componentDir = path.join(cwd, '.impeccable/live/artifacts/session-r2-svelte');
mkdirSync(componentDir, { recursive: true });
@@ -767,15 +815,15 @@ describe('Codex Live worker structured artifact boundary', () => {
artifactFile: '.impeccable/live/artifacts/session-r2-svelte/manifest.json',
};
assert.throws(() => applyCodexWorkerOutput({
output: { files: [{ path: 'params.json', content: '{}' }] },
output: { files: [{ path: 'v2.svelte', content: '<h1>Two replacement</h1>' }] },
prepared,
phase: 'final',
phase: 'remainder',
expectedVariants: 3,
cwd,
}), /worker_output_component_file_missing/);
});
it('builds phase prompts from bounded staged evidence', () => {
it('builds phase prompts from the exact staged artifact and durable thread context', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-worker-context-'));
const artifactPath = path.join(cwd, 'artifact.html');
writeFileSync(artifactPath, '<main>wrapped</main>');
@@ -790,7 +838,6 @@ describe('Codex Live worker structured artifact boundary', () => {
design: 'Design tokens',
actionReference: 'Polish rules',
contextMetadata: { productPath: 'docs/PRODUCT.md' },
sourceNeighborhood: { 'src/Button.jsx': 'export function Button() {}' },
});
assert.match(prompt, /Produce only variant 1/);
assert.match(prompt, /strongest low-risk, independently shippable/);
@@ -803,27 +850,31 @@ describe('Codex Live worker structured artifact boundary', () => {
assert.match(prompt, /Product facts/);
assert.match(prompt, /Design tokens/);
assert.match(prompt, /docs\/PRODUCT\.md/);
assert.match(prompt, /src\/Button\.jsx/);
assert.doesNotMatch(prompt, /source_neighborhood/);
const finalPrompt = buildGenerationTurnInput({
const remainderPrompt = buildGenerationTurnInput({
event: { id: 'abc', count: 3 },
phase: 'final',
phase: 'remainder',
prepared,
artifact,
variantPlan: variantPlan(),
});
assert.match(finalPrompt, /Follow the durable variant plan/);
assert.match(finalPrompt, /Composition/);
assert.match(remainderPrompt, /Follow the durable variant plan/);
assert.match(remainderPrompt, /variants 2 through 3 and the final tunable parameters together/);
assert.match(remainderPrompt, /parameterCss and paramsJson/);
assert.match(remainderPrompt, /Composition/);
const secondPrompt = buildGenerationTurnInput({
const paramsPrompt = buildGenerationTurnInput({
event: { id: 'abc', count: 3 },
phase: 'second',
phase: 'params',
prepared,
artifact,
variantPlan: variantPlan(),
});
assert.match(secondPrompt, /Produce only variant 2/);
assert.match(secondPrompt, /Defer tunable parameters/);
assert.match(paramsPrompt, /Return only parameterCss and paramsJson/);
assert.match(paramsPrompt, /Do not return markup or restyle any default appearance/);
assert.match(paramsPrompt, /Do not call tools or inspect the repository/);
assert.match(paramsPrompt, /Parameter schema examples: range/);
});
it('attaches the real skill and annotation image as first-class turn inputs', () => {
@@ -853,3 +904,7 @@ function variantPlan() {
],
};
}
function emptyParamsJson() {
return '{"1":[],"2":[],"3":[]}';
}
tests/live-e2e.test.mjs
+60 -3
View File
@@ -38,6 +38,7 @@ import {
clickAccept,
clickApplyEdits,
clickEditCopy,
clickDiscard,
clickSaveEdit,
clickGo,
clickNext,
@@ -379,6 +380,13 @@ for (const { name, fixture } of fixtures) {
for (const kind of ['range', 'steps', 'toggle']) {
assert.match(paramsSource, new RegExp(`"kind"\\s*:\\s*"${kind}"`), `param kind ${kind} present`);
}
await page.waitForFunction(() => {
const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.()
|| window.__IMPECCABLE_LIVE_UI_ROOT__
|| document;
const tune = root.querySelector('[data-iceq-tune="1"]');
return tune && tune.disabled === false && /Tune/.test(tune.textContent || '');
}, { timeout: 5_000 });
}
// 6. Cycle variants. Most fixtures stop at variant 2; Svelte Insert
@@ -688,7 +696,10 @@ for (const { name, fixture } of fixtures) {
assert.notEqual(partial.acceptPointerEvents, 'none', 'Accept is available for the first reviewable variant');
assert.notEqual(partial.discardPointerEvents, 'none', 'Discard can cancel unfinished generation');
assert.equal(partial.hasParams, false, 'variant 1 has no eager parameter manifest');
assert.equal(partial.paramsPanelVisible, false, 'Tune UI stays hidden until parameter delivery');
assert.equal(partial.tuneVisible, true, 'Tune stays visible while parameter generation is outstanding');
assert.equal(partial.tuneDisabled, true, 'pending Tune is non-interactive until controls arrive');
assert.match(partial.tuneTitle || '', /still being prepared/, 'pending Tune explains its loading state');
assert.equal(partial.paramsPanelVisible, false, 'the Tune popover stays closed until parameter delivery');
sourceFile = await locateSessionFile(tmp);
const isComponentPreview = sourceFile.endsWith('manifest.json');
@@ -711,7 +722,7 @@ for (const { name, fixture } of fixtures) {
await clickAccept(page, { expectedVariant: 1 });
await waitForBarHidden(page);
await page.waitForFunction(
() => document.documentElement.dataset.impeccableLiveState === 'PICKING',
() => window.__IMPECCABLE_LIVE_STATE__ === 'PICKING',
{ timeout: 2_000 },
);
const automationAcceptToPickingMs = Date.now() - acceptClickedAt;
@@ -844,7 +855,7 @@ for (const { name, fixture } of fixtures) {
await clickAccept(page, { expectedVariant: 2 });
await waitForBarHidden(page);
await page.waitForFunction(
() => document.documentElement.dataset.impeccableLiveState === 'PICKING',
() => window.__IMPECCABLE_LIVE_STATE__ === 'PICKING',
{ timeout: 2_000 },
);
const browserAcceptMs = Number(await page.evaluate(() => document.documentElement.dataset.impeccableAcceptToPickingMs));
@@ -870,6 +881,48 @@ for (const { name, fixture } of fixtures) {
await teardownAndResetBrowser(teardown);
}
});
it('promotes pending Tune controls when the params-only revision arrives', liveE2eTestOptions, async (t) => {
const session = await bootFixtureSession({
name,
fixture,
browser,
agent: createFakeAgent(),
wrapTarget: wrapTargetFromPickedElement,
progressive: true,
progressiveDelayMs: 1500,
log: (message) => t.diagnostic(message),
});
const { page, teardown } = session;
try {
await waitForHandshake(page);
await pickElement(page, fixture.runtime.pickSelector || 'h1.hero-title');
await clickGo(page);
const pending = await waitForProgressiveReviewState(page, 3);
assert.equal(pending.tuneVisible, true);
assert.equal(pending.tuneDisabled, true);
await page.waitForFunction(() => {
const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.()
|| window.__IMPECCABLE_LIVE_UI_ROOT__
|| document;
const tune = root.querySelector('[data-iceq-tune="1"]');
const wrapper = document.querySelector('[data-impeccable-variants]');
return tune?.disabled === false
&& !!wrapper?.querySelector('[data-impeccable-params]');
}, { timeout: 10_000 });
const ready = await readProgressiveReviewState(page);
assert.equal(ready.arrived, 3, 'all variants remain mounted after params publication');
assert.equal(ready.tuneVisible, true);
assert.equal(ready.tuneDisabled, false, 'Tune becomes actionable without another variant arrival');
await clickDiscard(page);
await page.waitForFunction(() => window.__IMPECCABLE_LIVE_STATE__ === 'PICKING', { timeout: 2_000 });
} finally {
await teardownAndResetBrowser(teardown);
}
});
}
if (shouldRunScenario('manual') && Array.isArray(fixture.runtime.manualEditScenarios) && fixture.runtime.manualEditScenarios.length > 0) {
@@ -1058,6 +1111,7 @@ async function readProgressiveReviewState(page) {
const accept = buttons.find((button) => /Accept/.test(button.textContent || ''));
const discard = buttons.find((button) => (button.textContent || '').includes('✕'));
const paramsPanel = root.querySelector('#impeccable-live-params-panel');
const tune = root.querySelector('[data-iceq-tune="1"]');
return {
arrived: isSveltePreview ? Number(debugState?.arrivedVariants || 0) : variants.length,
visible: isSveltePreview ? Number(debugState?.visibleVariant || 0) : Number(visibleVariant?.dataset.impeccableVariant || 0),
@@ -1065,6 +1119,9 @@ async function readProgressiveReviewState(page) {
acceptPointerEvents: accept ? getComputedStyle(accept).pointerEvents : null,
discardPointerEvents: discard ? getComputedStyle(discard).pointerEvents : null,
hasParams: variants.some((variant) => variant.hasAttribute('data-impeccable-params')),
tuneVisible: !!tune,
tuneDisabled: tune?.disabled ?? null,
tuneTitle: tune?.title || '',
paramsPanelVisible: !!paramsPanel
&& getComputedStyle(paramsPanel).pointerEvents !== 'none'
&& getComputedStyle(paramsPanel).clipPath === 'inset(0px)',
tests/live-e2e/agent.mjs
+25 -2
View File
@@ -1518,7 +1518,16 @@ async function publishSourceVariants({ tmp, wrapInfo, event, output }) {
return published;
}
async function publishVariantProgress({ base, token, event, wrapInfo, arrivedVariants, signal }) {
async function publishVariantProgress({
base,
token,
event,
wrapInfo,
arrivedVariants,
signal,
revision = 1,
publicationKind = 'variants',
}) {
const previewMode = wrapInfo.previewMode || 'source';
await fetch(`${base}/events`, {
method: 'POST',
@@ -1527,7 +1536,8 @@ async function publishVariantProgress({ base, token, event, wrapInfo, arrivedVar
token,
type: 'checkpoint',
id: event.id,
revision: 1,
revision,
revisionDomain: 'publication',
phase: 'cycling',
reason: 'variants_progress',
arrivedVariants,
@@ -1535,6 +1545,7 @@ async function publishVariantProgress({ base, token, event, wrapInfo, arrivedVar
sourceFile: wrapInfo.sourceFile || wrapInfo.file,
previewFile: wrapInfo.file,
previewMode,
publicationKind,
}),
signal,
});
@@ -1897,6 +1908,18 @@ export async function runAgentLoop({
await spliceVariantsIntoWrapper({ tmp, wrapInfo, sessionId: event.id, output });
}
trace('agent.write.end', { id: event.id, file: wrapInfo.file });
if (progressive) {
await publishVariantProgress({
base,
token,
event,
wrapInfo,
arrivedVariants: output.variants.length,
signal,
revision: 2,
publicationKind: 'params',
});
}
if (process.env.IMPECCABLE_E2E_DEBUG) {
const post = await fs.readFile(path.join(tmp, wrapInfo.file), 'utf-8');
log(`--- post-splice (variants written) ---\n${post}`);
tests/live-e2e/ui.mjs
+1 -1
View File
@@ -432,7 +432,7 @@ export async function pickElement(page, selector, opts = {}) {
const bar = query(barSel);
const pick = query(pickSel);
return {
liveState: document.documentElement.dataset.impeccableLiveState || null,
liveState: window.__IMPECCABLE_LIVE_STATE__ || null,
target: target ? { tag: target.tagName, classes: target.className, rect: rect?.toJSON?.() || null } : null,
hit: hit ? { tag: hit.tagName, classes: hit.className, text: (hit.textContent || '').slice(0, 80) } : null,
pickActive: pick?.dataset.active || null,
tests/live-generation-preflight.test.mjs
+11
View File
@@ -34,6 +34,17 @@ test('builds a replace preflight from the picker locator', () => {
]);
});
test('can request an isolated source preview for dedicated generation', () => {
const command = buildGenerationPreflight({
type: 'generate',
id: 'session-isolated',
count: 3,
element: { classes: ['hero'], tagName: 'SECTION' },
}, SCRIPTS_DIR, { isolated: true });
assert.equal(command.mode, 'replace');
assert.equal(command.args.includes('--isolated'), true);
});
test('builds an insert preflight from the anchor locator', () => {
const command = buildGenerationPreflight({
type: 'generate',
tests/live-generation-publisher.test.mjs
+55
View File
@@ -5,6 +5,7 @@ import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { createLiveSessionStore } from '../skill/scripts/live/session-store.mjs';
import { scaffoldSourceArtifactSession } from '../skill/scripts/live/source-artifact.mjs';
import {
prepareGenerationArtifact,
publishGenerationArtifact,
@@ -190,6 +191,60 @@ describe('transactional generation publisher', () => {
});
});
describe('transactional isolated source preview publisher', () => {
let tmp;
const id = 'isolatedpub';
beforeEach(() => {
tmp = mkdtempSync(join(tmpdir(), 'impeccable-isolated-publisher-'));
writeFileSync(join(tmp, 'page.html'), '<main><section class="hero">Original</section></main>');
createLiveSessionStore({ cwd: tmp, sessionId: id }).appendEvent({
type: 'generate', id, generationEpoch: 1, count: 3,
});
});
afterEach(() => rmSync(tmp, { recursive: true, force: true }));
it('publishes to the preview artifact while fencing the byte-identical source', () => {
const original = readFileSync(join(tmp, 'page.html'), 'utf-8');
const session = scaffoldSourceArtifactSession({
id,
count: 3,
sourceFile: 'page.html',
sourceStartLine: 1,
sourceEndLine: 1,
originalSource: '<section class="hero">Original</section>',
previewContent: '<main><div data-impeccable-variants="isolatedpub"><div data-impeccable-variant="original"><section class="hero">Original</section></div></div></main>',
cwd: tmp,
});
const prepared = prepareGenerationArtifact({ id, sourceFile: session.previewFile, cwd: tmp });
assert.equal(prepared.ok, true);
assert.equal(prepared.sourceFile, 'page.html');
assert.equal(prepared.previewFile, session.previewFile);
assert.equal(prepared.previewMode, 'source-artifact');
const candidate = readFileSync(join(tmp, prepared.artifactFile), 'utf-8')
.replace('</div></main>', '<div data-impeccable-variant="1"><section>Variant one</section></div></div></main>');
writeFileSync(join(tmp, prepared.artifactFile), candidate);
const published = publishGenerationArtifact({
id,
epoch: prepared.epoch,
sourceFile: session.previewFile,
artifactFile: prepared.artifactFile,
expectedSourceHash: prepared.expectedSourceHash,
arrivedVariants: 1,
expectedVariants: 3,
cwd: tmp,
});
assert.equal(published.ok, true, JSON.stringify(published));
assert.equal(published.sourceFile, 'page.html');
assert.equal(published.previewMode, 'source-artifact');
assert.equal(readFileSync(join(tmp, 'page.html'), 'utf-8'), original);
assert.match(readFileSync(join(tmp, session.previewFile), 'utf-8'), /Variant one/);
});
});
describe('transactional Svelte component publisher', () => {
let tmp;
let source;
tests/live-poll-stream.test.mjs
+83
View File
@@ -148,6 +148,12 @@ describe('live-poll --stream integration', () => {
assert.equal(secondEvent.type, 'steer');
assert.equal(secondEvent.id, '22222222');
assert.equal(secondEvent.message, 'stream test two');
await postReply(`http://localhost:${server.port}`, server.token, {
id: '22222222',
type: 'steer_done',
message: 'done two',
});
} finally {
streamProc.kill('SIGTERM');
}
@@ -200,4 +206,81 @@ describe('live-poll --stream integration', () => {
streamProc.kill('SIGTERM');
}
});
it('waits for carbonize cleanup, then resumes on the same stream process', async () => {
const streamProc = spawn('node', [
POLL_SCRIPT,
'--stream',
'--types=steer,manual_edit_apply,carbonize_cleanup,exit',
'--ack-timeout=15000',
], {
cwd: server.cwd,
stdio: ['ignore', 'pipe', 'pipe'],
env: { ...process.env },
});
try {
const streamPid = streamProc.pid;
const carbonizeLinePromise = readStdoutLine(streamProc);
await new Promise((resolve) => setTimeout(resolve, 150));
const carbonizeResponse = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'carbonize_cleanup',
id: 'c0ffee01',
sessionId: 'abc12345',
file: 'src/App.jsx',
variantId: '2',
acceptResult: { carbonize: true },
}),
});
assert.equal(carbonizeResponse.status, 200);
const carbonizeEvent = JSON.parse(await carbonizeLinePromise);
assert.equal(carbonizeEvent.type, 'carbonize_cleanup');
assert.equal(carbonizeEvent.id, 'c0ffee01');
assert.equal(streamProc.exitCode, null);
process.kill(streamPid, 0);
// Cleanup is performed by the main task through separate tool calls while
// this yielded process waits for its acknowledgement.
await postReply(`http://localhost:${server.port}`, server.token, {
id: 'c0ffee01',
type: 'complete',
file: 'src/App.jsx',
});
const steerLinePromise = readStdoutLine(streamProc);
const steerResponse = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'steer',
id: '33333333',
message: 'still listening after carbonize',
pageUrl: 'http://localhost:4321/',
}),
});
assert.equal(steerResponse.status, 200);
const steerEvent = JSON.parse(await steerLinePromise);
assert.equal(steerEvent.type, 'steer');
assert.equal(steerEvent.id, '33333333');
assert.equal(streamProc.pid, streamPid);
assert.equal(streamProc.exitCode, null);
await postReply(`http://localhost:${server.port}`, server.token, {
id: '33333333',
type: 'steer_done',
message: 'No-op: lifecycle test only.',
});
} finally {
streamProc.kill('SIGTERM');
}
});
});
tests/live-reference.test.mjs
+5 -4
View File
@@ -11,7 +11,7 @@ describe('live reference authoring contract', () => {
const skillSrc = readFileSync(join(ROOT, 'skill/SKILL.src.md'), 'utf-8');
const liveMd = readFileSync(join(ROOT, 'skill/reference/live.md'), 'utf-8');
assert.match(skillSrc, /If the user invoked a sub-command \(`audit`, `polish`, `live`, \.\.\.\), read \*\*`reference\/<command>\.md`\*\*/);
assert.match(skillSrc, /For any other invoked sub-command \(`audit`, `polish`, `live`, \.\.\.\), immediately read \*\*`reference\/<command>\.md`\*\*/);
assert.doesNotMatch(skillSrc, /Use this same scripts directory for all Impeccable helper commands/);
assert.doesNotMatch(skillSrc, /walk upward for the nearest project `\.agents`, `\.claude`, or `\.cursor` skill/);
assert.doesNotMatch(skillSrc, /## Context diagnostics/);
@@ -22,7 +22,7 @@ describe('live reference authoring contract', () => {
const skillSrc = readFileSync(join(ROOT, 'skill/SKILL.src.md'), 'utf-8');
const liveMd = readFileSync(join(ROOT, 'skill/reference/live.md'), 'utf-8');
assert.match(skillSrc, /If the user invoked a sub-command \(`audit`, `polish`, `live`, \.\.\.\), read \*\*`reference\/<command>\.md`\*\*/);
assert.match(skillSrc, /For any other invoked sub-command \(`audit`, `polish`, `live`, \.\.\.\), immediately read \*\*`reference\/<command>\.md`\*\*/);
assert.doesNotMatch(skillSrc, /TARGET_SELECTION_REQUIRED/);
assert.doesNotMatch(skillSrc, /productStatus/);
assert.doesNotMatch(skillSrc, /designStatus/);
@@ -41,14 +41,15 @@ describe('live reference authoring contract', () => {
const openingContract = liveMd.split('\n').slice(0, 60).join('\n');
assert.match(liveMd, /1\. `live\.mjs`: boot\./);
assert.match(liveMd, /3\. Poll loop with the default long timeout \(600000 ms\)\. After every event or `--reply`, run `live-poll\.mjs` again immediately\. Never pass a short `--timeout=`\./);
assert.match(liveMd, /3\. Poll loop with the default long timeout \(600000 ms\)\. Portable harnesses run `live-poll\.mjs` again immediately.*Codex with the dedicated worker keeps the returned `--stream` control command alive instead\./);
assert.match(openingContract, /## Poll loop/);
assert.match(openingContract, /No step skipped, no step reordered\./);
assert.doesNotMatch(liveMd, /live-copy-edits\.md/);
assert.doesNotMatch(liveMd, /IMPECCABLE_LIVE_COPY_AGENT|mock/);
assert.match(liveMd, /"manual_edit_apply" → Handle Manual Edit Apply/);
assert.match(liveMd, /## Handle `manual_edit_apply`/);
assert.match(liveMd, /live-poll\.mjs --types=steer,manual_edit_apply,carbonize_cleanup,exit/);
assert.match(liveMd, /live-poll\.mjs --stream --types=steer,manual_edit_apply,carbonize_cleanup,exit/);
assert.match(liveMd, /narrow, reasoned per-candidate waivers/);
assert.match(liveMd, /Accept emits a foreground `carbonize_cleanup` control event/);
assert.ok(
liveMd.indexOf('## Handle `manual_edit_apply`') > liveMd.indexOf('## Handle `prefetch`'),
tests/live-server.test.mjs
+36
View File
@@ -281,6 +281,40 @@ describe('live-server integration', () => {
assert.equal(data.agentPolling, false);
});
it('/status stops reporting agentPolling as soon as a poll returns an event', async () => {
await drainPolls(server);
const pollPromise = fetch(
`http://localhost:${server.port}/poll?token=${server.token}&timeout=5000&leaseMs=30000`,
).then((response) => response.json());
await new Promise((resolve) => setTimeout(resolve, 50));
const eventRes = await fetch(`http://localhost:${server.port}/events`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
token: server.token,
type: 'generate',
id: 'aabbcc77',
action: 'impeccable',
count: 1,
pageUrl: '/',
element: { outerHTML: '<button>Truthful poll</button>', tagName: 'BUTTON' },
}),
});
assert.equal(eventRes.status, 200);
const event = await pollPromise;
assert.equal(event.id, 'aabbcc77');
const status = await fetch(`http://localhost:${server.port}/status?token=${server.token}`).then((response) => response.json());
assert.equal(status.agentPolling, false);
await fetch(`http://localhost:${server.port}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token: server.token, id: event.id, type: 'done', sourceEventType: 'generate' }),
});
});
it('/live.js serves script with token injected', async () => {
const res = await fetch(`http://localhost:${server.port}/live.js`);
assert.equal(res.status, 200);
@@ -2494,6 +2528,7 @@ colors: {}
previewMode: 'source',
previewFile: 'app/pages/index.vue',
sourceFile: 'app/pages/index.vue',
publicationKind: 'params',
}),
});
assert.equal(res.status, 200);
@@ -2504,6 +2539,7 @@ colors: {}
assert.match(message, /"arrivedVariants":1/);
assert.match(message, /"previewMode":"source"/);
assert.match(message, /"previewFile":"app\/pages\/index.vue"/);
assert.match(message, /"publicationKind":"params"/);
controller.abort();
});
tests/live-session-store.test.mjs
+39
View File
@@ -80,6 +80,21 @@ describe('live-session-store', () => {
assert.deepEqual(restarted.getSnapshot('planned-session').variantPlan, plan);
});
it('tracks parameter publication separately from variant arrival', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'parameter-phase' });
store.appendEvent({ type: 'generate', id: 'parameter-phase', count: 3, generationEpoch: 1 });
store.appendEvent({
type: 'variant_published', id: 'parameter-phase', revision: 1,
generationEpoch: 1, arrivedVariants: 3, publicationKind: 'variants',
});
assert.equal(store.getSnapshot('parameter-phase').paramsPublished, false);
store.appendEvent({
type: 'variant_published', id: 'parameter-phase', revision: 2,
generationEpoch: 1, arrivedVariants: 3, publicationKind: 'params',
});
assert.equal(store.getSnapshot('parameter-phase').paramsPublished, true);
});
it('tombstones generation on early accept and ignores late generation writes', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'early-accept' });
store.appendEvent({
@@ -224,6 +239,30 @@ describe('live-session-store', () => {
);
});
it('tracks publication and browser checkpoint revisions independently', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'split-revisions' });
store.appendEvent({
type: 'generate', id: 'split-revisions', count: 3,
element: { outerHTML: '<section>Hero</section>', tagName: 'section' },
});
store.appendEvent({
type: 'checkpoint', id: 'split-revisions', revision: 8, revisionDomain: 'browser',
owner: 'browser-a', phase: 'cycling', visibleVariant: 2,
});
store.appendEvent({
type: 'checkpoint', id: 'split-revisions', revision: 3, revisionDomain: 'publication',
reason: 'variants_progress', phase: 'cycling', arrivedVariants: 3,
});
const snapshot = store.getSnapshot('split-revisions');
assert.equal(snapshot.browserCheckpointRevision, 8);
assert.equal(snapshot.checkpointRevision, 8);
assert.equal(snapshot.publicationCheckpointRevision, 3);
assert.equal(snapshot.visibleVariant, 2);
assert.equal(snapshot.arrivedVariants, 3);
assert.equal(snapshot.diagnostics.some((entry) => entry.error === 'stale_checkpoint_ignored'), false);
});
it('keeps carbonize-required accepted sessions active until explicit completion', () => {
const store = createLiveSessionStore({ cwd: tmp, sessionId: 'carbonize-session' });
store.appendEvent({
tests/live-wrap.test.mjs
+22 -2
View File
@@ -6,9 +6,9 @@
import { describe, it, beforeEach, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, writeFileSync, readFileSync, rmSync, mkdirSync } from 'node:fs';
import { join } from 'node:path';
import { join, resolve } from 'node:path';
import { tmpdir } from 'node:os';
import { execSync } from 'node:child_process';
import { execFileSync, execSync } from 'node:child_process';
import {
buildSearchQueries,
@@ -253,6 +253,26 @@ describe('wrapCli integration', () => {
assert.ok(!modified.includes('data-impeccable-variant="original" style="display: none"'));
});
it('creates an isolated source preview without mutating the project file', () => {
const html = '<main>\n <section class="hero"><h1>Original</h1></section>\n</main>\n';
writeFileSync(join(tmp, 'index.html'), html);
const output = execFileSync(process.execPath, [
resolve('skill/scripts/live-wrap.mjs'),
'--id', 'isolated123', '--count', '3', '--classes', 'hero',
'--file', 'index.html', '--isolated',
], { cwd: tmp, encoding: 'utf-8' });
const result = JSON.parse(output);
assert.equal(readFileSync(join(tmp, 'index.html'), 'utf-8'), html);
assert.equal(result.sourceFile, 'index.html');
assert.equal(result.previewMode, 'source-artifact');
assert.match(result.file, /^\.impeccable\/live\/previews\/isolated123\/preview\.html$/);
assert.match(readFileSync(join(tmp, result.file), 'utf-8'), /data-impeccable-variants="isolated123"/);
const manifest = JSON.parse(readFileSync(join(tmp, result.previewManifest), 'utf-8'));
assert.equal(manifest.originalSource, ' <section class="hero"><h1>Original</h1></section>');
assert.equal(manifest.sourceFile, 'index.html');
});
it('wraps a JSX element and uses JSX comment syntax', () => {
const jsx = `export default function App() {
return (
tests/skill-behavior/README.md
+10 -11
View File
@@ -1,9 +1,8 @@
# Skill-behavior tests
LLM-backed scenarios that verify how the impeccable skill drives
PRODUCT.md / DESIGN.md loading. Each scenario runs against the cheapest
tier of each major provider (Anthropic, OpenAI, Google) so a full sweep
costs a few cents and finishes in ~2 minutes.
LLM-backed scenarios that verify how the impeccable skill drives context,
command-reference, new-work, and native-platform loading. Each scenario runs
against one current model from each major provider (Anthropic, OpenAI, Google).
These are the tests you re-run when you refactor anything in SKILL.md's
`## Setup` section. They fail when the agent stops following the loading
@@ -41,18 +40,18 @@ The trace is the source of truth, not the model's free-form reply.
| # | Setup | Assertion |
|---|---|---|
| 1 | empty workspace | runs `context.mjs` (which prints a `NO_PRODUCT_MD` directive); agent then loads `reference/init.md` via Read or `cat`; does **not** start writing HTML/CSS |
| 2 | PRODUCT.md only (with `## Register: brand`) | runs `context.mjs` 1-3 times; loads `reference/brand.md` |
| 3 | PRODUCT.md + DESIGN.md (brand register) | runs `context.mjs` 1-3 times; loads `reference/brand.md`; consults the design system (DESIGN.md bundled in output, but CSS / tokens / directory listing also count) |
| 4 | PRODUCT.md + DESIGN.md, context already loaded in turn 1 | turn 2 does **not** re-run `context.mjs`; `reference/brand.md` is loaded across turns 1+2 |
| 5 | PRODUCT.md WITHOUT a `## Register` field; task cue says "landing page" | runs `context.mjs` (which emits a generic register directive); agent loads `reference/brand.md` via task-cue cascade |
| 1 | empty workspace | runs `context.mjs`; loads `reference/init.md` when it treats the run as attended or `reference/new-work.md` when it recognizes the one-shot exception; resolves that gate before implementation |
| 2 | PRODUCT.md only | runs `context.mjs` 1-3 times; loads `reference/new-work.md` because no committed design system exists |
| 3 | PRODUCT.md + DESIGN.md | runs `context.mjs` 1-3 times; receives or explores the committed design system |
| 4 | PRODUCT.md + DESIGN.md, context already loaded in turn 1 | turn 2 does **not** re-run `context.mjs` |
| 5 | PRODUCT.md without the legacy `## Register` field | runs `context.mjs`; greenfield craft still loads `reference/new-work.md` |
| 6 | PRODUCT.md + DESIGN.md + a minimal `index.html`; prompt is `/impeccable polish` | loads `reference/polish.md` |
| 7 | same fixture; prompt is `/impeccable audit` | loads `reference/audit.md` |
| 8 | PRODUCT.md + DESIGN.md + a SvelteKit scaffold (`src/app.css`, components, `+page.svelte`); prompt is `/impeccable polish src/routes/+page.svelte` | reads at least one project code file (CSS / component / page) — not just the skill's reference files |
| 9 | PRODUCT.md + `index.html` + a seeded update cache with a newer version (`skillVersion` copy-mode so `context.mjs` has a `SKILL.md` to version-check against); prompt is `/impeccable polish index.html` | `context.mjs` runs and its output carries the `UPDATE_AVAILABLE` directive (proven via captured bash output); the agent does **not** auto-run `npx impeccable update` (it must ask first) |
| 10 | no PRODUCT.md + a minimal `index.html`; prompt is `/impeccable polish index.html` | runs `context.mjs`, loads `reference/polish.md`, and does **not** divert into `reference/init.md` |
| 11 | empty workspace; prompt is `/impeccable shape ...` | runs `context.mjs`, diverts into `reference/init.md`, and does **not** start writing HTML/CSS |
| 12 | empty workspace; prompt is natural-language build intent with no command word | runs `context.mjs`, diverts into `reference/init.md`, and does **not** start writing HTML/CSS |
| 11 | empty workspace; prompt is `/impeccable shape ...` | runs `context.mjs`; resolves `reference/init.md` (attended) or `reference/new-work.md` (unattended) before implementation |
| 12 | empty workspace; prompt is natural-language build intent with no command word | runs `context.mjs`; resolves `reference/init.md` (attended) or `reference/new-work.md` (unattended) before implementation |
| 13 | empty workspace; prompt is `/impeccable teach` | runs `context.mjs` and diverts into `reference/init.md` because `teach` aliases `init` |
| 14 | PRODUCT.md with `## Register: product` + `## Platform: ios` (native iOS app); prompt is `/impeccable craft a tide detail screen` | `context.mjs` runs and emits a NEXT STEP pointing at `reference/ios.md` (proven via captured bash output); agent loads `reference/ios.md` (Setup step 5, native conventions on top of the register reference) |
| 15 | same iOS fixture; prompt is `/impeccable audit` | agent loads `reference/audit.native.md` (the Commands-table native variant, routed instead of `audit.md`) |
tests/skill-behavior/harness.mjs
+3 -3
View File
@@ -4,7 +4,7 @@
* Each scenario:
* 1. Creates a temp workspace.
* 2. Symlinks the real .claude/skills/impeccable into the workspace so
* scripts (load-context.mjs, etc.) resolve from the canonical path
* scripts (context.mjs, etc.) resolve from the canonical path
* the skill references.
* 3. Optionally writes PRODUCT.md / DESIGN.md fixtures.
* 4. Inlines SKILL.md as the system prompt (placeholders stripped to
@@ -178,7 +178,7 @@ export function makeTools(workspace, extraEnv = {}) {
const tools = {
bash: tool({
description:
'Run a bash command in the workspace root. Use this to invoke skill scripts (e.g. `node .claude/skills/impeccable/scripts/load-context.mjs`).',
'Run a bash command in the workspace root. Use this to invoke skill scripts (e.g. `node .claude/skills/impeccable/scripts/context.mjs`).',
inputSchema: z.object({
command: z.string().describe('The bash command to execute.'),
}),
@@ -267,7 +267,7 @@ export async function runTurn({ workspace, model, userPrompt, priorMessages = []
stopWhen: [stepCountIs(maxSteps)],
});
} catch (err) {
return { trace, error: String(err), text: '', responseMessages: messages, finishReason: 'error' };
throw new Error(`LLM behavior turn failed before completing: ${String(err)}`, { cause: err });
}
const generatedResponseMessages = result.responseMessages ?? result.response?.messages ?? [];
const responseMessages = [...messages, ...generatedResponseMessages];
tests/skill-behavior/scenarios.test.mjs
+74 -97
View File
@@ -51,6 +51,31 @@ function logTrace(label, scenario, model, trace, extras = {}) {
);
}
function loadedBeforeImplementationWrite(trace, filename) {
const needle = filename.toLowerCase();
const loadIndex = trace.toolCalls.findIndex(({ name, input }) => {
if (name === 'read') return input?.path?.toLowerCase().includes(needle);
if (name === 'bash') return input?.command?.toLowerCase().includes(needle);
return false;
});
const writeIndex = trace.toolCalls.findIndex(
({ name, input }) => name === 'write' && /\.(html?|css|svelte|jsx?|tsx?)$/i.test(input?.path ?? ''),
);
return loadIndex >= 0 && (writeIndex < 0 || loadIndex < writeIndex);
}
function executedUpdateCommands(trace) {
const executableSegments = trace.bashCommands.flatMap((command) =>
command
.split(/\r?\n|&&|\|\||;|\|/)
.map((segment) => segment.trim())
.filter((segment) => segment && !/^(?:#|echo\b|printf\b)/.test(segment)),
);
return executableSegments.filter((segment) =>
/^(?:(?:npx|bunx|pnpx)\s+)?(?:impeccable|skills)\s+update\b/.test(segment),
);
}
for (const modelId of resolveModelList()) {
const provider = detectProvider(modelId);
const keyPresent = hasKey(provider);
@@ -61,6 +86,12 @@ for (const modelId of resolveModelList()) {
return;
}
const model = getModel(modelId);
// Gemini Flash tends to inspect one file at a time, while the production
// Anthropic/OpenAI models batch setup reads and then begin implementation.
// Keep the latter tightly bounded so this routing suite does not turn into
// a page-generation benchmark, but leave Gemini enough room to reach the
// same required reference.
const setupMaxSteps = provider === 'google' ? 6 : 3;
it('scenario 1: no PRODUCT.md / DESIGN.md', async () => {
const workspace = prepareWorkspace({ files: {} });
@@ -69,33 +100,29 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: CRAFT_PROMPT,
maxSteps: 6,
maxSteps: setupMaxSteps,
});
logTrace('S1', 'no-context', modelId, trace, { textSample: text.slice(0, 400) });
// Agent runs context.mjs, sees NO_PRODUCT_MD directive, loads
// init.md and follows it. Accept either Read or bash `cat` for
// the init.md load — different models pick different tools.
const loadCalls = bashCommandsMatching(trace, 'context.mjs');
assert.ok(
loadCalls.length >= 1,
`expected agent to run context.mjs at least once; got ${loadCalls.length}.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
const initLoaded =
readsMatching(trace, 'init.md').length > 0 ||
bashCommandsMatching(trace, 'init.md').length > 0;
const resolvedBuildGate =
fileLoaded(trace, 'init.md') || fileLoaded(trace, 'new-work.md');
assert.ok(
initLoaded,
`expected agent to load init.md (via Read or bash cat) after context.mjs reported NO_PRODUCT_MD.\n` +
resolvedBuildGate,
`craft should load init.md for an attended run or new-work.md when it treats the harness as unattended.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
// We do NOT want it to silently barrel into design work.
const wroteHtml = trace.writePaths.some((p) => /\.(html?|css|svelte|jsx?|tsx?)$/i.test(p));
assert.equal(
wroteHtml,
false,
`agent should not write implementation files before resolving missing PRODUCT.md.\n` +
`wrote: ${trace.writePaths.join(', ')}`,
const gatePrecededImplementation =
loadedBeforeImplementationWrite(trace, 'init.md') ||
loadedBeforeImplementationWrite(trace, 'new-work.md');
assert.ok(
gatePrecededImplementation,
`agent should resolve the init/new-work gate before writing implementation files.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
} finally {
cleanupWorkspace(workspace);
@@ -111,7 +138,7 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: CRAFT_PROMPT,
maxSteps: 6,
maxSteps: setupMaxSteps,
});
logTrace('S2', 'product-only', modelId, trace, { textSample: text.slice(0, 400) });
const loadCalls = bashCommandsMatching(trace, 'context.mjs');
@@ -120,11 +147,9 @@ for (const modelId of resolveModelList()) {
`expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` +
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
);
// Fixture sets `register: brand`. Step 3 of Setup says load the
// matching register reference. Accept Read or bash cat.
assert.ok(
fileLoaded(trace, 'brand.md'),
`agent should load brand.md (PRODUCT.md register is brand).\n` +
fileLoaded(trace, 'new-work.md'),
`greenfield craft should load new-work.md when PRODUCT.md exists without a committed design system.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
} finally {
@@ -141,7 +166,7 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: CRAFT_PROMPT,
maxSteps: 6,
maxSteps: setupMaxSteps,
});
logTrace('S3', 'product-and-design', modelId, trace, { textSample: text.slice(0, 400) });
const loadCalls = bashCommandsMatching(trace, 'context.mjs');
@@ -150,19 +175,13 @@ for (const modelId of resolveModelList()) {
`expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` +
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
);
// Register reference: PRODUCT.md fixture is brand, so brand.md
// should be loaded per Setup step 3.
assert.ok(
fileLoaded(trace, 'brand.md'),
`agent should load brand.md (PRODUCT.md register is brand).\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
// The skill tells the agent to also familiarize with the existing
// design system. DESIGN.md is bundled in context.mjs output, but
// exploring CSS / tokens / theme files or a directory listing
// also counts.
const designSignal =
readsMatching(trace, 'design.md').length > 0 ||
trace.bashOutputs.some((output) => output.includes('# DESIGN.md')) ||
trace.readPaths.some((p) => /\.(css|scss|sass|less|ts|tsx|js|jsx|json|svelte|astro)$/i.test(p)) ||
trace.listPaths.length > 0;
assert.ok(
@@ -186,7 +205,7 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: PRIMER_PROMPT,
maxSteps: 5,
maxSteps: setupMaxSteps,
});
logTrace('S4-T1', 'primer', modelId, turn1.trace, { textSample: turn1.text.slice(0, 200) });
const turn1Loads = bashCommandsMatching(turn1.trace, 'context.mjs');
@@ -202,7 +221,7 @@ for (const modelId of resolveModelList()) {
model,
userPrompt: 'Now, /impeccable craft a landing page based on what you saw.',
priorMessages: turn1.responseMessages,
maxSteps: 5,
maxSteps: setupMaxSteps,
});
logTrace('S4-T2', 'follow-up', modelId, turn2.trace, { textSample: turn2.text.slice(0, 400) });
const turn2Loads = bashCommandsMatching(turn2.trace, 'context.mjs');
@@ -212,28 +231,12 @@ for (const modelId of resolveModelList()) {
`agent re-ran context.mjs on turn 2 despite it being in prior conversation. ` +
`bashCommands: ${JSON.stringify(turn2.trace.bashCommands, null, 2)}`,
);
// Register reference must land somewhere across the two turns —
// craft work without brand.md (for a brand-register project) means
// Setup step 3 was skipped.
const brandLoadedAcrossTurns =
fileLoaded(turn1.trace, 'brand.md') || fileLoaded(turn2.trace, 'brand.md');
assert.ok(
brandLoadedAcrossTurns,
`agent should load brand.md across turn 1 or turn 2 (project is brand register).\n` +
`turn 1 readPaths: ${JSON.stringify(turn1.trace.readPaths)}, bash: ${JSON.stringify(turn1.trace.bashCommands)}\n` +
`turn 2 readPaths: ${JSON.stringify(turn2.trace.readPaths)}, bash: ${JSON.stringify(turn2.trace.bashCommands)}`,
);
} finally {
cleanupWorkspace(workspace);
}
});
it('scenario 5: PRODUCT.md WITHOUT register field (cascade via task cue)', async () => {
// PRODUCT.md has no `## Register` section, so context.mjs cannot
// detect the register and emits a generic "pick by cascade"
// directive. The agent must infer brand from the user's task cue
// ("landing page") per SKILL.md's priority list (1) task cue,
// (2) surface in focus, (3) register field.
it('scenario 5: PRODUCT.md without legacy register metadata still follows new-work', async () => {
const workspace = prepareWorkspace({
files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_NO_REGISTER },
});
@@ -242,7 +245,7 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: CRAFT_PROMPT,
maxSteps: 6,
maxSteps: setupMaxSteps,
});
logTrace('S5', 'no-register-field', modelId, trace, { textSample: text.slice(0, 400) });
const loadCalls = bashCommandsMatching(trace, 'context.mjs');
@@ -251,10 +254,9 @@ for (const modelId of resolveModelList()) {
`expected context.mjs invocation; got ${loadCalls.length}.\n` +
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
);
// Task cue is "landing page" → brand register → brand.md should load.
assert.ok(
fileLoaded(trace, 'brand.md'),
`agent should load brand.md via task-cue cascade (no register field, "landing page" cue).\n` +
fileLoaded(trace, 'new-work.md'),
`greenfield craft should load new-work.md regardless of legacy register metadata.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
} finally {
@@ -275,7 +277,7 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable polish index.html',
maxSteps: 6,
maxSteps: setupMaxSteps,
});
logTrace('S6', 'polish-routing', modelId, trace, { textSample: text.slice(0, 300) });
assert.ok(
@@ -301,7 +303,7 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable audit index.html',
maxSteps: 6,
maxSteps: setupMaxSteps,
});
logTrace('S7', 'audit-routing', modelId, trace, { textSample: text.slice(0, 300) });
assert.ok(
@@ -370,7 +372,7 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable polish index.html',
maxSteps: 6,
maxSteps: setupMaxSteps,
env: { IMPECCABLE_UPDATE_CACHE: path.join(workspace, '.impeccable-update.json') },
});
logTrace('S9', 'update-available', modelId, trace, { textSample: text.slice(0, 400) });
@@ -388,10 +390,7 @@ for (const modelId of resolveModelList()) {
`bashOutputs: ${JSON.stringify(trace.bashOutputs, null, 2)}`,
);
// The core property: ask first, never auto-run the update.
const ranUpdate = [
...bashCommandsMatching(trace, 'impeccable update'),
...bashCommandsMatching(trace, 'skills update'),
];
const ranUpdate = executedUpdateCommands(trace);
assert.equal(
ranUpdate.length,
0,
@@ -417,7 +416,7 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable polish index.html',
maxSteps: 6,
maxSteps: setupMaxSteps,
});
logTrace('S10', 'scoped-no-product', modelId, trace, { textSample: text.slice(0, 400) });
// Boot still runs.
@@ -448,18 +447,14 @@ for (const modelId of resolveModelList()) {
}
});
it('scenario 11: shape with no PRODUCT.md still diverts into init', async () => {
// `shape` is a from-scratch build flow, like `craft` (scenario 1): with
// no captured context it must still divert into init before planning.
// This pins the third member of the init/craft/shape guard, so a future
// edit that drops `shape` from the list is caught here.
it('scenario 11: shape with no PRODUCT.md resolves the build gate', async () => {
const workspace = prepareWorkspace({ files: {} });
try {
const { trace, text } = await runTurn({
workspace,
model,
userPrompt: SHAPE_PROMPT,
maxSteps: 6,
maxSteps: setupMaxSteps,
});
logTrace('S11', 'shape-no-context', modelId, trace, { textSample: text.slice(0, 400) });
assert.ok(
@@ -467,38 +462,27 @@ for (const modelId of resolveModelList()) {
`expected agent to run context.mjs at least once.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
const initLoaded =
readsMatching(trace, 'init.md').length > 0 ||
bashCommandsMatching(trace, 'init.md').length > 0;
const gatePrecededImplementation =
loadedBeforeImplementationWrite(trace, 'init.md') ||
loadedBeforeImplementationWrite(trace, 'new-work.md');
assert.ok(
initLoaded,
`from-scratch /impeccable shape should divert into init.md when PRODUCT.md is missing.\n` +
gatePrecededImplementation,
`shape should resolve init.md (attended) or new-work.md (unattended) before implementation.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
// Like craft, it must not barrel into writing implementation files first.
const wroteHtml = trace.writePaths.some((p) => /\.(html?|css|svelte|jsx?|tsx?)$/i.test(p));
assert.equal(
wroteHtml,
false,
`agent should not write implementation files before resolving missing PRODUCT.md.\n` +
`wrote: ${trace.writePaths.join(', ')}`,
);
} finally {
cleanupWorkspace(workspace);
}
});
it('scenario 12: intent-routed build with no PRODUCT.md still diverts into init', async () => {
// Setup runs before the routing table maps natural language like "build a
// landing page" to `craft`, so the NO_PRODUCT_MD guard itself must catch
// clearly-from-scratch build intent.
it('scenario 12: intent-routed build with no PRODUCT.md resolves the build gate', async () => {
const workspace = prepareWorkspace({ files: {} });
try {
const { trace, text } = await runTurn({
workspace,
model,
userPrompt: NATURAL_BUILD_PROMPT,
maxSteps: 6,
maxSteps: setupMaxSteps,
});
logTrace('S12', 'natural-build-no-context', modelId, trace, { textSample: text.slice(0, 400) });
assert.ok(
@@ -506,21 +490,14 @@ for (const modelId of resolveModelList()) {
`expected agent to run context.mjs at least once.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
const initLoaded =
readsMatching(trace, 'init.md').length > 0 ||
bashCommandsMatching(trace, 'init.md').length > 0;
const gatePrecededImplementation =
loadedBeforeImplementationWrite(trace, 'init.md') ||
loadedBeforeImplementationWrite(trace, 'new-work.md');
assert.ok(
initLoaded,
`natural-language build intent should divert into init.md when PRODUCT.md is missing.\n` +
gatePrecededImplementation,
`build intent should resolve init.md (attended) or new-work.md (unattended) before implementation.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
const wroteHtml = trace.writePaths.some((p) => /\.(html?|css|svelte|jsx?|tsx?)$/i.test(p));
assert.equal(
wroteHtml,
false,
`agent should not write implementation files before resolving missing PRODUCT.md.\n` +
`wrote: ${trace.writePaths.join(', ')}`,
);
} finally {
cleanupWorkspace(workspace);
}
@@ -570,7 +547,7 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable craft a tide detail screen for the project in this workspace',
maxSteps: 6,
maxSteps: provider === 'google' ? 8 : 6,
});
logTrace('S14', 'native-ios', modelId, trace, { textSample: text.slice(0, 400) });
const loadCalls = bashCommandsMatching(trace, 'context.mjs');