diff --git a/tests/live-e2e-accept-cleanup-regression.test.mjs b/tests/live-e2e-accept-cleanup-regression.test.mjs index 01b5e48a1..343b2723f 100644 --- a/tests/live-e2e-accept-cleanup-regression.test.mjs +++ b/tests/live-e2e-accept-cleanup-regression.test.mjs @@ -70,7 +70,7 @@ describe('live-e2e accept cleanup regression', () => { log: (msg) => t.diagnostic(msg), }); - const { page, tmp, teardown } = session; + const { page, tmp, appRoot, teardown } = session; try { t.diagnostic(`Using LLM agent (provider=${llmConfig.provider} model=${llmConfig.model})`); await waitForHandshake(page); @@ -104,6 +104,10 @@ describe('live-e2e accept cleanup regression', () => { await clickNext(page); assert.equal(await getVisibleVariant(page), 2, 'variant 2 visible after one Next'); + const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('impeccable-live-session') || 'null')); + assert.ok(saved?.id && /^[a-zA-Z0-9_-]+$/.test(saved.id), 'cycling session has a safe durable id'); + const snapshotPath = join(appRoot, '.impeccable/live/sessions', `${saved.id}.snapshot.json`); + t.diagnostic('Accepting variant 2'); await clickAccept(page, { expectedVariant: 2 }); @@ -120,6 +124,19 @@ describe('live-e2e accept cleanup regression', () => { sourceFile, finalSource, }); + + // Source/DOM cleanup can precede live-complete, or succeed while its + // durable acknowledgement fails. Do not count that as a finished accept. + const deadline = Date.now() + 30_000; + let snapshot; + do { + snapshot = JSON.parse(readFileSync(snapshotPath, 'utf8')); + if (snapshot.phase === 'completed') break; + await new Promise((resolve) => setTimeout(resolve, 100)); + } while (Date.now() < deadline); + assert.equal(snapshot.phase, 'completed', 'accept must reach durable completed phase without forcing completion'); + assert.doesNotMatch(readFileSync(sourceFile, 'utf8'), /data-impeccable-[\w-]+\s*=/, 'accepted source contains no reserved runtime attributes'); + t.diagnostic(`Durable completion verified for ${saved.id}`); } finally { await teardown(); } diff --git a/tests/live-e2e-llm-agent.test.mjs b/tests/live-e2e-llm-agent.test.mjs index 9f06a9c38..1ca6b7747 100644 --- a/tests/live-e2e-llm-agent.test.mjs +++ b/tests/live-e2e-llm-agent.test.mjs @@ -1520,6 +1520,12 @@ describe('live-e2e LLM agent variant prompt', () => { assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /bare text element/); assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /Accept persists a real source change/); }); + + it('uses permanent styling hooks outside the reserved live-runtime namespace', () => { + assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /data-design-variant/); + assert.doesNotMatch(VARIANT_SYSTEM_INSTRUCTIONS, /add[^\n]*data-impeccable-e2e-variant/); + assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /Never invent data-impeccable-\*/); + }); }); describe('live-e2e LLM agent variant copy validation', () => { diff --git a/tests/live-e2e/agents/llm-agent.mjs b/tests/live-e2e/agents/llm-agent.mjs index 7368c23e9..ad24ee2a2 100644 --- a/tests/live-e2e/agents/llm-agent.mjs +++ b/tests/live-e2e/agents/llm-agent.mjs @@ -83,7 +83,7 @@ export const VARIANT_SYSTEM_INSTRUCTIONS = [ '- Replace mode: for bare text elements, keep the full visible copy in one editable text node. If you add child markup for styling, wrap the entire copy; never split the copy across sibling text nodes.', '- Replace mode: PRESERVE existing class-bearing descendant elements in place. If the picked element contains

and

, keep those elements/classes as direct descendants of the replacement root; do not wrap them in a new structural div such as

.', '- Replace mode: Do not return source-identical variants. For a bare text element, preserve the root tag/class/copy but add a small child span or styling hook so Accept persists a real source change.', - '- Replace mode: for non-bare elements where the existing children must stay in place, add a harmless root attribute such as data-impeccable-e2e-variant="1" or another non-copy styling hook so the markup is materially changed without changing visible text.', + '- Replace mode: for non-bare elements where the existing children must stay in place, add a permanent styling hook such as data-design-variant="1" so the markup is materially changed without changing visible text. Never invent data-impeccable-* attributes: that namespace is reserved for temporary runtime state.', '- Generate exactly event.count variants — no more, no fewer.', '- Mix the param kinds across the variant set: include at least one range, one steps, and one toggle when count >= 3.', '- The scopedCss must follow wrapInfo.cssAuthoring exactly: use its selector strategy, rulePattern, requirements, and forbidden patterns.', diff --git a/tests/skill-behavior/README.md b/tests/skill-behavior/README.md index 68853ae03..8db385c3f 100644 --- a/tests/skill-behavior/README.md +++ b/tests/skill-behavior/README.md @@ -145,6 +145,24 @@ workspaces' filesystem checks. No paid calls or skill prose changes were needed. The earlier reference misses above are now diagnostics, not release blockers by themselves; this does not establish an all-green full-workflow matrix. +### Bounded release verification (2026-09-08) + +The focused Anthropic live-accept test now verifies the durable session reaches +`completed`, not just clean source/DOM. Its agent instruction had recommended +`data-impeccable-e2e-variant`, inside the reserved runtime namespace; the test +prompt now uses a permanent `data-design-variant` styling hook. The strengthened +test passed in 14s without forced completion. The full non-billed suite passed. +No runtime or skill source was changed for this correction. + +Claude post-review controls: new world passed in 77s, writing token-bearing +DESIGN.md and the v2 sidecar; approved redesign failed in 16s. The latter read the +page and old system, then wrote prose-only DESIGN.md without consulting the +documentation spec or creating `.impeccable/design.json`, and claimed nothing +remained outstanding. This is a missing required artifact, not merely missing +reference coverage. The test stays red; no retry was purchased. Release remains +held pending disposition. These are synthetic post-review checkpoints, not +proof of a complete redesign lifecycle. + Each scenario: 1. `prepareWorkspace()` uses the production transformer to build current source diff --git a/tests/skill-workflow/finish-handoff.test.mjs b/tests/skill-workflow/finish-handoff.test.mjs index 0e15a96f9..20bbe2777 100644 --- a/tests/skill-workflow/finish-handoff.test.mjs +++ b/tests/skill-workflow/finish-handoff.test.mjs @@ -28,14 +28,17 @@ const PAGE = 'Keyboa const BRIEF = '# Keyboard guide\n\n## Direction contract\nTHESIS: A short reading page.\nOWN-WORLD: Inherit Field Manual.\nSTORY: Read keyboard instructions.\nFIRST VIEWPORT: Title, paragraph, link.\nFORM: Direct, precisely specified page; no seed required.\nFINISH: unreviewed and undocumented is unfinished.\n'; for (const modelId of (process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS || 'claude-sonnet-5').split(',').map((id) => id.trim()).filter(Boolean)) { - for (const existingSystem of [true, false]) { - it(`post-review ${existingSystem ? 'extension preserves' : 'new world records'} its system :: ${modelId}`, + for (const mode of ['extension', 'new world', 'redesign']) { + const existingSystem = mode !== 'new world'; + const preserveSystem = mode === 'extension'; + it(`post-review ${mode} ${preserveSystem ? 'preserves' : 'records'} its system :: ${modelId}`, { skip: !ENGINE_BIN || !hasKey(detectProvider(modelId)) }, async (t) => { const files = { 'PRODUCT.md': '# Field Manual\n\n## Platform\nweb\n\nA reference guide for keyboard users.\n', - ...(existingSystem ? { 'DESIGN.md': DESIGN } : {}), + ...(existingSystem ? { 'DESIGN.md': preserveSystem ? DESIGN : '# Old Field Manual\n\nBeige cards, serif body type, orange links.\n' } : {}), 'index.html': PAGE, - '.impeccable/surfaces/index-html.md': BRIEF, + '.impeccable/surfaces/index-html.md': mode === 'redesign' + ? BRIEF.replace('Inherit Field Manual.', 'Approved replacement: white, blue links, system-ui, single column.') : BRIEF, }; const workspace = prepareWorkspace({ files }); try { @@ -44,14 +47,18 @@ for (const modelId of (process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS || 'claude-s workspace, model: getModel(modelId), maxSteps: 12, timeoutMs: 180000, contextOnlyBash: true, environment: 'This is a resumed post-review checkpoint. No subagent or browser tools are available. The review is closed; no further UI edits or screenshots are needed. Read/list/write tools are available.', priorMessages: [ - { role: 'user', content: existingSystem + { role: 'user', content: preserveSystem ? 'Use /impeccable to add the specified keyboard guide page inside the established Field Manual world. Keep the existing visual system. Do not repair unrelated project drift.' - : 'Use /impeccable to create Field Manual’s first keyboard guide page. The chosen identity is plain, single-column, system fonts, white background and blue links.' }, + : mode === 'redesign' + ? 'Use /impeccable to redesign the keyboard guide. I approve replacing the old beige-card/serif/orange world with the plain single-column, system-font, white-background and blue-link identity. Update the system documentation from the finished page.' + : 'Use /impeccable to create Field Manual’s first keyboard guide page. The chosen identity is plain, single-column, system fonts, white background and blue links.' }, { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'load-new-work', toolName: 'read', input: { path: '.claude/skills/impeccable/reference/new-work.md' } }] }, { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'load-new-work', toolName: 'read', output: { type: 'text', value: reference } }] }, - { role: 'assistant', content: `Checkpoint: context and PRODUCT.md were loaded. The user confirmed the exact page and identity. The surface brief and index.html are written. Desktop/mobile captures were validated, the detector ran once, and the shipped finish reviewer returned ship with no open findings. ${existingSystem + { role: 'assistant', content: `Checkpoint: context and PRODUCT.md were loaded. The user confirmed the exact page and identity. The surface brief and index.html are written. Desktop/mobile captures were validated, the detector ran once, and the shipped finish reviewer returned ship with no open findings. ${preserveSystem ? 'DESIGN.md was loaded. No durable system changes were requested or introduced. The pre-existing missing .impeccable/design.json was reported but not repaired.' - : 'This is the first completed surface of the approved new world. No DESIGN.md or design sidecar exists yet.'}` }, + : mode === 'redesign' + ? 'The approved replacement world is implemented in index.html. DESIGN.md still describes the superseded identity; no design sidecar exists yet.' + : 'This is the first completed surface of the approved new world. No DESIGN.md or design sidecar exists yet.'}` }, ], userPrompt: 'Continue from this checkpoint and finish the task.', }); @@ -62,21 +69,26 @@ for (const modelId of (process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS || 'claude-s assert.ok(fileLoaded(result.trace, name), `documentation must check ${name}, not merely announce a no-op`); } for (const [name, contents] of Object.entries(files)) { + if (mode === 'redesign' && name === 'DESIGN.md') continue; assert.equal(fs.readFileSync(path.join(workspace, name), 'utf8'), contents, `${name} must remain unchanged`); } - if (existingSystem) { + if (preserveSystem) { assertNoChangeDocumentation(result, { target: 'index.html', evidence: [/system-ui/i, /65\s*ch/i, /#0645ad/i] }); assert.equal(fs.existsSync(path.join(workspace, '.impeccable/design.json')), false, 'must not repair pre-existing sidecar drift unasked'); assert.deepEqual(result.trace.toolCalls.flatMap((call) => call.mutatedPaths || []), [], 'a no-change check must not mutate other project files'); } else { assert.ok(fileLoaded(result.trace, 'degraded/documenter.md'), 'new-world documentation must run the shipped documentation pass'); const design = fs.readFileSync(path.join(workspace, 'DESIGN.md'), 'utf8'); + if (mode === 'redesign') assert.notEqual(design, files['DESIGN.md'], 'approved redesign must replace the old system'); assert.match(design, /^---\n/); assert.match(design, /^colors:/m); assert.match(design, /system-ui/); const sidecar = JSON.parse(fs.readFileSync(path.join(workspace, '.impeccable/design.json'), 'utf8')); assert.equal(sidecar.schemaVersion, 2); assert.ok(sidecar.extensions && sidecar.narrative); + const writes = result.trace.toolCalls.flatMap((call) => call.mutatedPaths || []); + assert.ok(writes.includes('DESIGN.md') && writes.includes('.impeccable/design.json'), 'both documentation artifacts must be written'); + assert.deepEqual(writes.filter((file) => !['DESIGN.md', '.impeccable/design.json'].includes(file)), [], 'documentation must stay inside its write boundary'); } } finally { cleanupWorkspace(workspace);