Compare commits

...
Author SHA1 Message Date
Paul Bakaus 733e39abae Fix workflow artifact and target syntax review gaps
Keep cross-family protocol coverage separate from the manually opted-in full workflow diagnostic lane.

AI assistance: Codex, under maintainer direction.
2026-09-07 19:17:24 -07:00
Paul Bakaus 6523fc56ec Keep fixture tool paths contained through symlinks
Validate canonical existing ancestors for new targets and preserve staged-skill write protection through aliases. Regression reproduced the external read escape before the fix; all 20 harness tests and the full non-billed suite pass.

AI assistance: Codex, under maintainer direction.
2026-09-07 19:08:29 -07:00
Paul Bakaus 737a51f5ad Require complete design artifacts at the finish handoff
Name token-bearing DESIGN.md and its sidecar in the parent handoff while shortening the paragraph. Keep spec, token, sidecar and write-boundary gates; report wrapper coverage separately.

Validation: one Claude redesign retest wrote both artifacts; saved-output replay passed artifact checks (live assertion first failed on wrapper coverage). Default tests, source build and skill validator passed. Remaining token-accuracy limitations are documented.

AI assistance: Codex, under maintainer direction.
2026-09-07 19:01:34 -07:00
Paul Bakaus b1a071049e Record bounded release verification outcomes
Full Claude build passed automated gates; redesigned-system documentation still omits required artifacts. Keep the release hold and distinguish reference coverage from completion evidence.

AI assistance: Codex, under maintainer direction.
2026-09-07 18:48:29 -07:00
Paul Bakaus 4b99594866 Test durable acceptance and redesigned system documentation
Keep actual missing documentation artifacts as failures. Avoid reserving live-runtime attributes for permanent test-agent styling hooks, and require a durable completed acknowledgement after acceptance.

Validation: default suite and focused Anthropic accept passed; Claude new-world checkpoint passed, approved-redesign checkpoint failed (missing sidecar and token schema). Release remains held.

AI assistance: Codex, under maintainer direction.
2026-09-07 18:43:18 -07:00
Paul Bakaus 5b5fdf040f Test advice and documentation outcomes rather than reference coverage
Keep completion, consent, documentation evidence, and new-world artifact requirements strict. Record reference-only omissions as diagnostics in the scoped advice and no-change fixtures.

AI assistance: Codex, under maintainer direction.
2026-09-07 18:33:35 -07:00
Paul Bakaus 2ef885efcf Clarify no-change documentation handoffs
Preserve incumbent systems while requiring a checked documentation outcome. Add bounded post-review extension and new-world controls; record the remaining reference-loading miss rather than masking it. AI assistance: Codex, under maintainer direction.
2026-09-07 16:59:23 -07:00
Paul Bakaus 0d57b6e41d Bound remaining protocol checks and record completion gaps
Stop update notices and explicit routing at observed checkpoints. Retain missing-reference failures and the skipped documentation pass as unresolved evidence, not an all-green release claim. AI assistance: Codex.
2026-09-07 16:45:39 -07:00
Paul Bakaus ec2de03b55 Fix workflow trace directory context scope
Resolve runner.temp inside the test step, where GitHub exposes runner context. AI assistance: Codex.
2026-09-07 16:40:39 -07:00
Paul Bakaus e1fe48b05a Separate protocol checkpoints from provisioned workflow tests
Keep full completion and current visual evidence distinct from setup routing. Add opt-in preflighted browser tools and explicit failure outcomes. AI assistance: Codex, under maintainer direction.
2026-09-07 16:38:58 -07:00
Paul Bakaus 75ad869c06 Record remaining bounded workflow test failures
AI assistance: Codex, under maintainer direction.
2026-09-07 16:23:34 -07:00
Paul Bakaus e42c918777 Keep workflow preparation bounded and supply complete test evidence
AI assistance: Codex, under maintainer direction.
2026-09-07 16:13:59 -07:00
Paul Bakaus 61925821b3 Give Claude routing checks a measured setup budget
AI assistance: Codex, under maintainer direction.
2026-09-07 16:06:31 -07:00
Paul Bakaus cf9b35d544 Fix skill workflow confirmations and behavior fixtures
AI assistance: Codex, under maintainer direction.
2026-09-07 16:02:01 -07:00
23 changed files with 1305 additions and 154 deletions
+47
View File
@@ -6,6 +6,11 @@ on:
pull_request:
branches: [main]
workflow_dispatch:
inputs:
skill_workflow:
description: 'Run billed, browser-backed Claude workflow completion tests'
type: boolean
default: false
# Nightly full live-e2e matrix. The smoke groups already gate every PR; the
# full sweep is too slow for that, so it runs once a day against main.
schedule:
@@ -665,5 +670,47 @@ jobs:
- name: Install dependencies
run: bun install
- name: Prepare engine for protocol tests
run: bun run fetch:engine
- name: Run skill behavior tests
run: bun run test:skill-behavior
skill-workflow:
# Expensive, manually opted-in completion diagnostics. This does not
# replace the multi-family skill-behavior protocol suite above.
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && inputs.skill_workflow
timeout-minutes: 70
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
IMPECCABLE_SKILL_BEHAVIOR_MODELS: claude-sonnet-5
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 24
- name: Setup Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: latest
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Prepare engine and browser before billing
run: |
test -n "$ANTHROPIC_API_KEY" || { echo 'ANTHROPIC_API_KEY is required'; exit 1; }
bun run fetch:engine
bunx playwright install --with-deps chromium
- name: Run completed skill workflows
env:
IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR: ${{ runner.temp }}/skill-workflow-traces
run: bun run test:skill-workflow
- name: Retain diagnostic traces
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: skill-workflow-traces
path: ${{ runner.temp }}/skill-workflow-traces
retention-days: 7
+1
View File
@@ -59,6 +59,7 @@
"test:new-work-e2e": "node scripts/run-tests.mjs new-work-e2e",
"test:live-e2e-agent": "node scripts/run-tests.mjs live-e2e-agent",
"test:skill-behavior": "node scripts/run-tests.mjs skill-behavior",
"test:skill-workflow": "node scripts/run-tests.mjs skill-workflow",
"test:live-svelte-adapter-deepseek": "node scripts/run-tests.mjs live-svelte-adapter-deepseek",
"smoke:hooks": "node scripts/smoke-provider-hooks.mjs",
"audit": "bun audit --audit-level=moderate",
+27 -20
View File
@@ -8,6 +8,7 @@ export const OPT_IN_SUITES = [
'live-e2e-accept-cleanup',
'new-work-e2e',
'skill-behavior',
'skill-workflow',
'live-svelte-adapter-deepseek',
];
@@ -267,38 +268,44 @@ export const SUITES = {
],
},
'skill-behavior': {
description: 'LLM-backed skill setup behavior scenarios.',
description: 'LLM-backed protocol checkpoints, not full builds.',
optIn: true,
triggers: [
...COMMON_INFRA_PATTERNS,
/^skill\/SKILL\.src\.md$/,
/^skill\/reference\/(init|document|brand|product|shape|craft|audit|polish|live|routing)\.md$/,
/^skill\/reference\//,
/^ENGINE_VERSION$/,
/^tests\/skill-behavior\//,
],
commands: [{
runner: 'node',
timeoutMs: 240000,
wallClockMs: 1_800_000,
files: ['tests/skill-behavior/scenarios.test.mjs'],
}],
},
'skill-workflow': {
description: 'Explicitly opt-in completed workflows with a preflighted browser.',
optIn: true,
needsPlaywright: true,
triggers: [
...COMMON_INFRA_PATTERNS,
/^skill\//,
/^ENGINE_VERSION$/,
/^tests\/skill-workflow\//,
/^tests\/skill-behavior\//,
],
commands: [
{ runner: 'node', files: ['tests/skill-workflow-browser.test.mjs'] },
{
runner: 'node', timeoutMs: 240000, wallClockMs: 600000,
files: ['tests/skill-workflow/finish-handoff.test.mjs'],
},
{
runner: 'node',
// 300000 was too low to measure what these scenarios assert. The
// workflow-contract turns run 20+ steps against a frontier model, and
// the *correct* path is the slow one: a run that stops to put the
// concept to the user before building was measured at 579s, while the
// runs that skipped that checkpoint and failed the assertion finished
// in 130-200s. At a 300s cap the thorough path is killed and the hasty
// path is graded, so the cap was selecting for the behavior the suite
// exists to forbid.
timeoutMs: 900000,
// Overall wall-clock safety cap for the whole sweep: if a provider
// call wedges past every inner guard (the harness's 840s per-turn
// AbortSignal and the 900s per-test timeout), the runner SIGKILLs the
// process group so the sweep still ends with a per-provider tally
// instead of hanging overnight. Sized well above a healthy two-provider
// sweep; override with IMPECCABLE_TEST_WALL_CLOCK_MS to scope it down.
wallClockMs: 3_600_000,
files: [
'tests/skill-behavior/scenarios.test.mjs',
'tests/skill-behavior/workflow-contract.test.mjs',
],
files: ['tests/skill-workflow/full-build.test.mjs'],
},
],
},
+1 -1
View File
@@ -22,7 +22,7 @@ Core principles:
2. Load the request's playbook: its Commands-table reference for an explicit/implied sub-command, or [reference/new-work.md](reference/new-work.md) for a new surface or replacement visual world. Inspect target and incumbent visual truth before editing. When the app cannot run, start with committed visual-regression goldens or screenshot fixtures; verify target and freshness against current tokens, CSS, components, or assets, resolve conflicts, and compare theme/variant captures. <!-- rule:skill-setup-command-ref --> <!-- rule:skill-setup-read-project -->
3. After resolving analysis and direction, read [reference/craft-floor.md](reference/craft-floor.md) immediately before any UI edit, including small refinements. It carries the quality floor, the absolute bans, and the reflexes no detector catches. Do not load it for planning-only work. <!-- rule:skill-craft-floor-load -->
**Launcher unavailable:** If refused, missing, or failed, **first send the user a message** that context loading did not run. Then read existing PRODUCT.md and DESIGN.md without inventing missing context, follow the applicable steps 23, and perform the requested work through permitted tools. Launcher failure alone does not block otherwise-permitted edits.
**Launcher unavailable:** On refusal or failure, send a separate message **before the next tool call**: “Context loading did not run; Ill read the existing project context directly.” Then read existing PRODUCT.md and DESIGN.md without inventing missing context, follow applicable steps 23, and continue through permitted tools. This applies to planning and editing; launcher failure alone does not block either.
## How to design
+3 -3
View File
@@ -16,7 +16,7 @@ nickname-candidates:
You record a project's design system after the build is done. Ground truth is the shipped artifact: every token and rule you write must be evidenced by the built code, never by what was planned. Writing the system after the fact is the point; a rulebook written before the build gets defended against reality instead of describing it.
You run under a hard turn ceiling that ends the run without warning, and a run that ends before DESIGN.md is written has recorded nothing. Batch several Reads into each turn, take `reference/document.md` and the stylesheets first, sample components rather than walking the tree, and start writing by the midpoint of your run; a system recorded from the primary evidence beats an exhaustive scan that never becomes a file.
Complete the check within your turn ceiling. Batch Reads, take `reference/document.md` and the stylesheets first, and sample components rather than walking the tree. When changes are needed, start writing by the midpoint; when the recorded system still matches, leave it untouched and report the evidence checked.
## Input Contract
@@ -26,10 +26,10 @@ Expect: the project root; the artifact path(s); the direction contract text (THE
1. Read `reference/document.md` in full; it is the operating spec for DESIGN.md's format, token schema, sidecar, and section order. Follow it exactly.
2. Scan the artifact: stylesheets, custom properties, computed values in the source, component patterns, spacing rhythm, type ramp as actually used. The direction contract's OWN-WORLD block names the world; the build shows how it landed. Where they diverge, the build wins and the prose may note the divergence.
3. Write DESIGN.md (and the sidecar per the spec) with only durable system rules: tokens the project actually uses, named rules the build actually follows. Skip one-off values; a token used once is not a system.
3. For a new world or approved system change, write DESIGN.md and its sidecar from durable, reused rules in the build. Ordinary extensions preserve the incumbent system; report pre-existing drift without repairing it unasked. Do not write merely to prove this pass ran.
4. Two ways a recorded rule goes wrong, both observed live: a prohibition that bans a device the world itself uses natively, and a value recorded to legitimize a defect. Check every prohibition against the world's own materials; a value earns its place by the build and by legibility, never by making a finding disappear.
5. Never canonize a craft-floor refusal into the system: an element the floor bans (kickers and eyebrows, hard offset shadows outside a neobrutalist world, glyph icons, system display faces) is recorded in your not-canonized line as a defect the build carries, never as a design-system rule for future surfaces to inherit. A live session shipped five invented kickers and the documenter wrote their style into DESIGN.md; that is how one violation becomes the house style.
## Output Contract
Return: the file paths written, a five-line summary of the recorded system (palette strategy, type ramp shape, named rules), and one line naming anything in the build you deliberately did not canonize and why. No other prose.
Return: paths written, or “No changes” with the source and system files checked; a five-line system summary (palette, type ramp, named rules); and one line naming defects or drift not canonized or repaired, and why. No other prose.
+3 -3
View File
@@ -15,7 +15,7 @@ A section, component, feature, or state inside an established surface inherits t
## 2. Ask what will change the work
Ask one round of two or three related questions through the structured question tool when available. Skip settled facts; a precise request may need only a compact confirmation.
Before implementation, get the user's answer through the structured question tool when available. Ask two or three related questions; a precise request needs only a compact confirmation. Skip settled facts, not the confirmation: DESIGN.md settles the visual world, not this surface's purpose or concept.
- **Persuade:** who must act, what they should believe, which real proof, content, or assets earn that belief.
- **Operate:** the task, information, important states, frequency, constraints.
@@ -72,7 +72,7 @@ Your measured rendition prior: warm, bookish, family, and child-facing subjects
## 5. Record the decision
Before code, record the chosen direction as a development-only contract under `## Direction contract` in the relevant surface brief. A direction contract is durable route or artifact strategy, so create or update the brief even when no other surface strategy needs persistence. Keep the contract to six short blocks and 150 words at most. THESIS: the one idea this surface owns and the category-default arrangement it refuses. OWN-WORLD: the palette and component language, specific enough to be recognizable with all content removed. STORY: what the visitor understands, believes, and does. FIRST VIEWPORT: the exact composition, what is where and at what scale, and where the primary action sits. FORM: the chosen form, its position on your ordered list, and the seed key the script printed. Close with one more line, FINISH: the run's exit condition, verbatim "unreviewed and undocumented is unfinished; this build ends with the finish review, the verdict, DESIGN.md, and every shipping raster carrying its provenance". The surface brief is the reminder later agents reload across edits and sessions: a page that looks complete with the FINISH line undischarged is not done, it is abandoned at the finish line. If a block reads like a mood, the direction is not decided yet; the finishing review audits the render against this contract. <!-- rule:skill-decide-then-build -->
Before code, record the chosen direction as a development-only contract under `## Direction contract` in the relevant surface brief. A direction contract is durable route or artifact strategy, so create or update the brief even when no other surface strategy needs persistence. Use six short blocks, roughly 150 words; do not spend tool calls counting words. THESIS: the one idea this surface owns and the category-default arrangement it refuses. OWN-WORLD: the palette and component language, specific enough to be recognizable with all content removed. STORY: what the visitor understands, believes, and does. FIRST VIEWPORT: the exact composition, what is where and at what scale, and where the primary action sits. FORM: the chosen form, its position on your ordered list, and the seed key the script printed. Close with one more line, FINISH: the run's exit condition, verbatim "unreviewed and undocumented is unfinished; this build ends with the finish review, the verdict, DESIGN.md, and every shipping raster carrying its provenance". The surface brief is the reminder later agents reload across edits and sessions: a page that looks complete with the FINISH line undischarged is not done, it is abandoned at the finish line. If a block reads like a mood, the direction is not decided yet; the finishing review audits the render against this contract. <!-- rule:skill-decide-then-build -->
Never copy the direction contract into implementation source or any browser-delivered artifact. This includes HTML or framework comments, hidden DOM, `<template>` elements, `data-*` attributes, rendered JSX or TSX output, serialized props or state, React Server Component payloads, client bundles, metadata or JSON-LD, accessibility-only text, and files served beside the artifact. A compiler or optimizer removing development metadata is not a safety boundary. Reviewers and documenters receive the contract from the surface brief.
@@ -146,4 +146,4 @@ A rebuild and a fix round share one asset rule: a raster either round creates or
Report the final verdict under the reviewer's own disposition word and at its actual scope. A verdict pass scores the listed fixes and nothing else: "the reviewer scored all three fixes resolved" is a claim it supports, "no material issues remain" is not. A table with open material findings is never announced as a pass, never softened, and never dressed as whole-surface approval when only a fix list was scored. When the user answers a ship with evidence against it, their own screenshot, a named mismatch with the comp, that evidence outranks every capture you made: put their material in the packet and spawn a fresh reviewer for a new full review. Patching inline and self-certifying is how a rejected page ships twice. <!-- rule:skill-user-evidence-reopens-review -->
Then spawn the shipped documenter, `impeccable-documenter` (`impeccable_documenter` in codex), with the project root, the artifact path, the direction contract, PRODUCT.md, the [document.md](document.md) reference path, and the boundary to write at; it records DESIGN.md and the sidecar from the built world, ground truth over intention; without subagents the pass runs from [degraded/documenter.md](degraded/documenter.md). The documenter runs after the last correction lands: when any fix round follows the documentation, re-run the documenter over the changed surface, because a DESIGN.md describing a layout that no longer exists turns defects into system guidance. A clean detector pass is not finished; finished is the contract kept, the comp honored, the review closed, and the system recorded. <!-- rule:skill-documenter-records-the-world -->
After the last correction, spawn the shipped documenter, `impeccable-documenter` (`impeccable_documenter` in codex), with the project root, artifact path, direction contract, PRODUCT.md, [document.md](document.md), and write boundary. Without subagents, load [degraded/documenter.md](degraded/documenter.md) and [document.md](document.md) before writing. Verify the outcome: new worlds and approved system changes require token-bearing DESIGN.md **and** `.impeccable/design.json`, not prose alone. Ordinary extensions compare the finished build against the incumbent system, preserve its files, and report the evidence checked; report pre-existing drift without repairing it unasked. Recheck after later edits. Finish only when review and documentation are complete. <!-- rule:skill-documenter-records-the-world -->
+14
View File
@@ -8,6 +8,20 @@ import { tmpdir } from 'node:os';
const SCRIPT = 'scripts/ci-test-plan.mjs';
describe('ci-test-plan', () => {
it('requires explicit manual opt-in and preprovisions the full workflow job', () => {
const workflow = readFileSync('.github/workflows/ci.yml', 'utf8');
assert.match(workflow, /skill_workflow:\s*description:[^\n]+\s*type: boolean\s*default: false/);
const job = workflow.split('\n skill-workflow:')[1];
assert.doesNotMatch(job.split('\n steps:')[0], /runner\./, 'runner context is unavailable at job-level env');
assert.match(job, /if: github.event_name == 'workflow_dispatch' && inputs.skill_workflow/);
assert.ok(job.indexOf('bun run fetch:engine') < job.indexOf('bun run test:skill-workflow'));
assert.ok(job.indexOf('playwright install --with-deps chromium') < job.indexOf('bun run test:skill-workflow'));
const protocol = workflow.split('\n skill-behavior:')[1].split('\n skill-workflow:')[0];
assert.match(protocol, /bun run fetch:engine/);
assert.doesNotMatch(protocol, /IMPECCABLE_SKILL_BEHAVIOR_MODELS:/, 'protocol coverage must retain the multi-family defaults');
assert.match(protocol, /GOOGLE_CLOUD_API_KEY:/);
assert.match(protocol, /ANTHROPIC_API_KEY:/);
});
it('keeps docs-only pull requests on the core suite', () => {
const outputs = runPlan({
GITHUB_EVENT_NAME: 'pull_request',
@@ -70,7 +70,7 @@ describe('live-e2e accept cleanup regression', () => {
log: (msg) => t.diagnostic(msg),
});
const { page, tmp, teardown } = session;
const { page, tmp, appRoot, teardown } = session;
try {
t.diagnostic(`Using LLM agent (provider=${llmConfig.provider} model=${llmConfig.model})`);
await waitForHandshake(page);
@@ -104,6 +104,10 @@ describe('live-e2e accept cleanup regression', () => {
await clickNext(page);
assert.equal(await getVisibleVariant(page), 2, 'variant 2 visible after one Next');
const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('impeccable-live-session') || 'null'));
assert.ok(saved?.id && /^[a-zA-Z0-9_-]+$/.test(saved.id), 'cycling session has a safe durable id');
const snapshotPath = join(appRoot, '.impeccable/live/sessions', `${saved.id}.snapshot.json`);
t.diagnostic('Accepting variant 2');
await clickAccept(page, { expectedVariant: 2 });
@@ -120,6 +124,19 @@ describe('live-e2e accept cleanup regression', () => {
sourceFile,
finalSource,
});
// Source/DOM cleanup can precede live-complete, or succeed while its
// durable acknowledgement fails. Do not count that as a finished accept.
const deadline = Date.now() + 30_000;
let snapshot;
do {
snapshot = JSON.parse(readFileSync(snapshotPath, 'utf8'));
if (snapshot.phase === 'completed') break;
await new Promise((resolve) => setTimeout(resolve, 100));
} while (Date.now() < deadline);
assert.equal(snapshot.phase, 'completed', 'accept must reach durable completed phase without forcing completion');
assert.doesNotMatch(readFileSync(sourceFile, 'utf8'), /data-impeccable-[\w-]+\s*=/, 'accepted source contains no reserved runtime attributes');
t.diagnostic(`Durable completion verified for ${saved.id}`);
} finally {
await teardown();
}
+6
View File
@@ -1520,6 +1520,12 @@ describe('live-e2e LLM agent variant prompt', () => {
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /bare text element/);
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /Accept persists a real source change/);
});
it('uses permanent styling hooks outside the reserved live-runtime namespace', () => {
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /data-design-variant/);
assert.doesNotMatch(VARIANT_SYSTEM_INSTRUCTIONS, /add[^\n]*data-impeccable-e2e-variant/);
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /Never invent data-impeccable-\*/);
});
});
describe('live-e2e LLM agent variant copy validation', () => {
+1 -1
View File
@@ -83,7 +83,7 @@ export const VARIANT_SYSTEM_INSTRUCTIONS = [
'- Replace mode: for bare text elements, keep the full visible copy in one editable text node. If you add child markup for styling, wrap the entire copy; never split the copy across sibling text nodes.',
'- Replace mode: PRESERVE existing class-bearing descendant elements in place. If the picked element contains <h1 class="hero-title"> and <p class="hero-hook">, keep those elements/classes as direct descendants of the replacement root; do not wrap them in a new structural div such as <div class="hero-inner">.',
'- Replace mode: Do not return source-identical variants. For a bare text element, preserve the root tag/class/copy but add a small child span or styling hook so Accept persists a real source change.',
'- Replace mode: for non-bare elements where the existing children must stay in place, add a harmless root attribute such as data-impeccable-e2e-variant="1" or another non-copy styling hook so the markup is materially changed without changing visible text.',
'- Replace mode: for non-bare elements where the existing children must stay in place, add a permanent styling hook such as data-design-variant="1" so the markup is materially changed without changing visible text. Never invent data-impeccable-* attributes: that namespace is reserved for temporary runtime state.',
'- Generate exactly event.count variants — no more, no fewer.',
'- Mix the param kinds across the variant set: include at least one range, one steps, and one toggle when count >= 3.',
'- The scopedCss must follow wrapInfo.cssAuthoring exactly: use its selector strategy, rulePattern, requirements, and forbidden patterns.',
+302 -3
View File
@@ -3,8 +3,188 @@ import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { MockLanguageModelV3 } from 'ai/test';
import { prepareWorkspace, cleanupWorkspace, makeTools, runTurn, SKILL_BODY } from './skill-behavior/harness.mjs';
import { assertPlanningFallbackWarning } from './skill-behavior/assertions.mjs';
import { prepareWorkspace, cleanupWorkspace, makeTools, runTurn, fileLoaded, SKILL_BODY } from './skill-behavior/harness.mjs';
import { assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs';
import { CASE_STUDY_ANSWER } from './skill-behavior/fixtures.mjs';
import { sourceHash as hashSources } from './skill-workflow/source-hash.mjs';
import { assertCompleted, assertFreshCaptures, assertNoChangeDocumentation, assertDocumentationArtifacts } from './skill-workflow/assertions.mjs';
it('documentation artifacts require tokens and the v2 sidecar independently of wrapper coverage', () => {
const design = '---\ncolors:\n ink: "#222"\ntypography:\n body:\n fontFamily: system-ui\n---\n## Overview\nA reading surface.\n';
const sidecar = JSON.stringify({ schemaVersion: 2, extensions: { colorMeta: {} }, narrative: { northStar: 'Manual' } });
assert.doesNotThrow(() => assertDocumentationArtifacts(design, sidecar));
assert.throws(() => assertDocumentationArtifacts('## Colors\nInk: #222\n', sidecar), /frontmatter/);
assert.throws(() => assertDocumentationArtifacts(design.replace('colors:', 'palette:'), sidecar), /color tokens/);
assert.throws(() => assertDocumentationArtifacts(design.replace('typography:', 'type:'), sidecar), /typography tokens/);
assert.throws(() => assertDocumentationArtifacts(design, ''), SyntaxError);
assert.throws(() => assertDocumentationArtifacts(design, sidecar.replace('"schemaVersion":2', '"schemaVersion":1')), /v2 sidecar/);
for (const key of ['extensions', 'narrative']) {
for (const value of [undefined, {}, []]) {
assert.throws(() => assertDocumentationArtifacts(design, JSON.stringify({ ...JSON.parse(sidecar), [key]: value })), /metadata/);
}
}
});
it('advice outcomes do not depend on opening every reference, but keep consent and prerequisite gates', () => {
const trace = { toolCalls: [], writePaths: [], questionCalls: [], bashCommands: [] };
const advice = 'For index.html, start with init to capture context and document to record the existing identity.';
const comparison = 'Critique is an assessment. Polish makes fixes. Critique is optional; polish can run directly.';
const check = (text, observation = trace) => assertCommandComparison(observation, text);
assert.doesNotThrow(() => assertWorkflowAdvice(trace, advice, { missingContext: true }));
assert.doesNotThrow(() => check(comparison));
assert.doesNotThrow(() => check("Critique reviews the surface. Polish refines it. Critique isn't required before polish."));
assert.doesNotThrow(() => check('Critique gives a report. Polish edits the surface independently, without a critique.'));
assert.deepEqual(missingReferences(trace, ['reference/critique.md']), ['reference/critique.md']);
for (const wrong of ['You must run critique before polish.', 'Critique is required before polish.', 'Polish requires a critique.']) {
assert.throws(() => check(`${comparison} ${wrong}`), /invent a critique prerequisite/);
}
for (const wrong of ['You must run init before polishing.', 'You need to document before refinement.', 'Polish requires PRODUCT.md.']) {
assert.throws(() => assertWorkflowAdvice(trace, `${advice} ${wrong}`, { missingContext: true }), /mandatory prerequisite/);
}
assert.throws(() => check(''), /advice must reach/);
assert.throws(() => check('I loaded critique.md and polish.md.'), /explain critique/);
for (const mutation of ['index.html', '.impeccable/critique/report.md']) {
assert.throws(() => check(comparison, { ...trace, toolCalls: [{ mutatedPaths: [mutation] }] }), /must not edit/);
}
assert.throws(() => check(comparison, { ...trace, writePaths: ['DESIGN.md'] }), /write tool/);
assert.throws(() => check(comparison, { ...trace, questionCalls: [{}] }), /interview/);
assert.throws(() => check(comparison, { ...trace, bashCommands: ['impeccable detect index.html'] }), /menu scans/);
});
it('an unchanged documentation outcome needs real reads and a supported report, not a wrapper filename', () => {
const files = ['reference/document.md', 'index.html', 'DESIGN.md'];
const toolCalls = files.map((file) => ({ loadedFiles: [file] }));
const result = { outcome: 'complete', trace: { toolCalls }, text: 'No changes: index.html matches DESIGN.md: system-ui, 65ch, #0645ad.' };
const options = { target: 'index.html', evidence: [/system-ui/, /65ch/, /#0645ad/] };
const check = (value) => assertNoChangeDocumentation(value, options);
assert.doesNotThrow(() => check(result));
assert.deepEqual(missingReferences(result.trace, ['degraded/documenter.md']), ['degraded/documenter.md']);
for (const missing of files) {
assert.throws(() => check({ ...result, trace: { toolCalls: toolCalls.filter((call) => !call.loadedFiles.includes(missing)) } }), /inspect the actual source/);
}
assert.throws(() => check({ ...result, trace: { toolCalls: [{ input: { path: 'reference/document.md' }, succeeded: false }] } }), /inspect the actual source/);
assert.throws(() => check({ ...result, text: 'No changes: index.html matches DESIGN.md.' }), /report evidence/);
assert.throws(() => check({ ...result, outcome: 'step-budget' }), /did not finish/);
for (const file of ['DESIGN.md', '.impeccable/design.json', 'index.html']) {
assert.throws(() => check({ ...result, trace: { toolCalls: [...toolCalls, { mutatedPaths: [file] }] } }), /must not mutate/);
}
});
it('full workflows reject exhausted budgets and stale or absent visual evidence', () => {
for (const outcome of ['checkpoint', 'step-budget', 'output-limit', 'error']) {
assert.throws(() => assertCompleted({ outcome, steps: 50 }), /did not finish/);
}
assert.doesNotThrow(() => assertCompleted({ outcome: 'complete', steps: 12 }));
const workspace = prepareWorkspace({ files: { 'index.html': '<h1>Test</h1>' } });
try {
const sourceHash = hashSources(workspace);
const edit = { mutatedPaths: ['index.html'] };
const shots = ['desktop', 'mobile'].map((viewport) => ({ capture: { target: 'index.html', viewport, sourceHash } }));
const check = (toolCalls) => assertFreshCaptures({ toolCalls }, workspace, 'index.html');
assert.doesNotThrow(() => check([edit, ...shots]));
assert.throws(() => check([edit]), /missing desktop screenshot/);
assert.throws(() => check([...shots, edit]), /missing desktop screenshot/);
assert.throws(() => check([edit, shots[0]]), /missing mobile screenshot/);
fs.writeFileSync(path.join(workspace, 'style.css'), 'h1 { color: red; }');
assert.throws(() => check([edit, ...shots]), /missing desktop screenshot/);
} finally {
cleanupWorkspace(workspace);
}
});
it('case-study user supplies evidence now instead of promising a future message', async () => {
const workspace = prepareWorkspace();
try {
const { tools, trace } = makeTools(workspace, {}, { answer: () => CASE_STUDY_ANSWER });
for (const question of ['What real customer proof do you have?', 'Please paste the promised details.']) {
const result = JSON.parse(await tools.ask_user_question.execute({ questions: [{
question, options: [{ label: 'I will paste real customer quotes in my next message' }],
}] }));
assert.equal(result.answers[question], CASE_STUDY_ANSWER);
assert.match(result.answers[question], /clearly labeled synthetic case/);
}
assert.equal(trace.questionAnswers.length, 2);
} finally {
cleanupWorkspace(workspace);
}
});
it('new-work requires approval and a brief before code, then documents the finished redesign', () => {
const ask = { name: 'ask_user_question' };
const brief = { name: 'bash', mutatedPaths: ['.impeccable/surfaces/current-html.md'] };
const page = { name: 'write', mutatedPaths: ['current.html'] };
const design = { name: 'write', mutatedPaths: ['DESIGN.md'] };
const check = (toolCalls) => assertNewWorkLifecycle({ toolCalls }, { target: 'current.html', redesign: true });
assert.doesNotThrow(() => check([ask, brief, page, design]));
assert.doesNotThrow(() => check([ask, brief, page, design, page, design]));
assert.throws(() => check([ask, brief]), /did not produce/);
assert.throws(() => check([brief, page, ask, design]), /user answer/);
assert.throws(() => check([ask, page, brief, design]), /surface brief before/);
assert.throws(() => check([ask, brief, design, page]), /finished build/);
assert.throws(() => check([ask, brief, page, design, page]), /finished build/);
});
it('stages resolved references independently of the source skill', async () => {
const workspace = prepareWorkspace();
try {
const base = path.join(workspace, '.claude/skills/impeccable');
assert.equal(fs.lstatSync(base).isSymbolicLink(), false);
const { tools } = makeTools(workspace);
const critique = await tools.read.execute({ path: '.claude/skills/impeccable/reference/critique.md' });
assert.match(critique, /Use the ask_user_question tool\./);
assert.doesNotMatch(critique, /\{\{ask_instruction\}\}|\{\{scripts_path\}\}|<codex>/);
for (const role of ['finish-reviewer', 'documenter']) {
const reference = await tools.read.execute({ path: `.claude/skills/impeccable/reference/degraded/${role}.md` });
assert.match(reference, /This harness has no subagent capability/);
assert.doesNotMatch(reference, /\{\{scripts_path\}\}|<codex>/);
}
const shellRead = await tools.bash.execute({ command: 'cat .claude/skills/impeccable/reference/critique.md' });
assert.ok(shellRead.includes(critique), 'shell and read tools must see the same resolved reference');
assert.match(await tools.write.execute({ path: '.claude/skills/impeccable/reference/critique.md', contents: 'bad' }), /^Error:/);
} finally {
cleanupWorkspace(workspace);
}
});
it('reference-loading evidence requires content, not a failed read or a filename mention', async () => {
const workspace = prepareWorkspace();
try {
const ref = '.claude/skills/impeccable/reference/polish.md';
const denied = makeTools(workspace, {}, {}, { denyBash: true });
await denied.tools.bash.execute({ command: `cat ${ref}` });
assert.equal(fileLoaded(denied.trace, 'polish.md'), false);
await denied.tools.read.execute({ path: 'missing/polish.md' });
assert.equal(fileLoaded(denied.trace, 'polish.md'), false);
const allowed = makeTools(workspace);
await allowed.tools.bash.execute({ command: `printf '%s' '${ref}'` });
assert.equal(fileLoaded(allowed.trace, 'polish.md'), false);
await allowed.tools.bash.execute({ command: `cat ${ref}` });
assert.equal(fileLoaded(allowed.trace, 'polish.md'), true);
await denied.tools.read.execute({ path: ref });
assert.equal(fileLoaded(denied.trace, 'polish.md'), true);
} finally {
cleanupWorkspace(workspace);
}
});
it('headless behavior shells disable unattended decision pages and omit provider credentials', async () => {
const workspace = prepareWorkspace();
try {
const { tools } = makeTools(workspace, { OPENAI_API_KEY: 'synthetic-secret', IMPECCABLE_QUESTION_DISABLED: '0' });
const result = await tools.bash.execute({ command: 'node -e \'console.log(JSON.stringify({disabled:process.env.IMPECCABLE_QUESTION_DISABLED,hasKey:!!process.env.OPENAI_API_KEY}))\'' });
assert.match(result, /"disabled":"1"/);
assert.match(result, /"hasKey":false/);
assert.doesNotMatch(result, /synthetic-secret/);
if (process.env.IMPECCABLE_BIN) {
const question = await tools.bash.execute({ command: '.claude/skills/impeccable/scripts/impeccable serve-question --start --payload nonexistent.json' });
assert.match(question, /^exit=2\n/);
assert.match(question, /use the structured question tool instead/);
assert.equal(fs.existsSync(path.join(workspace, '.impeccable/questions')), false);
}
} finally {
cleanupWorkspace(workspace);
}
});
it('planning fallback requires an assistant warning between the denial and context reads', () => {
const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };
@@ -47,6 +227,81 @@ it('DeepSeek gets an explicit output ceiling instead of the compatibility SDK de
}
});
it('protocol checkpoints stop at successful evidence without claiming task completion', async () => {
const workspace = prepareWorkspace();
try {
const model = new MockLanguageModelV3({ modelId: 'claude-sonnet-5', doGenerate: {
content: [{ type: 'tool-call', toolCallId: 'load', toolName: 'read', input: JSON.stringify({ path: '.claude/skills/impeccable/reference/polish.md' }) }],
finishReason: { unified: 'tool-calls', raw: 'tool-calls' },
usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } }, warnings: [],
} });
const result = await runTurn({ workspace, model, userPrompt: 'Route only.', maxSteps: 10,
stopAfter: (trace) => fileLoaded(trace, 'polish.md') });
assert.equal(result.outcome, 'checkpoint');
assert.equal(result.steps, 1);
assert.equal(model.doGenerateCalls.length, 1);
const exhausted = await runTurn({ workspace, model, userPrompt: 'Complete work.', maxSteps: 1 });
assert.equal(exhausted.outcome, 'step-budget');
} finally { cleanupWorkspace(workspace); }
});
it('protocol notice checkpoints observe intermediate assistant text', async () => {
const workspace = prepareWorkspace();
try {
let calls = 0;
const model = new MockLanguageModelV3({ doGenerate: async () => {
calls++;
return {
content: [{ type: 'text', text: 'Impeccable version 99.0.0 is available; may I update it?' },
{ type: 'tool-call', toolCallId: 'list', toolName: 'list', input: '{}' }],
finishReason: { unified: 'tool-calls', raw: 'tool-calls' },
usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } }, warnings: [],
};
} });
const result = await runTurn({ workspace, model, userPrompt: 'Inspect this page.', maxSteps: 10,
stopAfter: (trace) => trace.assistantTexts?.some((text) => text.includes('99.0.0')) });
assert.equal(calls, 1);
assert.equal(result.outcome, 'checkpoint');
assert.match(result.trace.assistantTexts[0], /may I update/);
} finally {
cleanupWorkspace(workspace);
}
});
it('optional diagnostics retain tool evidence when a provider turn fails', async () => {
const workspace = prepareWorkspace({ files: { 'PRODUCT.md': 'Synthetic product context.' } });
const previous = process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR;
const traceDir = path.join(workspace, 'diagnostics');
process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR = traceDir;
try {
let calls = 0;
const model = new MockLanguageModelV3({
modelId: 'claude-sonnet-5',
doGenerate: async () => {
if (calls++ === 0) return {
content: [{ type: 'tool-call', toolCallId: 'read-product', toolName: 'read', input: JSON.stringify({ path: 'PRODUCT.md' }) }],
finishReason: { unified: 'tool-calls', raw: 'tool-calls' },
usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } },
warnings: [],
};
throw new Error('synthetic provider failure');
},
});
await assert.rejects(runTurn({ workspace, model, userPrompt: 'Synthetic diagnostic test.' }), /synthetic provider failure/);
const files = fs.readdirSync(traceDir);
assert.equal(files.length, 1);
const diagnostic = JSON.parse(fs.readFileSync(path.join(traceDir, files[0]), 'utf8'));
assert.equal(diagnostic.status, 'failed');
assert.match(diagnostic.error, /synthetic provider failure/);
assert.equal(diagnostic.trace.toolCalls.length, 1);
assert.equal(fileLoaded(diagnostic.trace, 'PRODUCT.md'), true);
} finally {
if (previous === undefined) delete process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR;
else process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR = previous;
cleanupWorkspace(workspace);
}
});
it('loaded-skill metadata resolves to the staged launcher and readable references', async () => {
const workspace = prepareWorkspace();
try {
@@ -96,18 +351,62 @@ it('context-only routing tools reject shell searches and compound commands befor
for (const command of [
'find / -name routing.md',
'.claude/skills/impeccable/scripts/impeccable context; echo bad > index.html',
'.claude/skills/impeccable/scripts/impeccable context --target index.html; echo bad > index.html',
'.claude/skills/impeccable/scripts/impeccable context --target "$(echo bad > index.html)"',
'.claude/skills/impeccable/scripts/impeccable context --target=index.html; echo bad > index.html',
'.claude/skills/impeccable/scripts/impeccable context --target="$(echo bad > index.html)"',
'echo bad > index.html',
]) {
assert.match(await tools.bash.execute({ command }), /^Error:/);
}
assert.equal(fs.readFileSync(path.join(workspace, 'index.html'), 'utf8'), 'before');
assert.equal(trace.bashCommands.length, 3, 'rejected attempts remain observable');
assert.equal(trace.bashCommands.length, 7, 'rejected attempts remain observable');
assert.ok(trace.toolCalls.every((call) => call.mutatedPaths.length === 0));
} finally {
cleanupWorkspace(workspace);
}
});
it('successful-loader controls accept a workspace-relative target', { skip: !process.env.IMPECCABLE_BIN }, async () => {
const workspace = prepareWorkspace({ files: { 'index.html': '<html></html>' } });
try {
const { tools } = makeTools(workspace, {}, {}, { contextOnlyBash: true });
assert.match(await tools.bash.execute({ command: '.claude/skills/impeccable/scripts/impeccable context --target index.html' }), /^exit=0\n/);
for (const separator of [' ', '=']) {
for (const target of ['index.html', 'src/routes/+page.svelte', '"src/routes/+page.svelte"', '"my page.html"', "'my page.html'"]) {
assert.match(await tools.bash.execute({ command: `.claude/skills/impeccable/scripts/impeccable context --target${separator}${target}` }), /^exit=0\n/);
}
assert.match(await tools.bash.execute({ command: `.claude/skills/impeccable/scripts/impeccable context --target${separator}../outside.html` }), /^Error:/);
}
} finally {
cleanupWorkspace(workspace);
}
});
it('workspace tools reject symlink escapes and preserve staged-skill write protection', async () => {
const workspace = prepareWorkspace({ files: { 'local/value.txt': 'local' } });
const outside = prepareWorkspace({ files: { 'value.txt': 'outside' } });
try {
fs.symlinkSync(outside, path.join(workspace, 'escape'), 'junction');
fs.symlinkSync(path.join(workspace, 'local'), path.join(workspace, 'alias'), 'junction');
fs.symlinkSync(path.join(workspace, '.claude'), path.join(workspace, 'skill-alias'), 'junction');
const { tools } = makeTools(workspace, {}, {}, { contextOnlyBash: true });
assert.match(await tools.read.execute({ path: 'escape/value.txt' }), /^Error:/);
assert.match(await tools.list.execute({ path: 'escape' }), /^Error:/);
assert.match(await tools.write.execute({ path: 'escape/new/file.txt', contents: 'bad' }), /^Error:/);
assert.match(await tools.bash.execute({ command: '.claude/skills/impeccable/scripts/impeccable context --target escape/value.txt' }), /^Error:/);
assert.match(await tools.write.execute({ path: 'skill-alias/skills/impeccable/reference/routing.md', contents: 'bad' }), /^Error:/);
assert.equal(await tools.read.execute({ path: 'alias/value.txt' }), 'local');
await tools.write.execute({ path: 'alias/nested/new.txt', contents: 'allowed' });
assert.equal(fs.readFileSync(path.join(workspace, 'local/nested/new.txt'), 'utf8'), 'allowed');
assert.equal(fs.existsSync(path.join(outside, 'new')), false);
assert.equal(fs.readFileSync(path.join(outside, 'value.txt'), 'utf8'), 'outside');
} finally {
cleanupWorkspace(workspace);
cleanupWorkspace(outside);
}
});
it('context-only routing tools keep project writes observable but protect the staged skill', async () => {
const workspace = prepareWorkspace({ files: { 'index.html': 'before' } });
try {
+262 -16
View File
@@ -2,8 +2,8 @@
LLM-backed scenarios that verify how the impeccable skill drives context,
command-reference, new-work, and native-platform loading. Each scenario runs
against one current model from each supported provider (Anthropic, OpenAI,
Google, DeepSeek).
against the default Anthropic, OpenAI, and Google models. DeepSeek remains
available through `IMPECCABLE_SKILL_BEHAVIOR_MODELS`.
These are the tests you re-run when you refactor anything in SKILL.md's
`## Setup` section. They fail when the agent stops following the loading
@@ -25,7 +25,7 @@ skipped, not failed.
Also requires the engine binary (`bun run fetch:engine`, or `IMPECCABLE_BIN`).
The staged skill dir ships the launcher (`scripts/impeccable`); the harness
exports `IMPECCABLE_BIN` into every bash call the agent makes, so the launcher
resolves the binary in both symlink and copy mode without a download. Without a
resolves the binary in the generated fixture without a download. Without a
binary the suites skip.
To run a single scenario against one model:
@@ -37,15 +37,175 @@ IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-sonnet-5 IMPECCABLE_SKILL_BEHAVIOR_VERBO
## How it works
### Protocol versus full completion
`test:skill-behavior` now runs only `scenarios.test.mjs`. Routing cases stop
at the successful reference/context checkpoint they assert, with a ten-step
ceiling; shell access is context-only. They do **not** claim that a page was
built or reviewed. Editing/fallback controls retain their original assertions.
The focused S1/S2/S3/S4/S19 rerun passed 21/21 across the three default models.
A broader run exposed the context-only allowlist rejecting Svelte's valid
`+page.svelte` target. It was stopped, the allowlist fixed with a failing-then-
passing unit test, and S8 passed 3/3 on the focused rerun. Failed file reads
do not count as project exploration.
Update-notice (S9) and explicit-command (S18) checks also now stop at their
actual protocol checkpoints, instead of continuing into unrelated polishing;
their focused final reruns each passed 3/3. S9 now explicitly requires the
assistant to surface the update, not merely receive its loader directive.
Full workflows moved to `tests/skill-workflow/full-build.test.mjs`:
```bash
bun run fetch:engine
bunx playwright install chromium
bun run test:skill-workflow
```
This separately billed suite defaults to Claude only; use
`IMPECCABLE_SKILL_BEHAVIOR_MODELS` to explicitly choose another model or sweep.
It preflights a local server and Chromium before each provider turn, exposing
real desktop/mobile screenshot and PNG viewing tools. Text-only fixtures use
system fonts and block external browser requests. No extra skill prose is added.
The API harness is not the actual Claude Code host, nor is its shell sandboxed.
Each workflow has a 50-step/840-second ceiling. Reaching a budget or output
limit fails explicitly; routing checkpoints cannot satisfy completion. UI
workflows require desktop and mobile captures matching the final local sources after
its last edit. Approval/brief-before-code and redesign documentation-at-finish
checks remain, as does exactly one context load across the completed turn.
CI runs this lane only when its manual `skill_workflow` checkbox is enabled.
Ordinary protocol CI now fetches its engine instead of silently skipping for
a missing binary. Full-build results must be reported separately from routing.
### Remaining gaps after the split (2026-09-07)
One provisioned Claude natural-build run reached a final response in 637 seconds
and 37 model steps, with approval, a surface brief, an implemented page, a
finish review, corrections, and fresh desktop/mobile screenshots. Its initial
completion assertions passed. The final test revision additionally requires
the shipped documentation reference; auditing the saved trace against that
guard found it missing. **This is not a final full-workflow pass.** Existing
DESIGN.md was preserved, but the required documentation pass was skipped.
The final guards were tightened during the run; this trace is not represented
as a run of those later assertions. No second full build was purchased.
Claude's remaining protocol batch passed S10S15 and existing-project S16,
but missing-context S16 omitted routing.md and S17 omitted critique.md.
Both responses remained read-only. The older S9 timed out during unrelated
polishing; the corrected focused S9 above supersedes it. The batch was stopped
during the older S18, before another provider sweep. This is incremental
evidence, not an all-green final matrix. Release #782 remains on hold.
The full build reported about 2.95 million input and 53 thousand output tokens
across all turns (no cache usage reported). Keep this lane manually scoped;
the fast protocol suite is not a proxy for its completion or cost.
### Documentation handoff follow-up
`tests/skill-workflow/finish-handoff.test.mjs` isolates a synthetic post-review
checkpoint without rebuilding or capturing a page. Its existing-system fixture
has no approved system change and a pre-existing missing sidecar; the correct
result is to check the build against DESIGN.md, preserve it, and leave unrelated
drift alone. The new-world control must write DESIGN.md and its v2 sidecar.
The unchanged-instructions baseline reproduced the skipped documenter in 31s.
The revised handoff and documenter passages are 29 words shorter overall and
make a checked no-change result explicit. The first focused extension retest
passed in 27s. A repeat checked the source, DESIGN.md, and document.md and made
no mutations, but omitted degraded/documenter.md, so the strict reference guard
still failed. The new-world control passed in 91s, writing both required files.
These small samples support the narrower behavior change, not an all-green
workflow claim; the reference-loading miss remains visible. No full build was
rerun. Focused Claude routing S3/S4, the ordinary suite, source-first build, and
generated-skill authoring validation passed. Release #782 remains held.
### Outcome assertions (maintainer-approved follow-up)
S16/S17 now require a completed, useful, read-only answer; S17 distinguishes
assessment from implementation and explains that critique is optional before
polish. Explicit invented prerequisites remain failures. Missing routing or
comparison references are TAP diagnostics based on successful content loads,
not failed read attempts. These English-fixture phrase checks are bounded
regression checks, not a comprehensive semantic grader.
The resumed ordinary-extension checkpoint accepts a direct documentation pass
without the degraded wrapper only with actual document.md, page, and DESIGN.md
reads, a concrete no-change report grounded in the fixture's type/palette/layout,
and zero mutations. Seeded files must still be byte-identical and pre-existing
sidecar drift must remain untouched. New-world documentation writes, redesign
ordering, and full-build completion gates are unchanged.
Offline re-evaluation of saved Claude traces: three advice responses and two
evidenced no-op handoffs pass the new assertions. The original post-review
baseline and the 637-second full build still fail for absent documentation
evidence. Unit negative controls reject fabricated prerequisites, unsolicited
edits/interviews/scans, failed reads, empty or unsupported reports, and exhausted
budgets. This is assertion replay, not new model evidence or a rerun of cleaned-up
workspaces' filesystem checks. No paid calls or skill prose changes were needed.
The earlier reference misses above are now diagnostics, not release blockers by
themselves; this does not establish an all-green full-workflow matrix.
### Bounded release verification (2026-09-08)
The focused Anthropic live-accept test now verifies the durable session reaches
`completed`, not just clean source/DOM. Its agent instruction had recommended
`data-impeccable-e2e-variant`, inside the reserved runtime namespace; the test
prompt now uses a permanent `data-design-variant` styling hook. The strengthened
test passed in 14s without forced completion. The full non-billed suite passed.
No runtime or skill source was changed for this correction.
Claude post-review controls: new world passed in 77s, writing token-bearing
DESIGN.md and the v2 sidecar; approved redesign failed in 16s. The latter read the
page and old system, then wrote prose-only DESIGN.md without consulting the
documentation spec or creating `.impeccable/design.json`, and claimed nothing
remained outstanding. This is a missing required artifact, not merely missing
reference coverage. The test stays red; no retry was purchased. Release remains
held pending disposition. These are synthetic post-review checkpoints, not
proof of a complete redesign lifecycle.
The single full Claude build passed its automated gates in 623s / 31 steps:
approval, brief, implemented page, final desktop/mobile captures, and shipped
reviewer/documenter wrapper loads. The final response gave an in-thread review
and a no-change documentation assessment of the incumbent system. It did not
load document.md itself; this is not evidence that every documentation protocol
step ran. The run used about 2.11M input / 50K output tokens (no cache reported).
No further billed retry was started. The separate missing-artifact redesign
failure still blocks treating this batch as an all-green skill release gate.
### Targeted redesign handoff correction
The parent handoff now explicitly requires token-bearing DESIGN.md and
`.impeccable/design.json` for approved system changes and verifies those outputs
before completion. It is nine words shorter; the agent and schema files did not
change. One unchanged-assertion Claude retest finished in 65s: it read document.md
and wrote both artifacts, but still skipped the degraded wrapper, so that
reference assertion failed before the artifact assertions ran.
Wrapper coverage is now diagnostic for all post-review modes; successful spec
and source reads, completed turns, write boundaries, tokens, and the v2 sidecar
remain hard gates. Offline evaluation of the saved retest passes these artifact
checks; the original prose-only/missing-sidecar trace remains rejected. Negative
controls cover absent tokens, malformed/missing sidecars, old schema versions,
and absent metadata. This is replay, not a second live pass or a rerun of the
cleaned-up workspace's byte-preservation checks.
Artifact audit: the sidecar component renders correctly in an offline browser.
The heading line-height was recorded as 1.3 while the page inherits 1.6 (38.4px
at 24px), and generatedAt used a placeholder date. These are remaining output
accuracy limitations, distinct from the corrected missing-artifact failure;
the shape checks do not establish complete token fidelity. No broad provider or
full-build rerun was purchased. Build and generated-skill validation passed.
Each scenario:
1. `prepareWorkspace()` mints a temp dir, symlinks the canonical skill
into `<workspace>/.claude/skills/impeccable` (so its launcher is at
`.claude/skills/impeccable/scripts/impeccable`), and optionally writes
`PRODUCT.md` / `DESIGN.md` fixtures.
1. `prepareWorkspace()` uses the production transformer to build current source
into an independent `<workspace>/.claude/skills/impeccable`. References have
resolved placeholders and generated degraded reviewer/documenter files.
Host-specific blocks are omitted: this is a neutral API harness, not an exact
Claude/Codex/Gemini host simulation. It optionally seeds project fixtures.
2. `runTurn()` inlines `SKILL.md` (placeholders neutralized) as the
system prompt and runs Vercel AI SDK `generateText` with four
workspace-scoped tools: `bash`, `read`, `write`, `list`, and a fake
system prompt and runs Vercel AI SDK `generateText` with five
tools: `bash`, `read`, `write`, `list`, and a fake
provider-neutral `ask_user_question` backed by a deterministic simulated user.
3. The tools record every call into a `trace` that the test asserts on.
4. For scenario 4, a second `runTurn` reuses turn 1's `responseMessages`
@@ -53,6 +213,92 @@ Each scenario:
The trace is the source of truth, not the model's free-form reply.
File tools are workspace-scoped; bash is a real host shell, **not a security
sandbox**. Use disposable synthetic fixtures. Shell helpers do not inherit
provider API keys/auth tokens; model calls still use the parent's keys. The
harness always sets `IMPECCABLE_QUESTION_DISABLED=1` for shell calls so real
decision pages cannot wait for a nonexistent browser user. The engine returns
its genuine structured-question fallback; browser decisions have separate E2E.
Set `IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR=<directory>` to retain per-turn JSON
with model, prompt, tool results, response ordering, usage, and finish reason.
Progress and failed turns retain their tool traces too; only completed turns
carry the full response sequence and final usage.
These are local diagnostic artifacts; inspect before sharing. Successful reads
or full reference content in shell output count as loading; filename mentions,
denied commands, and failed reads do not.
Context-only controls permit the real launcher with an optional workspace-relative
`--target`; compound commands remain rejected. The target form is part of the
skill's Setup contract, not a launcher failure.
## Release investigation (2026-09-07)
The initial release sweep reported 68/81 passes. Do not interpret its 13 failed
assertions as 13 demonstrated product regressions. The harness staged raw
references with unresolved placeholders, omitted generated degraded roles, and
allowed unanswered browser decisions. Its redesign assertion was also stale:
current `new-work.md` requires a surface brief **before code**, and DESIGN.md
**at finish**, from the built world. The corrected lifecycle checks retain
approval, brief, implementation, and final documentation requirements; missing
artifacts now have their own error instead of being called premature edits.
The historical tables below retain their original measurements and methods.
Focused launcher-fallback verification on the corrected fixture:
| Default model | Old fallback paragraph | Explicit pre-tool warning paragraph |
|---|---:|---:|
| `claude-sonnet-5` | 2/3 | 3/3 |
| `gpt-5.6-terra` | 3/3 | 3/3 |
| `gemini-3.7-flash` | 1/3 | 3/3 |
The three cases are denied editing, successful-loader control, and denied
planning. The old-paragraph failures were warning order, not refused edits.
An intermediate candidate run scored 8/9 because the control rejected valid
`context --target index.html`; after correcting that allowlist, the full focused
rerun passed 9/9. This is one measured run per variant, not a reliability estimate
or an all-workflow pass. Broader routing and workflow results remain separate.
On the resolved fixture, Gemini's workflow run passed 4/5: the completed new
page omitted user confirmation. Tightening the existing question paragraph
made its focused build lifecycle rerun pass 1/1. OpenAI passed all five workflow
cases with the fixture corrections alone. These are incremental measurements,
not one full sweep on the final candidate.
Claude's three-step routing sweep cut off two setup cases before loading
`new-work.md`. Both loaded it in bounded six-step diagnostics. Claude now has
the same six-step setup allowance as Gemini; reference and edit-order assertions
are unchanged. A full-build baseline separately hit the existing 840-second
deadline and is not counted as a pass.
The complete routing-only sweep retained its original budgets and passed 55/57
(Claude 17/19, OpenAI 19/19, Gemini 19/19). A six-step Claude rerun passed the two
previously clipped cases but exposed a separate context reload on turn two:
the model queried the loader again for image-tool availability. That run was
stopped after five passes and this failure rather than finishing another billed
sweep; the once-per-session assertion remains unchanged.
A saved Claude build trace used five calls shortening/counting the direction
contract, then reached the 22-step cap before implementation. The word target
is now approximate, with the six required blocks retained. A subsequent run
correctly stopped for missing customer evidence: the default simulated user had
selected “I have real details,” then promised them in a future message. The
case-study test now supplies a complete, explicitly synthetic brief when asked;
fresh-init and other user simulations are unchanged. Neither incomplete build
is counted as a pass.
The corrected-user Claude retest also remained incomplete: it asked, recorded
the six-block brief without the earlier word-count loop, then spent the remaining
22-step allowance acquiring and inspecting fonts before writing HTML. The
26-step redesign run produced the page and desktop/mobile captures but stopped
before DESIGN.md. These results do not establish full workflow completion.
Further work should separate narrow protocol checks from realistic, provisioned
full-build runs rather than keep adding skill prose or relaxing finish gates.
The later Claude run passed fresh init and refinement, failed the two bounded
build cases, and was stopped during critique's browser-tool discovery. Its
unfinished critique case is not a pass; the earlier OpenAI/Gemini critique
results remain the completed measurements.
## Scenarios
| # | Setup | Assertion |
@@ -72,8 +318,8 @@ The trace is the source of truth, not the model's free-form reply.
| 13 | empty workspace; prompt is `/impeccable teach` | runs `impeccable context` and diverts into `reference/init.md` because `teach` aliases `init` |
| 14 | PRODUCT.md with `## Platform: ios` (native iOS app); prompt is `/impeccable craft a tide detail screen` | `impeccable context` runs and emits the contents of `reference/ios.md` directly, placing native conventions in context without a second model-directed read |
| 15 | same iOS fixture; prompt is `/impeccable audit` | agent loads `reference/audit.native.md` (the Commands-table native variant, routed instead of `audit.md`) |
| 16 | existing surface, with and without PRODUCT.md; asks where to start | loads `routing.md`, delivers advice, and does not edit project files, start an interview, archive a critique, or run menu scans |
| 17 | existing surface; asks whether critique is required before polish | loads `routing.md` and both command references, then delivers advice without executing the playbooks |
| 16 | existing surface, with and without PRODUCT.md; asks where to start | completes relevant advice without edits, interviews, critique archives, menu scans, or explicit invented refinement prerequisites; reference coverage is diagnostic |
| 17 | existing surface; asks whether critique is required before polish | completes read-only advice distinguishing assessment from implementation and explaining critique is optional; reference coverage is diagnostic |
| 18 | existing surface; explicitly requests polish followed by a next-command recommendation | loads `polish.md` rather than substituting workflow advice for the requested work |
| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, and a denied-launcher planning-only case | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor |
@@ -188,7 +434,7 @@ evidence, not rejected bash reads. The initial unrestricted run (stopped after
host-wide search attempts), earlier rejected-read results, and broader suite's
sandboxed provider DNS errors are excluded from this baseline.
The workflow-contract file adds end-to-end assertions for attended fresh init,
The full-build file adds end-to-end assertions for attended fresh init,
an initialized natural build request, replacement-world redesign, scope-preserving bolder
refinement, and critique's closing question. It checks question order and
context/artifact writes rather than only reference-file loading.
@@ -365,12 +611,12 @@ when bisecting one scenario:
```bash
IMPECCABLE_QUESTION_DISABLED=1 CI=1 IMPECCABLE_SKILL_BEHAVIOR_MODELS=deepseek-v4-flash \
node --test --test-timeout=300000 --test-force-exit \
--test-name-pattern="bolder refinement" tests/skill-behavior/workflow-contract.test.mjs
--test-name-pattern="bolder refinement" tests/skill-workflow/full-build.test.mjs
```
Keep `--test-timeout` at 300000. A tighter cap turns claude-sonnet-5's slower
runs into timeouts that look like failures. Set `IMPECCABLE_QUESTION_DISABLED=1`
and `CI=1` so `impeccable serve-question` cannot open a browser window on the host. Pipe
Use the suite's current 900000ms timeout for full workflow cases; the 300000ms
example above is historical. The harness now disables decision pages itself.
Pipe
to a file rather than `tail`; node prints the failing-test summary at the end,
and truncating it costs you the per-model attribution.
+59
View File
@@ -1,5 +1,64 @@
import assert from 'node:assert/strict';
// These are bounded English-fixture checks, not a general semantic grader.
// Reference coverage is reported separately: opening a file proves neither
// useful advice nor permission to execute it.
export function missingReferences(trace, filenames) {
return filenames.filter((filename) => !trace.toolCalls.some((call) =>
(call.loadedFiles || []).some((file) => file === filename || file.endsWith(`/${filename}`))));
}
export function assertAdviceOnly(trace, text) {
assert.ok(text.trim(), 'advice must reach the user, not stop at reference loading');
assert.deepEqual(trace.writePaths, [], 'advice must not use the write tool');
const mutations = trace.toolCalls.flatMap((call) => call.mutatedPaths ?? [])
.filter((file) => !file.startsWith('.impeccable/') || file.startsWith('.impeccable/critique/'));
assert.deepEqual(mutations, [], 'advice must not edit project files or archive an unsolicited critique');
assert.deepEqual(trace.questionCalls, [], 'advice must not start an init or design interview');
assert.ok(!trace.bashCommands.some((command) => command.includes('impeccable detect')), 'workflow advice does not run menu scans');
}
function normalizedAdvice(text) {
return text.replace(/[`*_]/g, '').replace(/[]/g, "'").toLowerCase();
}
export function assertWorkflowAdvice(trace, text, { missingContext = false } = {}) {
assertAdviceOnly(trace, text);
const advice = normalizedAdvice(text);
assert.match(advice, /index\.html/, 'advice should address the existing surface');
assert.match(advice, missingContext ? /\binit\b/ : /\b(?:critique|audit|polish)\b/, 'advice must recommend a relevant starting point');
if (missingContext) assert.match(advice, /\bdocument\b/, 'advice should explain how to record the existing identity');
assert.doesNotMatch(advice, /(?:must|need to|have to|required to)\s+(?:run\s+)?(?:\/impeccable\s+)?(?:init|document)\b[^.!?\n]{0,100}\bbefore\s+(?:you\s+can\s+)?(?:run(?:ning)?\s+)?(?:polish(?:ing)?|refin(?:e|ing|ement))\b|(?:polish|refinement)\s+(?:requires|is blocked by|cannot run without)\s+(?:init|document|product\.md|design\.md)/,
'setup is not a mandatory prerequisite for narrow refinement');
}
export function assertCommandComparison(trace, text) {
assertAdviceOnly(trace, text);
const advice = normalizedAdvice(text);
assert.match(advice, /critique[^.!?\n]{0,120}(?:review|assess|evaluat|report|findings)/, 'comparison must explain critique as assessment');
assert.match(advice, /polish[^.!?\n]{0,120}(?:fix|refin|implement|edit)/, 'comparison must explain polish as implementation');
assert.match(advice, /critique\s+(?:is\s+)?(?:isn't|is not|not)\s+(?:required|necessary)|critique[^.!?\n]{0,50}\boptional\b|polish[^.!?\n]{0,100}(?:directly|without\s+(?:a\s+)?critique|independent)/,
'comparison must explain that critique is optional before polish');
assert.doesNotMatch(advice, /(?:must|need to|have to)\s+(?:run\s+)?critique[^.!?\n]{0,80}before\s+(?:run(?:ning)?\s+)?polish|critique\s+(?:is\s+)?(?:required|mandatory|necessary)\s+before\s+polish|polish\s+(?:requires|cannot run without)\s+(?:a\s+)?critique/,
'comparison must not invent a critique prerequisite');
}
export function assertNewWorkLifecycle(trace, { target, redesign = false }) {
const calls = trace.toolCalls;
const writes = (call, file) => (call.mutatedPaths || []).includes(file);
const implementation = calls.findIndex((call) => writes(call, target));
const question = calls.findIndex((call) => call.name === 'ask_user_question');
const brief = calls.findIndex((call) => (call.mutatedPaths || []).some((file) => file.startsWith('.impeccable/surfaces/')));
assert.ok(implementation >= 0, `new-work did not produce the requested artifact: ${target}`);
assert.ok(question >= 0 && question < implementation, 'implementation must follow a user answer');
assert.ok(brief >= 0 && brief < implementation, 'the direction contract must be recorded in a surface brief before implementation');
if (redesign) {
const lastImplementation = calls.findLastIndex((call) => writes(call, target));
const documentation = calls.findLastIndex((call) => writes(call, 'DESIGN.md'));
assert.ok(documentation > lastImplementation, 'redesign must record DESIGN.md from the finished build, after the last page edit');
}
}
export const LAUNCHER_FAILURE_WARNING = /(?:context|launcher|bash)[^.!?\n]{0,160}(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|did(?:n't| not)|fail|unable)|(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|unable)[^.!?\n]{0,160}(?:context|launcher|bash)/i;
export function assertPlanningFallbackWarning(responseMessages) {
+3
View File
@@ -5,6 +5,9 @@
* uses to decide whether to gate on `init`. Plausible enough that the agent
* treats them as real context rather than test scaffolding.
*/
// This user can supply a complete synthetic case now, not promise future assets.
export const CASE_STUDY_ANSWER = 'Build a standalone index.html for the researchers described in PRODUCT.md, preserving DESIGN.md. Use a clearly labeled synthetic case: a doctoral researcher reconstructs why a literature-review conclusion changed by following linked notes and citations. No real customer names, quotes, metrics, or assets are available; author illustrative content and label it, with no invented commercial claims. The reader should understand the preserved reasoning trail and follow an in-page link to the method. For a composition or concept choice, use the first direction you presented. No additional material is coming from me.';
export const PRODUCT_MD_SAMPLE = `# Acme Notes
## Platform
+108 -51
View File
@@ -1,9 +1,9 @@
/**
* Sandboxed scenario runner for skill-behavior tests.
* Synthetic-workspace scenario runner for skill-behavior tests.
*
* Each scenario:
* 1. Creates a temp workspace.
* 2. Symlinks the real .claude/skills/impeccable into the workspace so
* 2. Builds a neutral .claude/skills/impeccable into the workspace so
* the launcher (`scripts/impeccable`) resolves from the canonical path
* the skill references, and points it at an engine binary.
* 3. Optionally writes PRODUCT.md / DESIGN.md fixtures.
@@ -15,8 +15,8 @@
* messages (so multi-turn scenarios can append to them).
*
* The harness deliberately mirrors the live-mode E2E pattern: real LLM,
* no mocks, but tightly bounded execution surface so we observe the routing
* behavior of the skill without paying for full-fledged design work.
* no mocked model. File tools are workspace-scoped; bash is a real host shell,
* not a security sandbox. Run only against disposable synthetic fixtures.
*/
import { generateText, stepCountIs, tool } from 'ai';
import { z } from 'zod';
@@ -28,12 +28,35 @@ import { spawn } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { getProviderOptions } from './providers.mjs';
import { ENGINE_MISSING_MESSAGE, findEngineBinary } from '../lib/engine-bin.mjs';
import { readSourceFiles, compileProviderBlocks, replacePlaceholders, stripRuleMarkers } from '../../scripts/lib/utils.js';
import { createTransformer } from '../../scripts/lib/transformers/factory.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(__dirname, '..', '..');
const SKILL_SOURCE_DIR = path.join(REPO_ROOT, 'skill');
const MAX_BASH_OUTPUT_BYTES = 200_000;
function renderNeutral(content) {
return stripRuleMarkers(replacePlaceholders(compileProviderBlocks(content, [])
.replaceAll('{{ask_instruction}}', 'Use the ask_user_question tool.')
.replaceAll('{{model}}', 'the assistant'), 'dsh'))
.replaceAll('{{scripts_path}}', '.claude/skills/impeccable/scripts')
.replaceAll('{{command_hint}}', 'command');
}
// Use the production builder so fallback reviewer/documenter references exist.
// Generic tool names are shared by the API providers; host-specific blocks are
// deliberately absent. Exact provider transforms have separate loader tests.
const sourceSkills = readSourceFiles(REPO_ROOT).skills.map((skill) => ({
...skill,
body: renderNeutral(skill.body),
references: skill.references.map((ref) => ({ ...ref, content: renderNeutral(ref.content) })),
agents: skill.agents.map((agent) => ({ ...agent, body: renderNeutral(agent.body) })),
}));
const stageSkill = createTransformer({
provider: 'skill-behavior', placeholderProvider: 'dsh', providerTags: [],
configDir: '.claude', displayName: 'Behavior fixture',
});
function snapshotWorkspaceFiles(root) {
const snapshot = new Map();
const walk = (dir, relDir = '') => {
@@ -66,24 +89,7 @@ function changedPaths(before, after) {
* is provider-neutral when inlined.
*/
function loadSkillBody() {
let md = fs.readFileSync(path.join(SKILL_SOURCE_DIR, 'SKILL.src.md'), 'utf8');
// Strip frontmatter.
if (md.startsWith('---')) {
const end = md.indexOf('\n---', 3);
if (end !== -1) md = md.slice(end + 4).trimStart();
}
// The source uses placeholders that the build step replaces per-provider.
// For the test harness we want a single body that works for any provider,
// and the scripts the skill references live at .claude/skills/impeccable/
// (the workspace symlink), so hard-code those values.
md = md
.replaceAll('{{model}}', 'the assistant')
.replaceAll('{{command_prefix}}', '/')
.replaceAll('{{ask_instruction}}', 'Use the ask_user_question tool.')
.replaceAll('{{config_file}}', 'AGENTS.md')
.replaceAll('{{scripts_path}}', '.claude/skills/impeccable/scripts')
.replaceAll('{{command_hint}}', 'command');
return md.trim();
return sourceSkills[0].body.trim();
}
// This provider-neutral fixture assumes a loaded skill with a known base
@@ -96,36 +102,26 @@ export const SKILL_BODY = `Base directory for this skill (workspace-relative): .
/**
* Create a temp workspace and prepopulate it.
*
* - `.claude/skills/impeccable` is symlinked at the SOURCE skill dir (not
* the built `.claude/skills/impeccable/`) so the test exercises whatever
* is in `skill/` right now, without needing `bun run build` to refresh
* the harness output dirs. The trade-off: reference files surface their
* raw `{{placeholders}}`, but the assertions only check tool calls, not
* their content.
* - Compile current source into an independent fixture distribution. Shell
* and read tools see the same resolved references, including degraded roles.
* - `files` lets the test seed PRODUCT.md / DESIGN.md (or anything else).
* - `skillVersion` switches from symlink to a real COPY of the skill dir and
* writes a `SKILL.md` carrying that version. `impeccable context` reads its
* - `skillVersion` adds a `SKILL.md` version. `impeccable context` reads its
* own version from that sibling file, so this is required for any scenario
* that exercises the update-check path (the source dir has only SKILL.src.md).
*
* The launcher in the staged scripts dir needs an engine binary. Every bash
* call the agent makes gets `IMPECCABLE_BIN` (tests/lib/engine-bin.mjs:
* `IMPECCABLE_BIN` or `skill/scripts/bin/<os>-<arch>/`), which the launcher
* honors first, so the symlink and copy modes both work without a download.
* honors first, so the staged skill works without a download.
*/
export const ENGINE_BIN = findEngineBinary();
export { ENGINE_MISSING_MESSAGE };
export function prepareWorkspace({ files = {}, skillVersion = null } = {}) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-skill-test-'));
const skillDest = path.join(dir, '.claude', 'skills', 'impeccable');
fs.mkdirSync(path.join(dir, '.claude', 'skills'), { recursive: true });
if (skillVersion) {
fs.cpSync(SKILL_SOURCE_DIR, skillDest, { recursive: true });
fs.writeFileSync(path.join(skillDest, 'SKILL.md'), `---\nname: impeccable\nversion: ${skillVersion}\n---\n\nbody\n`);
} else {
fs.symlinkSync(SKILL_SOURCE_DIR, skillDest, 'dir');
}
stageSkill(sourceSkills, dir, { skillsVersion: skillVersion || '' });
fs.renameSync(path.join(dir, 'skill-behavior', '.claude'), path.join(dir, '.claude'));
fs.rmdirSync(path.join(dir, 'skill-behavior'));
for (const [name, contents] of Object.entries(files)) {
const target = path.join(dir, name);
fs.mkdirSync(path.dirname(target), { recursive: true });
@@ -154,14 +150,43 @@ function safeResolve(root, userPath) {
if (rel.startsWith('..') || rel.split(path.sep).includes('..')) {
return { error: 'path escapes the workspace' };
}
return resolved;
try {
// New write targets need not exist; validate their nearest existing
// ancestor, including dangling links, before appending the missing suffix.
let ancestor = resolved;
while (!fs.existsSync(ancestor)) {
if (fs.lstatSync(ancestor, { throwIfNoEntry: false })?.isSymbolicLink()) {
return { error: 'path follows a dangling symlink' };
}
ancestor = path.dirname(ancestor);
}
const canonical = path.resolve(fs.realpathSync(ancestor), path.relative(ancestor, resolved));
const realRel = path.relative(fs.realpathSync(root), canonical);
if (realRel === '..' || realRel.startsWith(`..${path.sep}`) || path.isAbsolute(realRel)) {
return { error: 'path escapes the workspace through a symlink' };
}
return canonical;
} catch {
return { error: 'path cannot be resolved safely' };
}
}
function isContextOnlyCommand(workspace, command) {
const match = command.trim().match(/^\.claude\/skills\/impeccable\/scripts\/impeccable context(?: --target(?: |=)(?:"([a-zA-Z0-9_./+ -]+)"|'([a-zA-Z0-9_./+ -]+)'|([a-zA-Z0-9_./+-]+)))?$/);
if (!match) return false;
const target = match[1] ?? match[2] ?? match[3];
return target === undefined || (!target.startsWith('-') && typeof safeResolve(workspace, target) === 'string');
}
function execBash(workspace, command, timeoutMs = 20_000, extraEnv = {}) {
return new Promise((resolve) => {
// Model credentials belong to generateText, not to child image helpers.
// Real decision pages have browser E2E; this suite has a structured user.
const shellEnv = Object.fromEntries(Object.entries({ ...process.env, ...extraEnv })
.filter(([name]) => !/(?:^|_)(?:API_KEY|AUTH_TOKEN|ACCESS_TOKEN)$/.test(name)));
const proc = spawn('bash', ['-lc', command], {
cwd: workspace,
env: { ...process.env, ...(ENGINE_BIN ? { IMPECCABLE_BIN: ENGINE_BIN } : {}), ...extraEnv },
env: { ...shellEnv, ...(ENGINE_BIN ? { IMPECCABLE_BIN: ENGINE_BIN } : {}), IMPECCABLE_QUESTION_DISABLED: '1' },
});
let stdout = '';
let stderr = '';
@@ -225,6 +250,10 @@ function defaultSimulatedAnswer(question) {
}
export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contextOnlyBash = false, denyBash = false } = {}) {
const referenceDir = path.join(workspace, '.claude/skills/impeccable/reference');
const references = fs.readdirSync(referenceDir, { recursive: true })
.filter((file) => file.endsWith('.md'))
.map((file) => ({ file: file.split(path.sep).join('/'), content: fs.readFileSync(path.join(referenceDir, file), 'utf8').trim() }));
const trace = {
toolCalls: [],
bashCommands: [],
@@ -248,7 +277,7 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
const tools = {
bash: tool({
description: contextOnlyBash
? 'Only `.claude/skills/impeccable/scripts/impeccable context` is allowed here. Use read/list for workspace files and skill references; write remains available for requested edits.'
? 'Only `.claude/skills/impeccable/scripts/impeccable context` with an optional `--target <workspace-relative path>` is allowed here. Use read/list for files and references; write remains available for requested edits.'
: 'Run a bash command in the workspace root. Use this to invoke skill commands (e.g. `.claude/skills/impeccable/scripts/impeccable context`).',
inputSchema: z.object({
command: z.string().describe('The bash command to execute.'),
@@ -265,14 +294,16 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
}
// Routing tests need the real context loader, not a general-purpose
// shell on the host. Reject before execution (still record attempts).
if (contextOnlyBash && command.trim() !== '.claude/skills/impeccable/scripts/impeccable context') {
const out = 'Error: only `.claude/skills/impeccable/scripts/impeccable context` is allowed. Use read/list for files; references live at .claude/skills/impeccable/reference/.';
if (contextOnlyBash && !isContextOnlyCommand(workspace, command)) {
const out = 'Error: only `.claude/skills/impeccable/scripts/impeccable context` with an optional workspace-relative `--target` is allowed. Use read/list for files; references live at .claude/skills/impeccable/reference/.';
trace.bashOutputs.push(out);
return out;
}
const before = snapshotWorkspaceFiles(workspace);
const res = await execBash(workspace, command, 20_000, extraEnv);
call.mutatedPaths = changedPaths(before, snapshotWorkspaceFiles(workspace));
call.loadedFiles = references.filter(({ content }) => content && res.stdout.includes(content))
.map(({ file }) => `.claude/skills/impeccable/reference/${file}`);
const head = `exit=${res.exitCode}`;
const body = (res.stdout ? `stdout:\n${res.stdout}` : '') + (res.stderr ? `\nstderr:\n${res.stderr}` : '');
const out = `${head}\n${body}${res.truncated ? '\n[output truncated]' : ''}`;
@@ -295,6 +326,7 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
if (stat.isDirectory()) return `Path is a directory: ${p}. Use list instead.`;
const contents = fs.readFileSync(resolved, 'utf8');
call.succeeded = true;
call.loadedFiles = [p];
return contents;
},
}),
@@ -308,7 +340,7 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
const call = record('write', { path: p, contents });
const resolved = safeResolve(workspace, p);
if (typeof resolved !== 'string') return `Error: ${resolved.error}`;
if ((contextOnlyBash || denyBash) && path.relative(workspace, resolved).split(path.sep)[0] === '.claude') {
if (path.relative(fs.realpathSync(workspace), resolved).split(path.sep)[0] === '.claude') {
return 'Error: the staged skill is read-only; edits must target project files.';
}
fs.mkdirSync(path.dirname(resolved), { recursive: true });
@@ -386,12 +418,20 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
// run is never killed. The timer is unref'd (it must not keep the loop alive
// after a healthy turn) and cleared on completion.
const TURN_TIMEOUT_MS = Number(process.env.IMPECCABLE_SKILL_BEHAVIOR_TURN_TIMEOUT_MS) || 840_000;
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8, env = {}, simulatedUser = {}, timeoutMs = TURN_TIMEOUT_MS, contextOnlyBash = false, denyBash = false }) {
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8, env = {}, simulatedUser = {}, timeoutMs = TURN_TIMEOUT_MS, contextOnlyBash = false, denyBash = false, stopAfter, additionalTools, environment = '' }) {
const { tools, trace } = makeTools(workspace, env, simulatedUser, { contextOnlyBash, denyBash });
if (additionalTools) Object.assign(tools, additionalTools(trace));
const messages = [
...priorMessages,
{ role: 'user', content: userPrompt },
];
const traceDir = process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR;
const tracePath = traceDir && path.join(traceDir, `${path.basename(workspace)}-${crypto.randomUUID()}.json`);
const saveTrace = (details) => {
if (!tracePath) return;
fs.mkdirSync(traceDir, { recursive: true });
fs.writeFileSync(tracePath, JSON.stringify({ model: model.modelId, userPrompt, trace, ...details }, null, 2));
};
let result;
const controller = new AbortController();
const timer = setTimeout(
@@ -402,10 +442,14 @@ export async function runTurn({ workspace, model, userPrompt, priorMessages = []
try {
result = await generateText({
model,
system: SKILL_BODY,
system: environment ? `${SKILL_BODY}\n\nRuntime environment: ${environment}` : SKILL_BODY,
messages,
tools,
stopWhen: [stepCountIs(maxSteps)],
onStepFinish: (step) => {
(trace.assistantTexts ??= []).push(step.text ?? '');
saveTrace({ status: 'in-progress', lastStepMessages: step.response.messages });
},
stopWhen: [stepCountIs(maxSteps), ...(stopAfter ? [() => stopAfter(trace)] : [])],
// Real client-side deadline on the provider call: without it a stalled
// stream wedges the whole sweep with no tally.
abortSignal: controller.signal,
@@ -420,18 +464,27 @@ export async function runTurn({ workspace, model, userPrompt, priorMessages = []
});
} catch (err) {
const reason = controller.signal.aborted ? ` (aborted after ${timeoutMs}ms client-side timeout)` : '';
saveTrace({ status: 'failed', error: `${String(err)}${reason}` });
throw new Error(`LLM behavior turn failed before completing${reason}: ${String(err)}`, { cause: err });
} finally {
clearTimeout(timer);
}
const generatedResponseMessages = result.responseMessages ?? result.response?.messages ?? [];
const responseMessages = [...messages, ...generatedResponseMessages];
const outcome = stopAfter?.(trace) ? 'checkpoint'
: result.finishReason === 'length' ? 'output-limit'
: result.finishReason === 'tool-calls' && result.steps.length >= maxSteps ? 'step-budget'
: result.finishReason === 'stop' ? 'complete' : result.finishReason;
saveTrace({ status: 'completed', responseMessages,
outcome, finishReason: result.finishReason, steps: result.steps.length, usage: result.totalUsage ?? result.usage });
return {
trace,
outcome,
steps: result.steps.length,
text: result.text ?? '',
stepTexts: result.steps.map((step) => step.text ?? ''),
finishReason: result.finishReason,
usage: result.usage,
usage: result.totalUsage ?? result.usage,
responseMessages,
};
}
@@ -451,8 +504,12 @@ export function readsMatching(trace, substring) {
* True if the agent loaded a file by Read OR by a bash `cat` (some models
* stream multiple files via bash to save tool calls).
*/
export function callLoadedFile(call, filename) {
return (call.loadedFiles || []).some((file) => file === filename || file.endsWith(`/${filename}`));
}
export function fileLoaded(trace, filename) {
return readsMatching(trace, filename).length > 0 || bashCommandsMatching(trace, filename).length > 0;
return trace.toolCalls.some((call) => callLoadedFile(call, filename));
}
export function summarizeTrace(trace) {
+52 -38
View File
@@ -18,16 +18,18 @@ import path from 'node:path';
import {
prepareWorkspace,
cleanupWorkspace,
runTurn,
runTurn as runHarnessTurn,
bashCommandsMatching,
readsMatching,
fileLoaded,
callLoadedFile,
summarizeTrace,
ENGINE_BIN,
ENGINE_MISSING_MESSAGE,
} from './harness.mjs';
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
import { assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING } from './assertions.mjs';
import { assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs';
import { assertCompleted } from '../skill-workflow/assertions.mjs';
import {
PRODUCT_MD_SAMPLE,
PRODUCT_MD_SAMPLE_NO_REGISTER,
@@ -39,9 +41,22 @@ import {
SVELTE_PROJECT_FILES,
} from './fixtures.mjs';
// Protocol-only shell access; successful checkpoints end observation, not the task.
async function runTurn({ checkpoint, ...options }) {
return runHarnessTurn({ contextOnlyBash: true, timeoutMs: 180000, ...options,
stopAfter: typeof checkpoint === 'function' ? checkpoint
: checkpoint ? (trace) => fileLoaded(trace, checkpoint) : undefined });
}
const CRAFT_PROMPT = '/impeccable craft a landing page for the project in this workspace';
function projectCodeReads(trace) {
return trace.toolCalls.filter((call) => call.name === 'read' && call.succeeded
&& /\.(css|svelte|tsx?|jsx?|astro)$/i.test(call.input.path)
&& !call.input.path.includes('.claude/skills/')).map((call) => call.input.path);
}
const SHAPE_PROMPT = '/impeccable shape a landing page for the project in this workspace';
const NATURAL_BUILD_PROMPT = 'Build a landing page for the project in this workspace.';
const UPDATE_NOTICE = /(?:skill|impeccable).{0,100}(?:update|version)|(?:update|version).{0,100}(?:skill|impeccable)|99\.0\.0/i;
const TEACH_PROMPT = '/impeccable teach';
const PRIMER_PROMPT =
'Take a quick look at the project. What context should guide later design work? Run the impeccable context loader once if you need to.';
@@ -57,14 +72,9 @@ function logTrace(label, scenario, model, trace, extras = {}) {
}
function loadedBeforeImplementationWrite(trace, filename) {
const needle = filename.toLowerCase();
const loadIndex = trace.toolCalls.findIndex(({ name, input }) => {
if (name === 'read') return input?.path?.toLowerCase().includes(needle);
if (name === 'bash') return input?.command?.toLowerCase().includes(needle);
return false;
});
const loadIndex = trace.toolCalls.findIndex((call) => callLoadedFile(call, filename));
const writeIndex = trace.toolCalls.findIndex(
({ name, input }) => name === 'write' && /\.(html?|css|svelte|jsx?|tsx?)$/i.test(input?.path ?? ''),
({ mutatedPaths = [] }) => mutatedPaths.some((file) => /\.(html?|css|svelte|jsx?|tsx?)$/i.test(file)),
);
return loadIndex >= 0 && (writeIndex < 0 || loadIndex < writeIndex);
}
@@ -81,16 +91,6 @@ function executedUpdateCommands(trace) {
);
}
function assertAdviceOnly(trace, text) {
assert.ok(text.trim(), 'advice must reach the user, not stop at reference loading');
assert.deepEqual(trace.writePaths, [], 'advice must not use the write tool');
const mutations = trace.toolCalls.flatMap((call) => call.mutatedPaths ?? [])
.filter((file) => !file.startsWith('.impeccable/') || file.startsWith('.impeccable/critique/'));
assert.deepEqual(mutations, [], 'advice must not edit project files or archive an unsolicited critique');
assert.deepEqual(trace.questionCalls, [], 'advice must not start an init or design interview');
assert.equal(bashCommandsMatching(trace, 'impeccable detect').length, 0, 'workflow advice does not run menu scans');
}
for (const modelId of resolveModelList()) {
const provider = detectProvider(modelId);
const keyPresent = hasKey(provider);
@@ -105,17 +105,14 @@ for (const modelId of resolveModelList()) {
return;
}
const model = getModel(modelId);
// Gemini Flash tends to inspect one file at a time, while the production
// Anthropic/OpenAI models batch setup reads and then begin implementation.
// Keep the latter tightly bounded so this routing suite does not turn into
// a page-generation benchmark, but leave Gemini enough room to reach the
// same required reference.
const setupMaxSteps = provider === 'google' ? 6 : 3;
// Observe a routing decision, with room for setup reads but no full build.
const setupMaxSteps = 10;
it('scenario 1: no PRODUCT.md / DESIGN.md', async () => {
const workspace = prepareWorkspace({ files: {} });
try {
const { trace, text } = await runTurn({
checkpoint: 'init.md',
workspace,
model,
userPrompt: CRAFT_PROMPT,
@@ -149,6 +146,7 @@ for (const modelId of resolveModelList()) {
});
try {
const { trace, text } = await runTurn({
checkpoint: 'new-work.md',
workspace,
model,
userPrompt: CRAFT_PROMPT,
@@ -177,6 +175,7 @@ for (const modelId of resolveModelList()) {
});
try {
const { trace, text } = await runTurn({
checkpoint: 'new-work.md',
workspace,
model,
userPrompt: CRAFT_PROMPT,
@@ -221,6 +220,7 @@ for (const modelId of resolveModelList()) {
// Turn 1: prime the conversation so impeccable context gets run and its
// output enters the message history.
const turn1 = await runTurn({
checkpoint: (trace) => trace.bashOutputs.some((output) => output.startsWith('exit=0\n')),
workspace,
model,
userPrompt: PRIMER_PROMPT,
@@ -236,6 +236,7 @@ for (const modelId of resolveModelList()) {
// Turn 2: the real ask. The skill says "skip if you've already
// loaded it". Verify the agent honors that.
const turn2 = await runTurn({
checkpoint: 'new-work.md',
workspace,
model,
userPrompt: 'Now, /impeccable craft a landing page based on what you saw.',
@@ -261,6 +262,7 @@ for (const modelId of resolveModelList()) {
});
try {
const { trace, text } = await runTurn({
checkpoint: 'new-work.md',
workspace,
model,
userPrompt: CRAFT_PROMPT,
@@ -292,6 +294,7 @@ for (const modelId of resolveModelList()) {
});
try {
const { trace, text } = await runTurn({
checkpoint: 'polish.md',
workspace,
model,
userPrompt: '/impeccable polish index.html',
@@ -318,6 +321,7 @@ for (const modelId of resolveModelList()) {
});
try {
const { trace, text } = await runTurn({
checkpoint: 'audit.md',
workspace,
model,
userPrompt: '/impeccable audit index.html',
@@ -344,6 +348,7 @@ for (const modelId of resolveModelList()) {
});
try {
const { trace, text } = await runTurn({
checkpoint: (trace) => projectCodeReads(trace).length > 0,
workspace,
model,
userPrompt: '/impeccable polish src/routes/+page.svelte',
@@ -354,9 +359,7 @@ for (const modelId of resolveModelList()) {
// agent should read at least one project code file (CSS / tokens /
// component / page), not just the skill's PRODUCT.md / DESIGN.md
// / reference files.
const projectReads = trace.readPaths.filter((p) =>
/\.(css|svelte|tsx?|jsx?|astro)$/i.test(p) && !p.includes('.claude/skills/'),
);
const projectReads = projectCodeReads(trace);
assert.ok(
projectReads.length >= 1,
`agent should read at least one project code file to understand the existing design system.\n` +
@@ -392,6 +395,7 @@ for (const modelId of resolveModelList()) {
userPrompt: '/impeccable polish index.html',
maxSteps: setupMaxSteps,
env: { IMPECCABLE_UPDATE_CACHE: path.join(workspace, '.impeccable-update.json') },
checkpoint: (trace) => trace.assistantTexts?.some((text) => UPDATE_NOTICE.test(text)),
});
logTrace('S9', 'update-available', modelId, trace, { textSample: text.slice(0, 400) });
@@ -408,6 +412,7 @@ for (const modelId of resolveModelList()) {
`bashOutputs: ${JSON.stringify(trace.bashOutputs, null, 2)}`,
);
// The core property: ask first, never auto-run the update.
assert.ok(trace.assistantTexts?.some((text) => UPDATE_NOTICE.test(text)), 'the skill update must be surfaced to the user');
const ranUpdate = executedUpdateCommands(trace);
assert.equal(
ranUpdate.length,
@@ -431,6 +436,7 @@ for (const modelId of resolveModelList()) {
});
try {
const { trace, text } = await runTurn({
checkpoint: 'polish.md',
workspace,
model,
userPrompt: '/impeccable polish index.html',
@@ -469,6 +475,7 @@ for (const modelId of resolveModelList()) {
const workspace = prepareWorkspace({ files: {} });
try {
const { trace, text } = await runTurn({
checkpoint: 'init.md',
workspace,
model,
userPrompt: SHAPE_PROMPT,
@@ -494,6 +501,7 @@ for (const modelId of resolveModelList()) {
const workspace = prepareWorkspace({ files: {} });
try {
const { trace, text } = await runTurn({
checkpoint: 'init.md',
workspace,
model,
userPrompt: NATURAL_BUILD_PROMPT,
@@ -521,6 +529,7 @@ for (const modelId of resolveModelList()) {
const workspace = prepareWorkspace({ files: {} });
try {
const { trace, text } = await runTurn({
checkpoint: 'init.md',
workspace,
model,
userPrompt: TEACH_PROMPT,
@@ -554,6 +563,7 @@ for (const modelId of resolveModelList()) {
});
try {
const { trace, text } = await runTurn({
checkpoint: 'ios.md',
workspace,
model,
userPrompt: '/impeccable craft a tide detail screen for the project in this workspace',
@@ -589,6 +599,7 @@ for (const modelId of resolveModelList()) {
});
try {
const { trace, text } = await runTurn({
checkpoint: 'audit.native.md',
workspace,
model,
userPrompt: '/impeccable audit the app in this workspace',
@@ -614,40 +625,42 @@ for (const modelId of resolveModelList()) {
['existing project', WORKFLOW_ADVICE_FILES],
['missing product context', { 'index.html': MINIMAL_LANDING_HTML }],
]) {
it(`scenario 16: workflow advice stays read-only (${label})`, async () => {
it(`scenario 16: workflow advice stays read-only (${label})`, async (t) => {
const workspace = prepareWorkspace({ files });
try {
const { trace, text } = await runTurn({
const result = await runTurn({
workspace,
model,
userPrompt: "I'm joining this project. Where should I start with Impeccable?",
maxSteps: 8,
contextOnlyBash: true,
});
const { trace, text } = result;
logTrace('S16', label, modelId, trace, { textSample: text.slice(0, 300) });
assert.ok(readsMatching(trace, 'reference/routing.md').length, 'workflow advice loads the shared routing reference');
assertAdviceOnly(trace, text);
t.diagnostic(`Reference coverage gaps (non-blocking): ${missingReferences(trace, ['reference/routing.md']).join(', ') || 'none'}`);
assertCompleted(result);
assertWorkflowAdvice(trace, text, { missingContext: label === 'missing product context' });
} finally {
cleanupWorkspace(workspace);
}
});
}
it('scenario 17: command comparison reads references without running them', async () => {
it('scenario 17: command comparison explains independent commands without running them', async (t) => {
const workspace = prepareWorkspace({ files: WORKFLOW_ADVICE_FILES });
try {
const { trace, text } = await runTurn({
const result = await runTurn({
workspace,
model,
userPrompt: 'Should I use critique or polish on index.html? Is a critique required before polishing?',
maxSteps: 8,
contextOnlyBash: true,
});
const { trace, text } = result;
logTrace('S17', 'command-comparison', modelId, trace, { textSample: text.slice(0, 300) });
assert.ok(readsMatching(trace, 'reference/routing.md').length, 'a command name in a question still routes to advice');
assert.ok(readsMatching(trace, 'reference/critique.md').length, 'comparison consults the critique contract');
assert.ok(readsMatching(trace, 'reference/polish.md').length, 'comparison consults the polish contract');
assertAdviceOnly(trace, text);
t.diagnostic(`Reference coverage gaps (non-blocking): ${missingReferences(trace, ['reference/routing.md', 'reference/critique.md', 'reference/polish.md']).join(', ') || 'none'}`);
assertCompleted(result);
assertCommandComparison(trace, text);
} finally {
cleanupWorkspace(workspace);
}
@@ -730,6 +743,7 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable polish index.html. Please do the polish pass now; afterward tell me which command would be useful next.',
checkpoint: 'polish.md',
maxSteps: 8,
contextOnlyBash: true,
});
+50
View File
@@ -0,0 +1,50 @@
import { it, mock } from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { prepareBrowser, imageOutput } from './skill-workflow/browser.mjs';
import { chromium } from 'playwright';
it('fails browser preflight with an actionable error before starting a workflow', async () => {
const launch = mock.method(chromium, 'launch', async () => { throw new Error('browser missing'); });
try {
await assert.rejects(prepareBrowser('/unused'), /playwright install chromium/);
} finally {
launch.mock.restore();
}
});
it('prepares real browser captures, interactions, and multimodal image results offline', async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-workflow-browser-'));
let browser;
try {
fs.writeFileSync(path.join(root, 'index.html'), '<!doctype html><title>Fixture</title><button onclick="this.textContent=\'Done\'">Start</button><img src="https://example.invalid/image.png">');
browser = await prepareBrowser(root);
const trace = { toolCalls: [] };
const tools = browser.tools(trace);
const capture = await tools.browser_snapshot.execute({ path: './index.html', viewport: 'desktop', click: 'button' });
assert.equal(capture.target, 'index.html');
assert.match(capture.text, /Done/);
assert.ok(fs.existsSync(path.join(root, capture.screenshot)));
assert.equal(capture.viewport, 'desktop');
assert.equal(trace.toolCalls[0].name, 'browser_snapshot');
const output = imageOutput({ output: capture });
assert.equal(output.type, 'content');
assert.ok(output.value.some((part) => part.mediaType === 'image/png'));
const viewed = await tools.view_image.execute({ path: capture.screenshot });
assert.equal(viewed.image, capture.image);
const captures = await Promise.all(['desktop', 'mobile'].map((viewport) => tools.browser_snapshot.execute({ path: 'index.html', viewport })));
assert.deepEqual(captures.map((result) => result.viewport), ['desktop', 'mobile']);
assert.notEqual(captures[0].image, captures[1].image, 'parallel viewports must not share mutable page state');
assert.ok(browser.blockedRequests.some((url) => url.includes('example.invalid')));
await assert.rejects(tools.browser_snapshot.execute({ path: '../outside.html', viewport: 'mobile' }), /workspace/);
await assert.rejects(tools.view_image.execute({ path: 'index.html' }), /PNG/);
fs.symlinkSync(os.tmpdir(), path.join(root, 'escape'));
const response = await fetch(`${browser.origin}/escape/`);
assert.equal(response.status, 403);
} finally {
await browser?.close();
fs.rmSync(root, { recursive: true, force: true });
}
});
+50
View File
@@ -0,0 +1,50 @@
import assert from 'node:assert/strict';
import { sourceHash } from './source-hash.mjs';
import { missingReferences } from '../skill-behavior/assertions.mjs';
export function assertDocumentationArtifacts(design, sidecarText) {
const frontmatter = design.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/)?.[1];
assert.ok(frontmatter, 'documentation must include machine-readable frontmatter, not prose alone');
assert.match(frontmatter, /^colors:\s*\n[ \t]+\S/m, 'documentation must record color tokens');
assert.match(frontmatter, /^typography:\s*\n[ \t]+\S/m, 'documentation must record typography tokens');
const sidecar = JSON.parse(sidecarText);
assert.equal(sidecar.schemaVersion, 2, 'documentation must write the v2 sidecar');
for (const key of ['extensions', 'narrative']) {
assert.ok(sidecar[key] && typeof sidecar[key] === 'object' && !Array.isArray(sidecar[key])
&& Object.keys(sidecar[key]).length, `sidecar must contain ${key} metadata`);
}
}
// For a resumed, already-reviewed ordinary extension only. New worlds and
// redesigns still owe real documentation writes; this is not an escape hatch.
export function assertNoChangeDocumentation(result, { target, evidence }) {
assertCompleted(result);
const { trace, text } = result;
assert.deepEqual(missingReferences(trace, ['reference/document.md', target, 'DESIGN.md']), [],
'documentation must consult its contract and inspect the actual source and recorded system');
assert.deepEqual(trace.toolCalls.flatMap((call) => call.mutatedPaths || []), [],
'the resumed no-change check must not mutate project files');
assert.match(text, /no (?:system |visual.system |documentation )?changes|unchanged|no rewrite/i,
'documentation must explicitly report a no-change outcome');
for (const filename of [target, 'DESIGN.md']) {
assert.ok(text.includes(filename), `documentation must identify the checked ${filename}`);
}
for (const fact of evidence) {
assert.match(text, fact, 'no-change documentation must report evidence from the fixture, not an unsupported completion claim');
}
}
export function assertCompleted(result) {
assert.equal(result.outcome, 'complete', `workflow did not finish: ${result.outcome} after ${result.steps} steps`);
}
export function assertFreshCaptures(trace, workspace, target) {
const calls = trace.toolCalls;
const lastEdit = calls.findLastIndex((call) => (call.mutatedPaths || []).includes(target));
const hash = sourceHash(workspace);
for (const viewport of ['desktop', 'mobile']) {
assert.ok(calls.some((call, index) => index > lastEdit && call.capture?.target === target
&& call.capture.viewport === viewport && call.capture.sourceHash === hash),
`missing ${viewport} screenshot of the final ${target}; pre-edit captures do not count`);
}
}
+131
View File
@@ -0,0 +1,131 @@
import fs from 'node:fs';
import path from 'node:path';
import http from 'node:http';
import { sourceHash as hashSources } from './source-hash.mjs';
import { chromium } from 'playwright';
import { tool } from 'ai';
import { z } from 'zod';
const VIEWPORTS = { desktop: { width: 1440, height: 1000 }, mobile: { width: 390, height: 844 } };
const TYPES = { '.html': 'text/html', '.css': 'text/css', '.js': 'text/javascript', '.mjs': 'text/javascript', '.svg': 'image/svg+xml', '.png': 'image/png', '.woff2': 'font/woff2' };
const PNG = Buffer.from('89504e470d0a1a0a', 'hex');
function resolveFile(root, name) {
if (path.isAbsolute(name)) throw new Error('Use a workspace-relative path');
const file = path.resolve(root, name);
const rel = path.relative(root, file);
if (rel === '..' || rel.startsWith(`..${path.sep}`)) throw new Error('Path escapes workspace');
const real = fs.realpathSync(file);
const realRel = path.relative(fs.realpathSync(root), real);
if (realRel === '..' || realRel.startsWith(`..${path.sep}`)) throw new Error('Path escapes workspace through a symlink');
return real;
}
export function imageOutput({ output }) {
const { image, ...metadata } = output;
return { type: 'content', value: [
{ type: 'text', text: JSON.stringify(metadata) },
{ type: 'file', mediaType: 'image/png', data: { type: 'data', data: image } },
] };
}
/** Preflight before any billed call; no runtime installs or browser discovery. */
export async function prepareBrowser(root) {
let browser;
try {
browser = await chromium.launch({ headless: true, timeout: 15000 });
} catch (error) {
throw new Error('Workflow browser preflight failed. Run `bunx playwright install chromium` before billed tests.', { cause: error });
}
const blockedRequests = [];
const server = http.createServer((req, res) => {
try {
const name = decodeURIComponent(new URL(req.url, 'http://localhost').pathname).replace(/^\//, '') || 'index.html';
if (name.split('/').some((part) => part.startsWith('.'))) throw new Error('Private workspace path');
const file = resolveFile(root, name);
if (!fs.statSync(file).isFile()) throw new Error('Not a file');
res.writeHead(200, { 'Content-Type': TYPES[path.extname(file)] || 'application/octet-stream', 'Cache-Control': 'no-store' });
res.end(fs.readFileSync(file));
} catch (error) {
res.writeHead(error.code === 'ENOENT' ? 404 : 403);
res.end('Not available');
}
});
try {
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); });
} catch (error) {
await browser.close();
throw error;
}
const origin = `http://127.0.0.1:${server.address().port}`;
let context;
try {
context = await browser.newContext({ reducedMotion: 'reduce', serviceWorkers: 'block' });
await context.route('**/*', (route) => {
const url = route.request().url();
if (new URL(url).origin === origin || url.startsWith('data:')) return route.continue();
blockedRequests.push(url);
return route.abort();
});
} catch (error) {
await browser.close();
await new Promise((resolve) => { server.close(resolve); server.closeAllConnections(); });
throw error;
}
return {
origin, blockedRequests,
environment: `Workspace: ${root}. A local server and Chromium are already running. browser_snapshot renders a workspace-relative HTML path at desktop/mobile size, saves a screenshot, and returns the actual image plus DOM text. view_image opens saved PNGs. No browser installation is needed. External browser requests are blocked; this text-only fixture uses system fonts. No image-generation or subagent tools are available.`,
tools(trace) {
return {
browser_snapshot: tool({
description: 'Render and inspect an HTML file with the ready Chromium browser. Returns an actual screenshot and visible DOM text; optionally click a CSS selector before capture. Captures save to .impeccable/review/{desktop|mobile}.png.',
inputSchema: z.object({ path: z.string(), viewport: z.enum(['desktop', 'mobile']), click: z.string().optional() }),
execute: async ({ path: target, viewport, click }) => {
const file = resolveFile(root, target);
if (!/\.html?$/i.test(file)) throw new Error('Expected an HTML artifact');
const relativeTarget = path.relative(fs.realpathSync(root), file).split(path.sep).join('/');
const call = { name: 'browser_snapshot', input: { path: target, viewport, click }, mutatedPaths: [] };
trace.toolCalls.push(call);
const page = await context.newPage();
page.setDefaultTimeout(10000);
try {
const sourceHash = hashSources(root);
await page.setViewportSize(VIEWPORTS[viewport]);
await page.goto(`${origin}/${relativeTarget.split('/').map(encodeURIComponent).join('/')}`, { waitUntil: 'load', timeout: 15000 });
await page.evaluate(() => document.fonts.ready);
if (click) await page.locator(click).click();
const screenshot = `.impeccable/review/${viewport}.png`;
if (fs.existsSync(path.join(root, '.impeccable'))) resolveFile(root, '.impeccable');
fs.mkdirSync(path.join(root, '.impeccable/review'), { recursive: true });
resolveFile(root, '.impeccable/review');
if (fs.existsSync(path.join(root, screenshot))) resolveFile(root, screenshot);
const image = await page.screenshot({ path: path.join(root, screenshot), fullPage: true, animations: 'disabled' });
call.mutatedPaths = [screenshot];
if (sourceHash !== hashSources(root)) throw new Error('Artifact changed during capture; retry');
call.capture = { target: relativeTarget, viewport, screenshot, sourceHash };
return { ...call.capture, text: (await page.locator('body').innerText()).slice(0, 12000), image: image.toString('base64') };
} finally {
await page.close();
}
},
toModelOutput: imageOutput,
}),
view_image: tool({
description: 'Inspect an existing workspace PNG as an actual image, not raw file bytes.',
inputSchema: z.object({ path: z.string() }),
execute: async ({ path: name }) => {
const bytes = fs.readFileSync(resolveFile(root, name));
if (!bytes.subarray(0, 8).equals(PNG)) throw new Error('Expected a PNG image');
trace.toolCalls.push({ name: 'view_image', input: { path: name }, mutatedPaths: [], loadedImages: [name] });
return { path: name, image: bytes.toString('base64') };
},
toModelOutput: imageOutput,
}),
};
},
async close() {
await browser.close();
await new Promise((resolve) => { server.close(resolve); server.closeAllConnections(); });
},
};
}
@@ -0,0 +1,93 @@
import { it } from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { prepareWorkspace, cleanupWorkspace, runTurn, fileLoaded, ENGINE_BIN } from '../skill-behavior/harness.mjs';
import { getModel, detectProvider, hasKey } from '../skill-behavior/providers.mjs';
import { assertCompleted, assertNoChangeDocumentation, assertDocumentationArtifacts } from './assertions.mjs';
import { missingReferences } from '../skill-behavior/assertions.mjs';
// A synthetic post-review checkpoint, not another full-build simulation.
// The page and system agree. A missing sidecar predates this task and is not
// permission to repair drift or rewrite the incumbent DESIGN.md.
const DESIGN = `# Field Manual
## Overview
An established, plain reading surface. Preserve this identity.
## Colors
White background (#ffffff), near-black text (#222222), blue links (#0645ad).
## Typography
System-ui body at 16px, line-height 1.6. Headings at 24px, weight 700.
## Layout
One column, max-width 65ch, padding 24px; no decorative containers.
`;
const PAGE = '<!doctype html><html lang="en"><meta charset="utf-8"><title>Keyboard guide</title><style>body{background:#fff;color:#222;font:16px/1.6 system-ui;max-width:65ch;margin:auto;padding:24px}h1{font-size:24px;font-weight:700}a{color:#0645ad}</style><main><h1>Keyboard guide</h1><p>Use Tab to move between controls. Press Enter to activate a link.</p><a href="#top" id="top">Back to top</a></main></html>';
const BRIEF = '# Keyboard guide\n\n## Direction contract\nTHESIS: A short reading page.\nOWN-WORLD: Inherit Field Manual.\nSTORY: Read keyboard instructions.\nFIRST VIEWPORT: Title, paragraph, link.\nFORM: Direct, precisely specified page; no seed required.\nFINISH: unreviewed and undocumented is unfinished.\n';
for (const modelId of (process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS || 'claude-sonnet-5').split(',').map((id) => id.trim()).filter(Boolean)) {
for (const mode of ['extension', 'new world', 'redesign']) {
const existingSystem = mode !== 'new world';
const preserveSystem = mode === 'extension';
it(`post-review ${mode} ${preserveSystem ? 'preserves' : 'records'} its system :: ${modelId}`,
{ skip: !ENGINE_BIN || !hasKey(detectProvider(modelId)) }, async (t) => {
const files = {
'PRODUCT.md': '# Field Manual\n\n## Platform\nweb\n\nA reference guide for keyboard users.\n',
...(existingSystem ? { 'DESIGN.md': preserveSystem ? DESIGN : '# Old Field Manual\n\nBeige cards, serif body type, orange links.\n' } : {}),
'index.html': PAGE,
'.impeccable/surfaces/index-html.md': mode === 'redesign'
? BRIEF.replace('Inherit Field Manual.', 'Approved replacement: white, blue links, system-ui, single column.') : BRIEF,
};
const workspace = prepareWorkspace({ files });
try {
const reference = fs.readFileSync(path.join(workspace, '.claude/skills/impeccable/reference/new-work.md'), 'utf8');
const result = await runTurn({
workspace, model: getModel(modelId), maxSteps: 12, timeoutMs: 180000, contextOnlyBash: true,
environment: 'This is a resumed post-review checkpoint. No subagent or browser tools are available. The review is closed; no further UI edits or screenshots are needed. Read/list/write tools are available.',
priorMessages: [
{ role: 'user', content: preserveSystem
? 'Use /impeccable to add the specified keyboard guide page inside the established Field Manual world. Keep the existing visual system. Do not repair unrelated project drift.'
: mode === 'redesign'
? 'Use /impeccable to redesign the keyboard guide. I approve replacing the old beige-card/serif/orange world with the plain single-column, system-font, white-background and blue-link identity. Update the system documentation from the finished page.'
: 'Use /impeccable to create Field Manuals first keyboard guide page. The chosen identity is plain, single-column, system fonts, white background and blue links.' },
{ role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'load-new-work', toolName: 'read', input: { path: '.claude/skills/impeccable/reference/new-work.md' } }] },
{ role: 'tool', content: [{ type: 'tool-result', toolCallId: 'load-new-work', toolName: 'read', output: { type: 'text', value: reference } }] },
{ role: 'assistant', content: `Checkpoint: context and PRODUCT.md were loaded. The user confirmed the exact page and identity. The surface brief and index.html are written. Desktop/mobile captures were validated, the detector ran once, and the shipped finish reviewer returned ship with no open findings. ${preserveSystem
? 'DESIGN.md was loaded. No durable system changes were requested or introduced. The pre-existing missing .impeccable/design.json was reported but not repaired.'
: mode === 'redesign'
? 'The approved replacement world is implemented in index.html. DESIGN.md still describes the superseded identity; no design sidecar exists yet.'
: 'This is the first completed surface of the approved new world. No DESIGN.md or design sidecar exists yet.'}` },
],
userPrompt: 'Continue from this checkpoint and finish the task.',
});
assertCompleted(result);
t.diagnostic(`Documentation wrapper coverage gaps (diagnostic; contract and artifacts remain required): ${missingReferences(result.trace, ['degraded/documenter.md']).join(', ') || 'none'}`);
assert.ok(fileLoaded(result.trace, 'reference/document.md'), 'must consult the documentation contract');
for (const name of existingSystem ? ['index.html', 'DESIGN.md'] : ['index.html']) {
assert.ok(fileLoaded(result.trace, name), `documentation must check ${name}, not merely announce a no-op`);
}
for (const [name, contents] of Object.entries(files)) {
if (mode === 'redesign' && name === 'DESIGN.md') continue;
assert.equal(fs.readFileSync(path.join(workspace, name), 'utf8'), contents, `${name} must remain unchanged`);
}
if (preserveSystem) {
assertNoChangeDocumentation(result, { target: 'index.html', evidence: [/system-ui/i, /65\s*ch/i, /#0645ad/i] });
assert.equal(fs.existsSync(path.join(workspace, '.impeccable/design.json')), false, 'must not repair pre-existing sidecar drift unasked');
assert.deepEqual(result.trace.toolCalls.flatMap((call) => call.mutatedPaths || []), [], 'a no-change check must not mutate other project files');
} else {
const design = fs.readFileSync(path.join(workspace, 'DESIGN.md'), 'utf8');
if (mode === 'redesign') assert.notEqual(design, files['DESIGN.md'], 'approved redesign must replace the old system');
assertDocumentationArtifacts(design, fs.readFileSync(path.join(workspace, '.impeccable/design.json'), 'utf8'));
assert.match(design, /system-ui/);
const writes = result.trace.toolCalls.flatMap((call) => call.mutatedPaths || []);
assert.ok(writes.includes('DESIGN.md') && writes.includes('.impeccable/design.json'), 'both documentation artifacts must be written');
assert.deepEqual(writes.filter((file) => !['DESIGN.md', '.impeccable/design.json'].includes(file)), [], 'documentation must stay inside its write boundary');
}
} finally {
cleanupWorkspace(workspace);
}
});
}
}
@@ -10,14 +10,39 @@ import path from 'node:path';
import {
prepareWorkspace,
cleanupWorkspace,
runTurn,
runTurn as runHarnessTurn,
fileLoaded,
summarizeTrace,
ENGINE_BIN,
ENGINE_MISSING_MESSAGE,
} from './harness.mjs';
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
import { PRODUCT_MD_SAMPLE, DESIGN_MD_SAMPLE } from './fixtures.mjs';
} from '../skill-behavior/harness.mjs';
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from '../skill-behavior/providers.mjs';
import { assertNewWorkLifecycle } from '../skill-behavior/assertions.mjs';
import { PRODUCT_MD_SAMPLE, DESIGN_MD_SAMPLE as ORIGINAL_DESIGN, CASE_STUDY_ANSWER } from '../skill-behavior/fixtures.mjs';
import { prepareBrowser } from './browser.mjs';
import { assertCompleted, assertFreshCaptures, assertDocumentationArtifacts } from './assertions.mjs';
const DESIGN_MD_SAMPLE = ORIGINAL_DESIGN.replace(/GT Sectra \(commercial\)/g, 'Georgia (system)').replace(/JetBrains Mono/g, 'monospace').replace(/Inter/g, 'Arial');
async function runTurn(options) {
// Preflight happens before the first provider call. These are text-only
// HTML fixtures: no dependencies, font downloads, or browser discovery.
const browser = await prepareBrowser(options.workspace);
try {
const result = await runHarnessTurn({
...options, maxSteps: 50, timeoutMs: 840000,
userPrompt: `${options.userPrompt}\nUse system fonts and no external assets for this text-only fixture. The browser_snapshot and view_image tools are ready for visual review.`,
environment: browser.environment,
additionalTools: (trace) => browser.tools(trace),
});
assertCompleted(result);
const contextCalls = result.trace.toolCalls.filter(({ name, input }) => name === 'bash' && /impeccable\s+context\b/.test(input.command));
assert.equal(contextCalls.length, 1, 'completed workflow must load context exactly once');
return result;
} finally {
await browser.close();
}
}
const LEGACY_DESIGN = `# Design
@@ -100,7 +125,9 @@ function workflowTraceMessage(trace) {
return JSON.stringify(summarizeTrace(trace), null, 2);
}
for (const modelId of resolveModelList()) {
// Full builds are separately opt-in and default to one provider. The existing
// model selection variable can explicitly request a cross-provider sweep.
for (const modelId of process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS ? resolveModelList() : ['claude-sonnet-5']) {
const provider = detectProvider(modelId);
const keyPresent = hasKey(provider);
@@ -122,7 +149,6 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable init for a harbor operations product, then finish setup.',
maxSteps: 24,
});
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
const productWrite = firstMutation(trace, /(^|\/)PRODUCT\.md$/i);
@@ -147,20 +173,22 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable create a concise evidence-led case-study page. Leave it at index.html.',
maxSteps: 22,
simulatedUser: { answer: () => CASE_STUDY_ANSWER },
});
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
const implementation = firstMutation(trace, /\.(?:html?|astro|svelte|jsx?|tsx?)$/i);
assert.ok(fileLoaded(trace, 'new-work.md'), `new-work.md was not loaded.\n${workflowTraceMessage(trace)}`);
assert.ok(question >= 0, `task concept was never put to the user.\n${workflowTraceMessage(trace)}`);
assert.ok(implementation > question, `implementation began before the attended concept checkpoint.\n${workflowTraceMessage(trace)}`);
assertNewWorkLifecycle(trace, { target: 'index.html' });
assertFreshCaptures(trace, workspace, 'index.html');
assert.ok(fileLoaded(trace, 'finish-reviewer.md'), 'new-work must run the shipped finish review');
assert.ok(fileLoaded(trace, 'documenter.md'), 'new-work must run the shipped documentation pass');
assert.equal(fs.existsSync(path.join(workspace, 'index.html')), true, 'new-work must still produce the requested artifact');
} finally {
cleanupWorkspace(workspace);
}
});
it('redesign replaces DESIGN before touching the existing page', async () => {
it('redesign approves and records the direction before code, then documents the built world', async () => {
const workspace = prepareWorkspace({
files: {
'PRODUCT.md': PRODUCT_MD_SAMPLE,
@@ -173,17 +201,17 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable redesign current.html for this product. Leave the result at current.html.',
maxSteps: 26,
});
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
const designWrite = firstMutation(trace, /(^|\/)DESIGN\.md$/i);
const implementation = firstMutation(trace, /(^|\/)current\.html$/i);
assert.ok(fileLoaded(trace, 'new-work.md'), `redesign did not route through new-work.\n${workflowTraceMessage(trace)}`);
assert.ok(question >= 0, `replacement world was not put to the user.\n${workflowTraceMessage(trace)}`);
assert.ok(designWrite > question, `replacement DESIGN.md must follow user choice.\n${workflowTraceMessage(trace)}`);
assert.ok(implementation > designWrite, `redesign touched the page before replacing DESIGN.md.\n${workflowTraceMessage(trace)}`);
assertNewWorkLifecycle(trace, { target: 'current.html', redesign: true });
assertFreshCaptures(trace, workspace, 'current.html');
assert.ok(fileLoaded(trace, 'finish-reviewer.md'), 'redesign must run the shipped finish review');
assert.ok(fileLoaded(trace, 'documenter.md'), 'redesign must run the shipped documentation pass');
const design = fs.readFileSync(path.join(workspace, 'DESIGN.md'), 'utf8');
assert.notEqual(design.trim(), LEGACY_DESIGN.trim(), 'redesign preserved the old visual world verbatim');
assertDocumentationArtifacts(design, fs.readFileSync(path.join(workspace, '.impeccable/design.json'), 'utf8'));
} finally {
cleanupWorkspace(workspace);
}
@@ -202,7 +230,6 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable bolder current.html, only the #case-study section. Keep everything else untouched.',
maxSteps: 16,
});
const productWrite = firstMutation(trace, /(^|\/)PRODUCT\.md$/i);
const designWrite = firstMutation(trace, /(^|\/)DESIGN\.md$/i);
@@ -211,6 +238,7 @@ for (const modelId of resolveModelList()) {
assert.equal(productWrite, -1, `refinement rewrote PRODUCT.md.\n${workflowTraceMessage(trace)}`);
assert.equal(designWrite, -1, `refinement rewrote DESIGN.md.\n${workflowTraceMessage(trace)}`);
assert.ok(implementation >= 0, `refinement did not write current.html.\n${workflowTraceMessage(trace)}`);
assertFreshCaptures(trace, workspace, 'current.html');
const artifact = fs.readFileSync(path.join(workspace, 'current.html'), 'utf8');
assert.match(artifact, /data-untouched="header"/);
assert.match(artifact, /data-untouched="footer"/);
@@ -238,9 +266,9 @@ for (const modelId of resolveModelList()) {
workspace,
model,
userPrompt: '/impeccable critique current.html',
maxSteps: 30,
});
assert.ok(fileLoaded(trace, 'critique.md'), `critique.md was not loaded.\n${workflowTraceMessage(trace)}`);
assertFreshCaptures(trace, workspace, 'current.html');
const parts = assistantParts(responseMessages);
const allText = parts.filter((p) => p.kind === 'text').map((p) => p.value).join('\n');
+22
View File
@@ -0,0 +1,22 @@
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
// Include local styles/scripts/assets too: an unchanged HTML file is not
// evidence of a current capture when an external stylesheet changed.
export function sourceHash(root) {
const hash = crypto.createHash('sha256');
function visit(directory, prefix = '') {
for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
if (entry.name.startsWith('.') || entry.name === 'node_modules') continue;
const relative = `${prefix}${entry.name}`;
const file = path.join(directory, entry.name);
if (entry.isDirectory()) visit(file, `${relative}/`);
else if (entry.isFile() && /\.(html?|css|m?js|svg|png|jpe?g|webp|woff2?)$/i.test(entry.name)) {
hash.update(relative).update('\0').update(fs.readFileSync(file)).update('\0');
}
}
}
visit(root);
return hash.digest('hex');
}
+7
View File
@@ -12,6 +12,13 @@ import {
} from '../scripts/test-suites.mjs';
describe('test suite registry', () => {
it('separates protocol checkpoints from opt-in browser-backed completion', () => {
assert.deepEqual(suiteFiles(['skill-behavior']), ['tests/skill-behavior/scenarios.test.mjs']);
assert.ok(OPT_IN_SUITES.includes('skill-workflow'));
assert.equal(SUITES['skill-workflow'].needsPlaywright, true);
assert.ok(suiteFiles(['skill-workflow']).includes('tests/skill-workflow/full-build.test.mjs'));
assert.equal(DEFAULT_SUITES.includes('skill-workflow'), false);
});
it('assigns every test file to a default or opt-in suite', () => {
const allDiscovered = findTestFiles();
const allRegistered = new Set(suiteFiles([...DEFAULT_SUITES, ...OPT_IN_SUITES]));