mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-11 21:57:14 +03:00
Fix skill workflow regression coverage (#783)
Clarify launcher fallback and completed documentation handoffs; separate bounded protocol checkpoints from opt-in browser-backed completion diagnostics. Correct fixture containment, target syntax, and artifact assertions. AI assistance: Codex, under maintainer direction.
This commit is contained in:
@@ -6,6 +6,11 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
skill_workflow:
|
||||||
|
description: 'Run billed, browser-backed Claude workflow completion tests'
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
# Nightly full live-e2e matrix. The smoke groups already gate every PR; the
|
# Nightly full live-e2e matrix. The smoke groups already gate every PR; the
|
||||||
# full sweep is too slow for that, so it runs once a day against main.
|
# full sweep is too slow for that, so it runs once a day against main.
|
||||||
schedule:
|
schedule:
|
||||||
@@ -665,5 +670,47 @@ jobs:
|
|||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: bun install
|
run: bun install
|
||||||
|
|
||||||
|
- name: Prepare engine for protocol tests
|
||||||
|
run: bun run fetch:engine
|
||||||
|
|
||||||
- name: Run skill behavior tests
|
- name: Run skill behavior tests
|
||||||
run: bun run test:skill-behavior
|
run: bun run test:skill-behavior
|
||||||
|
|
||||||
|
skill-workflow:
|
||||||
|
# Expensive, manually opted-in completion diagnostics. This does not
|
||||||
|
# replace the multi-family skill-behavior protocol suite above.
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.event_name == 'workflow_dispatch' && inputs.skill_workflow
|
||||||
|
timeout-minutes: 70
|
||||||
|
env:
|
||||||
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||||
|
IMPECCABLE_SKILL_BEHAVIOR_MODELS: claude-sonnet-5
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
|
- name: Setup Node
|
||||||
|
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||||
|
with:
|
||||||
|
node-version: 24
|
||||||
|
- name: Setup Bun
|
||||||
|
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||||
|
with:
|
||||||
|
bun-version: latest
|
||||||
|
- name: Install dependencies
|
||||||
|
run: bun install --frozen-lockfile
|
||||||
|
- name: Prepare engine and browser before billing
|
||||||
|
run: |
|
||||||
|
test -n "$ANTHROPIC_API_KEY" || { echo 'ANTHROPIC_API_KEY is required'; exit 1; }
|
||||||
|
bun run fetch:engine
|
||||||
|
bunx playwright install --with-deps chromium
|
||||||
|
- name: Run completed skill workflows
|
||||||
|
env:
|
||||||
|
IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR: ${{ runner.temp }}/skill-workflow-traces
|
||||||
|
run: bun run test:skill-workflow
|
||||||
|
- name: Retain diagnostic traces
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||||
|
with:
|
||||||
|
name: skill-workflow-traces
|
||||||
|
path: ${{ runner.temp }}/skill-workflow-traces
|
||||||
|
retention-days: 7
|
||||||
|
|||||||
@@ -59,6 +59,7 @@
|
|||||||
"test:new-work-e2e": "node scripts/run-tests.mjs new-work-e2e",
|
"test:new-work-e2e": "node scripts/run-tests.mjs new-work-e2e",
|
||||||
"test:live-e2e-agent": "node scripts/run-tests.mjs live-e2e-agent",
|
"test:live-e2e-agent": "node scripts/run-tests.mjs live-e2e-agent",
|
||||||
"test:skill-behavior": "node scripts/run-tests.mjs skill-behavior",
|
"test:skill-behavior": "node scripts/run-tests.mjs skill-behavior",
|
||||||
|
"test:skill-workflow": "node scripts/run-tests.mjs skill-workflow",
|
||||||
"test:live-svelte-adapter-deepseek": "node scripts/run-tests.mjs live-svelte-adapter-deepseek",
|
"test:live-svelte-adapter-deepseek": "node scripts/run-tests.mjs live-svelte-adapter-deepseek",
|
||||||
"smoke:hooks": "node scripts/smoke-provider-hooks.mjs",
|
"smoke:hooks": "node scripts/smoke-provider-hooks.mjs",
|
||||||
"audit": "bun audit --audit-level=moderate",
|
"audit": "bun audit --audit-level=moderate",
|
||||||
|
|||||||
+27
-20
@@ -8,6 +8,7 @@ export const OPT_IN_SUITES = [
|
|||||||
'live-e2e-accept-cleanup',
|
'live-e2e-accept-cleanup',
|
||||||
'new-work-e2e',
|
'new-work-e2e',
|
||||||
'skill-behavior',
|
'skill-behavior',
|
||||||
|
'skill-workflow',
|
||||||
'live-svelte-adapter-deepseek',
|
'live-svelte-adapter-deepseek',
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -267,38 +268,44 @@ export const SUITES = {
|
|||||||
],
|
],
|
||||||
},
|
},
|
||||||
'skill-behavior': {
|
'skill-behavior': {
|
||||||
description: 'LLM-backed skill setup behavior scenarios.',
|
description: 'LLM-backed protocol checkpoints, not full builds.',
|
||||||
optIn: true,
|
optIn: true,
|
||||||
triggers: [
|
triggers: [
|
||||||
...COMMON_INFRA_PATTERNS,
|
...COMMON_INFRA_PATTERNS,
|
||||||
/^skill\/SKILL\.src\.md$/,
|
/^skill\/SKILL\.src\.md$/,
|
||||||
/^skill\/reference\/(init|document|brand|product|shape|craft|audit|polish|live|routing)\.md$/,
|
/^skill\/reference\//,
|
||||||
/^ENGINE_VERSION$/,
|
/^ENGINE_VERSION$/,
|
||||||
/^tests\/skill-behavior\//,
|
/^tests\/skill-behavior\//,
|
||||||
],
|
],
|
||||||
|
commands: [{
|
||||||
|
runner: 'node',
|
||||||
|
timeoutMs: 240000,
|
||||||
|
wallClockMs: 1_800_000,
|
||||||
|
files: ['tests/skill-behavior/scenarios.test.mjs'],
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
'skill-workflow': {
|
||||||
|
description: 'Explicitly opt-in completed workflows with a preflighted browser.',
|
||||||
|
optIn: true,
|
||||||
|
needsPlaywright: true,
|
||||||
|
triggers: [
|
||||||
|
...COMMON_INFRA_PATTERNS,
|
||||||
|
/^skill\//,
|
||||||
|
/^ENGINE_VERSION$/,
|
||||||
|
/^tests\/skill-workflow\//,
|
||||||
|
/^tests\/skill-behavior\//,
|
||||||
|
],
|
||||||
commands: [
|
commands: [
|
||||||
|
{ runner: 'node', files: ['tests/skill-workflow-browser.test.mjs'] },
|
||||||
|
{
|
||||||
|
runner: 'node', timeoutMs: 240000, wallClockMs: 600000,
|
||||||
|
files: ['tests/skill-workflow/finish-handoff.test.mjs'],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
runner: 'node',
|
runner: 'node',
|
||||||
// 300000 was too low to measure what these scenarios assert. The
|
|
||||||
// workflow-contract turns run 20+ steps against a frontier model, and
|
|
||||||
// the *correct* path is the slow one: a run that stops to put the
|
|
||||||
// concept to the user before building was measured at 579s, while the
|
|
||||||
// runs that skipped that checkpoint and failed the assertion finished
|
|
||||||
// in 130-200s. At a 300s cap the thorough path is killed and the hasty
|
|
||||||
// path is graded, so the cap was selecting for the behavior the suite
|
|
||||||
// exists to forbid.
|
|
||||||
timeoutMs: 900000,
|
timeoutMs: 900000,
|
||||||
// Overall wall-clock safety cap for the whole sweep: if a provider
|
|
||||||
// call wedges past every inner guard (the harness's 840s per-turn
|
|
||||||
// AbortSignal and the 900s per-test timeout), the runner SIGKILLs the
|
|
||||||
// process group so the sweep still ends with a per-provider tally
|
|
||||||
// instead of hanging overnight. Sized well above a healthy two-provider
|
|
||||||
// sweep; override with IMPECCABLE_TEST_WALL_CLOCK_MS to scope it down.
|
|
||||||
wallClockMs: 3_600_000,
|
wallClockMs: 3_600_000,
|
||||||
files: [
|
files: ['tests/skill-workflow/full-build.test.mjs'],
|
||||||
'tests/skill-behavior/scenarios.test.mjs',
|
|
||||||
'tests/skill-behavior/workflow-contract.test.mjs',
|
|
||||||
],
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
|||||||
+1
-1
@@ -22,7 +22,7 @@ Core principles:
|
|||||||
2. Load the request's playbook: its Commands-table reference for an explicit/implied sub-command, or [reference/new-work.md](reference/new-work.md) for a new surface or replacement visual world. Inspect target and incumbent visual truth before editing. When the app cannot run, start with committed visual-regression goldens or screenshot fixtures; verify target and freshness against current tokens, CSS, components, or assets, resolve conflicts, and compare theme/variant captures. <!-- rule:skill-setup-command-ref --> <!-- rule:skill-setup-read-project -->
|
2. Load the request's playbook: its Commands-table reference for an explicit/implied sub-command, or [reference/new-work.md](reference/new-work.md) for a new surface or replacement visual world. Inspect target and incumbent visual truth before editing. When the app cannot run, start with committed visual-regression goldens or screenshot fixtures; verify target and freshness against current tokens, CSS, components, or assets, resolve conflicts, and compare theme/variant captures. <!-- rule:skill-setup-command-ref --> <!-- rule:skill-setup-read-project -->
|
||||||
3. After resolving analysis and direction, read [reference/craft-floor.md](reference/craft-floor.md) immediately before any UI edit, including small refinements. It carries the quality floor, the absolute bans, and the reflexes no detector catches. Do not load it for planning-only work. <!-- rule:skill-craft-floor-load -->
|
3. After resolving analysis and direction, read [reference/craft-floor.md](reference/craft-floor.md) immediately before any UI edit, including small refinements. It carries the quality floor, the absolute bans, and the reflexes no detector catches. Do not load it for planning-only work. <!-- rule:skill-craft-floor-load -->
|
||||||
|
|
||||||
**Launcher unavailable:** If refused, missing, or failed, **first send the user a message** that context loading did not run. Then read existing PRODUCT.md and DESIGN.md without inventing missing context, follow the applicable steps 2–3, and perform the requested work through permitted tools. Launcher failure alone does not block otherwise-permitted edits.
|
**Launcher unavailable:** On refusal or failure, send a separate message **before the next tool call**: “Context loading did not run; I’ll read the existing project context directly.” Then read existing PRODUCT.md and DESIGN.md without inventing missing context, follow applicable steps 2–3, and continue through permitted tools. This applies to planning and editing; launcher failure alone does not block either.
|
||||||
|
|
||||||
## How to design
|
## How to design
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ nickname-candidates:
|
|||||||
|
|
||||||
You record a project's design system after the build is done. Ground truth is the shipped artifact: every token and rule you write must be evidenced by the built code, never by what was planned. Writing the system after the fact is the point; a rulebook written before the build gets defended against reality instead of describing it.
|
You record a project's design system after the build is done. Ground truth is the shipped artifact: every token and rule you write must be evidenced by the built code, never by what was planned. Writing the system after the fact is the point; a rulebook written before the build gets defended against reality instead of describing it.
|
||||||
|
|
||||||
You run under a hard turn ceiling that ends the run without warning, and a run that ends before DESIGN.md is written has recorded nothing. Batch several Reads into each turn, take `reference/document.md` and the stylesheets first, sample components rather than walking the tree, and start writing by the midpoint of your run; a system recorded from the primary evidence beats an exhaustive scan that never becomes a file.
|
Complete the check within your turn ceiling. Batch Reads, take `reference/document.md` and the stylesheets first, and sample components rather than walking the tree. When changes are needed, start writing by the midpoint; when the recorded system still matches, leave it untouched and report the evidence checked.
|
||||||
|
|
||||||
## Input Contract
|
## Input Contract
|
||||||
|
|
||||||
@@ -26,10 +26,10 @@ Expect: the project root; the artifact path(s); the direction contract text (THE
|
|||||||
|
|
||||||
1. Read `reference/document.md` in full; it is the operating spec for DESIGN.md's format, token schema, sidecar, and section order. Follow it exactly.
|
1. Read `reference/document.md` in full; it is the operating spec for DESIGN.md's format, token schema, sidecar, and section order. Follow it exactly.
|
||||||
2. Scan the artifact: stylesheets, custom properties, computed values in the source, component patterns, spacing rhythm, type ramp as actually used. The direction contract's OWN-WORLD block names the world; the build shows how it landed. Where they diverge, the build wins and the prose may note the divergence.
|
2. Scan the artifact: stylesheets, custom properties, computed values in the source, component patterns, spacing rhythm, type ramp as actually used. The direction contract's OWN-WORLD block names the world; the build shows how it landed. Where they diverge, the build wins and the prose may note the divergence.
|
||||||
3. Write DESIGN.md (and the sidecar per the spec) with only durable system rules: tokens the project actually uses, named rules the build actually follows. Skip one-off values; a token used once is not a system.
|
3. For a new world or approved system change, write DESIGN.md and its sidecar from durable, reused rules in the build. Ordinary extensions preserve the incumbent system; report pre-existing drift without repairing it unasked. Do not write merely to prove this pass ran.
|
||||||
4. Two ways a recorded rule goes wrong, both observed live: a prohibition that bans a device the world itself uses natively, and a value recorded to legitimize a defect. Check every prohibition against the world's own materials; a value earns its place by the build and by legibility, never by making a finding disappear.
|
4. Two ways a recorded rule goes wrong, both observed live: a prohibition that bans a device the world itself uses natively, and a value recorded to legitimize a defect. Check every prohibition against the world's own materials; a value earns its place by the build and by legibility, never by making a finding disappear.
|
||||||
5. Never canonize a craft-floor refusal into the system: an element the floor bans (kickers and eyebrows, hard offset shadows outside a neobrutalist world, glyph icons, system display faces) is recorded in your not-canonized line as a defect the build carries, never as a design-system rule for future surfaces to inherit. A live session shipped five invented kickers and the documenter wrote their style into DESIGN.md; that is how one violation becomes the house style.
|
5. Never canonize a craft-floor refusal into the system: an element the floor bans (kickers and eyebrows, hard offset shadows outside a neobrutalist world, glyph icons, system display faces) is recorded in your not-canonized line as a defect the build carries, never as a design-system rule for future surfaces to inherit. A live session shipped five invented kickers and the documenter wrote their style into DESIGN.md; that is how one violation becomes the house style.
|
||||||
|
|
||||||
## Output Contract
|
## Output Contract
|
||||||
|
|
||||||
Return: the file paths written, a five-line summary of the recorded system (palette strategy, type ramp shape, named rules), and one line naming anything in the build you deliberately did not canonize and why. No other prose.
|
Return: paths written, or “No changes” with the source and system files checked; a five-line system summary (palette, type ramp, named rules); and one line naming defects or drift not canonized or repaired, and why. No other prose.
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ A section, component, feature, or state inside an established surface inherits t
|
|||||||
|
|
||||||
## 2. Ask what will change the work
|
## 2. Ask what will change the work
|
||||||
|
|
||||||
Ask one round of two or three related questions through the structured question tool when available. Skip settled facts; a precise request may need only a compact confirmation.
|
Before implementation, get the user's answer through the structured question tool when available. Ask two or three related questions; a precise request needs only a compact confirmation. Skip settled facts, not the confirmation: DESIGN.md settles the visual world, not this surface's purpose or concept.
|
||||||
|
|
||||||
- **Persuade:** who must act, what they should believe, which real proof, content, or assets earn that belief.
|
- **Persuade:** who must act, what they should believe, which real proof, content, or assets earn that belief.
|
||||||
- **Operate:** the task, information, important states, frequency, constraints.
|
- **Operate:** the task, information, important states, frequency, constraints.
|
||||||
@@ -72,7 +72,7 @@ Your measured rendition prior: warm, bookish, family, and child-facing subjects
|
|||||||
|
|
||||||
## 5. Record the decision
|
## 5. Record the decision
|
||||||
|
|
||||||
Before code, record the chosen direction as a development-only contract under `## Direction contract` in the relevant surface brief. A direction contract is durable route or artifact strategy, so create or update the brief even when no other surface strategy needs persistence. Keep the contract to six short blocks and 150 words at most. THESIS: the one idea this surface owns and the category-default arrangement it refuses. OWN-WORLD: the palette and component language, specific enough to be recognizable with all content removed. STORY: what the visitor understands, believes, and does. FIRST VIEWPORT: the exact composition, what is where and at what scale, and where the primary action sits. FORM: the chosen form, its position on your ordered list, and the seed key the script printed. Close with one more line, FINISH: the run's exit condition, verbatim "unreviewed and undocumented is unfinished; this build ends with the finish review, the verdict, DESIGN.md, and every shipping raster carrying its provenance". The surface brief is the reminder later agents reload across edits and sessions: a page that looks complete with the FINISH line undischarged is not done, it is abandoned at the finish line. If a block reads like a mood, the direction is not decided yet; the finishing review audits the render against this contract. <!-- rule:skill-decide-then-build -->
|
Before code, record the chosen direction as a development-only contract under `## Direction contract` in the relevant surface brief. A direction contract is durable route or artifact strategy, so create or update the brief even when no other surface strategy needs persistence. Use six short blocks, roughly 150 words; do not spend tool calls counting words. THESIS: the one idea this surface owns and the category-default arrangement it refuses. OWN-WORLD: the palette and component language, specific enough to be recognizable with all content removed. STORY: what the visitor understands, believes, and does. FIRST VIEWPORT: the exact composition, what is where and at what scale, and where the primary action sits. FORM: the chosen form, its position on your ordered list, and the seed key the script printed. Close with one more line, FINISH: the run's exit condition, verbatim "unreviewed and undocumented is unfinished; this build ends with the finish review, the verdict, DESIGN.md, and every shipping raster carrying its provenance". The surface brief is the reminder later agents reload across edits and sessions: a page that looks complete with the FINISH line undischarged is not done, it is abandoned at the finish line. If a block reads like a mood, the direction is not decided yet; the finishing review audits the render against this contract. <!-- rule:skill-decide-then-build -->
|
||||||
|
|
||||||
Never copy the direction contract into implementation source or any browser-delivered artifact. This includes HTML or framework comments, hidden DOM, `<template>` elements, `data-*` attributes, rendered JSX or TSX output, serialized props or state, React Server Component payloads, client bundles, metadata or JSON-LD, accessibility-only text, and files served beside the artifact. A compiler or optimizer removing development metadata is not a safety boundary. Reviewers and documenters receive the contract from the surface brief.
|
Never copy the direction contract into implementation source or any browser-delivered artifact. This includes HTML or framework comments, hidden DOM, `<template>` elements, `data-*` attributes, rendered JSX or TSX output, serialized props or state, React Server Component payloads, client bundles, metadata or JSON-LD, accessibility-only text, and files served beside the artifact. A compiler or optimizer removing development metadata is not a safety boundary. Reviewers and documenters receive the contract from the surface brief.
|
||||||
|
|
||||||
@@ -146,4 +146,4 @@ A rebuild and a fix round share one asset rule: a raster either round creates or
|
|||||||
|
|
||||||
Report the final verdict under the reviewer's own disposition word and at its actual scope. A verdict pass scores the listed fixes and nothing else: "the reviewer scored all three fixes resolved" is a claim it supports, "no material issues remain" is not. A table with open material findings is never announced as a pass, never softened, and never dressed as whole-surface approval when only a fix list was scored. When the user answers a ship with evidence against it, their own screenshot, a named mismatch with the comp, that evidence outranks every capture you made: put their material in the packet and spawn a fresh reviewer for a new full review. Patching inline and self-certifying is how a rejected page ships twice. <!-- rule:skill-user-evidence-reopens-review -->
|
Report the final verdict under the reviewer's own disposition word and at its actual scope. A verdict pass scores the listed fixes and nothing else: "the reviewer scored all three fixes resolved" is a claim it supports, "no material issues remain" is not. A table with open material findings is never announced as a pass, never softened, and never dressed as whole-surface approval when only a fix list was scored. When the user answers a ship with evidence against it, their own screenshot, a named mismatch with the comp, that evidence outranks every capture you made: put their material in the packet and spawn a fresh reviewer for a new full review. Patching inline and self-certifying is how a rejected page ships twice. <!-- rule:skill-user-evidence-reopens-review -->
|
||||||
|
|
||||||
Then spawn the shipped documenter, `impeccable-documenter` (`impeccable_documenter` in codex), with the project root, the artifact path, the direction contract, PRODUCT.md, the [document.md](document.md) reference path, and the boundary to write at; it records DESIGN.md and the sidecar from the built world, ground truth over intention; without subagents the pass runs from [degraded/documenter.md](degraded/documenter.md). The documenter runs after the last correction lands: when any fix round follows the documentation, re-run the documenter over the changed surface, because a DESIGN.md describing a layout that no longer exists turns defects into system guidance. A clean detector pass is not finished; finished is the contract kept, the comp honored, the review closed, and the system recorded. <!-- rule:skill-documenter-records-the-world -->
|
After the last correction, spawn the shipped documenter, `impeccable-documenter` (`impeccable_documenter` in codex), with the project root, artifact path, direction contract, PRODUCT.md, [document.md](document.md), and write boundary. Without subagents, load [degraded/documenter.md](degraded/documenter.md) and [document.md](document.md) before writing. Verify the outcome: new worlds and approved system changes require token-bearing DESIGN.md **and** `.impeccable/design.json`, not prose alone. Ordinary extensions compare the finished build against the incumbent system, preserve its files, and report the evidence checked; report pre-existing drift without repairing it unasked. Recheck after later edits. Finish only when review and documentation are complete. <!-- rule:skill-documenter-records-the-world -->
|
||||||
|
|||||||
@@ -8,6 +8,20 @@ import { tmpdir } from 'node:os';
|
|||||||
const SCRIPT = 'scripts/ci-test-plan.mjs';
|
const SCRIPT = 'scripts/ci-test-plan.mjs';
|
||||||
|
|
||||||
describe('ci-test-plan', () => {
|
describe('ci-test-plan', () => {
|
||||||
|
it('requires explicit manual opt-in and preprovisions the full workflow job', () => {
|
||||||
|
const workflow = readFileSync('.github/workflows/ci.yml', 'utf8');
|
||||||
|
assert.match(workflow, /skill_workflow:\s*description:[^\n]+\s*type: boolean\s*default: false/);
|
||||||
|
const job = workflow.split('\n skill-workflow:')[1];
|
||||||
|
assert.doesNotMatch(job.split('\n steps:')[0], /runner\./, 'runner context is unavailable at job-level env');
|
||||||
|
assert.match(job, /if: github.event_name == 'workflow_dispatch' && inputs.skill_workflow/);
|
||||||
|
assert.ok(job.indexOf('bun run fetch:engine') < job.indexOf('bun run test:skill-workflow'));
|
||||||
|
assert.ok(job.indexOf('playwright install --with-deps chromium') < job.indexOf('bun run test:skill-workflow'));
|
||||||
|
const protocol = workflow.split('\n skill-behavior:')[1].split('\n skill-workflow:')[0];
|
||||||
|
assert.match(protocol, /bun run fetch:engine/);
|
||||||
|
assert.doesNotMatch(protocol, /IMPECCABLE_SKILL_BEHAVIOR_MODELS:/, 'protocol coverage must retain the multi-family defaults');
|
||||||
|
assert.match(protocol, /GOOGLE_CLOUD_API_KEY:/);
|
||||||
|
assert.match(protocol, /ANTHROPIC_API_KEY:/);
|
||||||
|
});
|
||||||
it('keeps docs-only pull requests on the core suite', () => {
|
it('keeps docs-only pull requests on the core suite', () => {
|
||||||
const outputs = runPlan({
|
const outputs = runPlan({
|
||||||
GITHUB_EVENT_NAME: 'pull_request',
|
GITHUB_EVENT_NAME: 'pull_request',
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ describe('live-e2e accept cleanup regression', () => {
|
|||||||
log: (msg) => t.diagnostic(msg),
|
log: (msg) => t.diagnostic(msg),
|
||||||
});
|
});
|
||||||
|
|
||||||
const { page, tmp, teardown } = session;
|
const { page, tmp, appRoot, teardown } = session;
|
||||||
try {
|
try {
|
||||||
t.diagnostic(`Using LLM agent (provider=${llmConfig.provider} model=${llmConfig.model})`);
|
t.diagnostic(`Using LLM agent (provider=${llmConfig.provider} model=${llmConfig.model})`);
|
||||||
await waitForHandshake(page);
|
await waitForHandshake(page);
|
||||||
@@ -104,6 +104,10 @@ describe('live-e2e accept cleanup regression', () => {
|
|||||||
await clickNext(page);
|
await clickNext(page);
|
||||||
assert.equal(await getVisibleVariant(page), 2, 'variant 2 visible after one Next');
|
assert.equal(await getVisibleVariant(page), 2, 'variant 2 visible after one Next');
|
||||||
|
|
||||||
|
const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('impeccable-live-session') || 'null'));
|
||||||
|
assert.ok(saved?.id && /^[a-zA-Z0-9_-]+$/.test(saved.id), 'cycling session has a safe durable id');
|
||||||
|
const snapshotPath = join(appRoot, '.impeccable/live/sessions', `${saved.id}.snapshot.json`);
|
||||||
|
|
||||||
t.diagnostic('Accepting variant 2');
|
t.diagnostic('Accepting variant 2');
|
||||||
await clickAccept(page, { expectedVariant: 2 });
|
await clickAccept(page, { expectedVariant: 2 });
|
||||||
|
|
||||||
@@ -120,6 +124,19 @@ describe('live-e2e accept cleanup regression', () => {
|
|||||||
sourceFile,
|
sourceFile,
|
||||||
finalSource,
|
finalSource,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Source/DOM cleanup can precede live-complete, or succeed while its
|
||||||
|
// durable acknowledgement fails. Do not count that as a finished accept.
|
||||||
|
const deadline = Date.now() + 30_000;
|
||||||
|
let snapshot;
|
||||||
|
do {
|
||||||
|
snapshot = JSON.parse(readFileSync(snapshotPath, 'utf8'));
|
||||||
|
if (snapshot.phase === 'completed') break;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||||
|
} while (Date.now() < deadline);
|
||||||
|
assert.equal(snapshot.phase, 'completed', 'accept must reach durable completed phase without forcing completion');
|
||||||
|
assert.doesNotMatch(readFileSync(sourceFile, 'utf8'), /data-impeccable-[\w-]+\s*=/, 'accepted source contains no reserved runtime attributes');
|
||||||
|
t.diagnostic(`Durable completion verified for ${saved.id}`);
|
||||||
} finally {
|
} finally {
|
||||||
await teardown();
|
await teardown();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1520,6 +1520,12 @@ describe('live-e2e LLM agent variant prompt', () => {
|
|||||||
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /bare text element/);
|
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /bare text element/);
|
||||||
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /Accept persists a real source change/);
|
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /Accept persists a real source change/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('uses permanent styling hooks outside the reserved live-runtime namespace', () => {
|
||||||
|
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /data-design-variant/);
|
||||||
|
assert.doesNotMatch(VARIANT_SYSTEM_INSTRUCTIONS, /add[^\n]*data-impeccable-e2e-variant/);
|
||||||
|
assert.match(VARIANT_SYSTEM_INSTRUCTIONS, /Never invent data-impeccable-\*/);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('live-e2e LLM agent variant copy validation', () => {
|
describe('live-e2e LLM agent variant copy validation', () => {
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ export const VARIANT_SYSTEM_INSTRUCTIONS = [
|
|||||||
'- Replace mode: for bare text elements, keep the full visible copy in one editable text node. If you add child markup for styling, wrap the entire copy; never split the copy across sibling text nodes.',
|
'- Replace mode: for bare text elements, keep the full visible copy in one editable text node. If you add child markup for styling, wrap the entire copy; never split the copy across sibling text nodes.',
|
||||||
'- Replace mode: PRESERVE existing class-bearing descendant elements in place. If the picked element contains <h1 class="hero-title"> and <p class="hero-hook">, keep those elements/classes as direct descendants of the replacement root; do not wrap them in a new structural div such as <div class="hero-inner">.',
|
'- Replace mode: PRESERVE existing class-bearing descendant elements in place. If the picked element contains <h1 class="hero-title"> and <p class="hero-hook">, keep those elements/classes as direct descendants of the replacement root; do not wrap them in a new structural div such as <div class="hero-inner">.',
|
||||||
'- Replace mode: Do not return source-identical variants. For a bare text element, preserve the root tag/class/copy but add a small child span or styling hook so Accept persists a real source change.',
|
'- Replace mode: Do not return source-identical variants. For a bare text element, preserve the root tag/class/copy but add a small child span or styling hook so Accept persists a real source change.',
|
||||||
'- Replace mode: for non-bare elements where the existing children must stay in place, add a harmless root attribute such as data-impeccable-e2e-variant="1" or another non-copy styling hook so the markup is materially changed without changing visible text.',
|
'- Replace mode: for non-bare elements where the existing children must stay in place, add a permanent styling hook such as data-design-variant="1" so the markup is materially changed without changing visible text. Never invent data-impeccable-* attributes: that namespace is reserved for temporary runtime state.',
|
||||||
'- Generate exactly event.count variants — no more, no fewer.',
|
'- Generate exactly event.count variants — no more, no fewer.',
|
||||||
'- Mix the param kinds across the variant set: include at least one range, one steps, and one toggle when count >= 3.',
|
'- Mix the param kinds across the variant set: include at least one range, one steps, and one toggle when count >= 3.',
|
||||||
'- The scopedCss must follow wrapInfo.cssAuthoring exactly: use its selector strategy, rulePattern, requirements, and forbidden patterns.',
|
'- The scopedCss must follow wrapInfo.cssAuthoring exactly: use its selector strategy, rulePattern, requirements, and forbidden patterns.',
|
||||||
|
|||||||
@@ -3,8 +3,188 @@ import assert from 'node:assert/strict';
|
|||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import { MockLanguageModelV3 } from 'ai/test';
|
import { MockLanguageModelV3 } from 'ai/test';
|
||||||
import { prepareWorkspace, cleanupWorkspace, makeTools, runTurn, SKILL_BODY } from './skill-behavior/harness.mjs';
|
import { prepareWorkspace, cleanupWorkspace, makeTools, runTurn, fileLoaded, SKILL_BODY } from './skill-behavior/harness.mjs';
|
||||||
import { assertPlanningFallbackWarning } from './skill-behavior/assertions.mjs';
|
import { assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs';
|
||||||
|
import { CASE_STUDY_ANSWER } from './skill-behavior/fixtures.mjs';
|
||||||
|
import { sourceHash as hashSources } from './skill-workflow/source-hash.mjs';
|
||||||
|
import { assertCompleted, assertFreshCaptures, assertNoChangeDocumentation, assertDocumentationArtifacts } from './skill-workflow/assertions.mjs';
|
||||||
|
|
||||||
|
it('documentation artifacts require tokens and the v2 sidecar independently of wrapper coverage', () => {
|
||||||
|
const design = '---\ncolors:\n ink: "#222"\ntypography:\n body:\n fontFamily: system-ui\n---\n## Overview\nA reading surface.\n';
|
||||||
|
const sidecar = JSON.stringify({ schemaVersion: 2, extensions: { colorMeta: {} }, narrative: { northStar: 'Manual' } });
|
||||||
|
assert.doesNotThrow(() => assertDocumentationArtifacts(design, sidecar));
|
||||||
|
assert.throws(() => assertDocumentationArtifacts('## Colors\nInk: #222\n', sidecar), /frontmatter/);
|
||||||
|
assert.throws(() => assertDocumentationArtifacts(design.replace('colors:', 'palette:'), sidecar), /color tokens/);
|
||||||
|
assert.throws(() => assertDocumentationArtifacts(design.replace('typography:', 'type:'), sidecar), /typography tokens/);
|
||||||
|
assert.throws(() => assertDocumentationArtifacts(design, ''), SyntaxError);
|
||||||
|
assert.throws(() => assertDocumentationArtifacts(design, sidecar.replace('"schemaVersion":2', '"schemaVersion":1')), /v2 sidecar/);
|
||||||
|
for (const key of ['extensions', 'narrative']) {
|
||||||
|
for (const value of [undefined, {}, []]) {
|
||||||
|
assert.throws(() => assertDocumentationArtifacts(design, JSON.stringify({ ...JSON.parse(sidecar), [key]: value })), /metadata/);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('advice outcomes do not depend on opening every reference, but keep consent and prerequisite gates', () => {
|
||||||
|
const trace = { toolCalls: [], writePaths: [], questionCalls: [], bashCommands: [] };
|
||||||
|
const advice = 'For index.html, start with init to capture context and document to record the existing identity.';
|
||||||
|
const comparison = 'Critique is an assessment. Polish makes fixes. Critique is optional; polish can run directly.';
|
||||||
|
const check = (text, observation = trace) => assertCommandComparison(observation, text);
|
||||||
|
assert.doesNotThrow(() => assertWorkflowAdvice(trace, advice, { missingContext: true }));
|
||||||
|
assert.doesNotThrow(() => check(comparison));
|
||||||
|
assert.doesNotThrow(() => check("Critique reviews the surface. Polish refines it. Critique isn't required before polish."));
|
||||||
|
assert.doesNotThrow(() => check('Critique gives a report. Polish edits the surface independently, without a critique.'));
|
||||||
|
assert.deepEqual(missingReferences(trace, ['reference/critique.md']), ['reference/critique.md']);
|
||||||
|
for (const wrong of ['You must run critique before polish.', 'Critique is required before polish.', 'Polish requires a critique.']) {
|
||||||
|
assert.throws(() => check(`${comparison} ${wrong}`), /invent a critique prerequisite/);
|
||||||
|
}
|
||||||
|
for (const wrong of ['You must run init before polishing.', 'You need to document before refinement.', 'Polish requires PRODUCT.md.']) {
|
||||||
|
assert.throws(() => assertWorkflowAdvice(trace, `${advice} ${wrong}`, { missingContext: true }), /mandatory prerequisite/);
|
||||||
|
}
|
||||||
|
assert.throws(() => check(''), /advice must reach/);
|
||||||
|
assert.throws(() => check('I loaded critique.md and polish.md.'), /explain critique/);
|
||||||
|
for (const mutation of ['index.html', '.impeccable/critique/report.md']) {
|
||||||
|
assert.throws(() => check(comparison, { ...trace, toolCalls: [{ mutatedPaths: [mutation] }] }), /must not edit/);
|
||||||
|
}
|
||||||
|
assert.throws(() => check(comparison, { ...trace, writePaths: ['DESIGN.md'] }), /write tool/);
|
||||||
|
assert.throws(() => check(comparison, { ...trace, questionCalls: [{}] }), /interview/);
|
||||||
|
assert.throws(() => check(comparison, { ...trace, bashCommands: ['impeccable detect index.html'] }), /menu scans/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('an unchanged documentation outcome needs real reads and a supported report, not a wrapper filename', () => {
|
||||||
|
const files = ['reference/document.md', 'index.html', 'DESIGN.md'];
|
||||||
|
const toolCalls = files.map((file) => ({ loadedFiles: [file] }));
|
||||||
|
const result = { outcome: 'complete', trace: { toolCalls }, text: 'No changes: index.html matches DESIGN.md: system-ui, 65ch, #0645ad.' };
|
||||||
|
const options = { target: 'index.html', evidence: [/system-ui/, /65ch/, /#0645ad/] };
|
||||||
|
const check = (value) => assertNoChangeDocumentation(value, options);
|
||||||
|
assert.doesNotThrow(() => check(result));
|
||||||
|
assert.deepEqual(missingReferences(result.trace, ['degraded/documenter.md']), ['degraded/documenter.md']);
|
||||||
|
for (const missing of files) {
|
||||||
|
assert.throws(() => check({ ...result, trace: { toolCalls: toolCalls.filter((call) => !call.loadedFiles.includes(missing)) } }), /inspect the actual source/);
|
||||||
|
}
|
||||||
|
assert.throws(() => check({ ...result, trace: { toolCalls: [{ input: { path: 'reference/document.md' }, succeeded: false }] } }), /inspect the actual source/);
|
||||||
|
assert.throws(() => check({ ...result, text: 'No changes: index.html matches DESIGN.md.' }), /report evidence/);
|
||||||
|
assert.throws(() => check({ ...result, outcome: 'step-budget' }), /did not finish/);
|
||||||
|
for (const file of ['DESIGN.md', '.impeccable/design.json', 'index.html']) {
|
||||||
|
assert.throws(() => check({ ...result, trace: { toolCalls: [...toolCalls, { mutatedPaths: [file] }] } }), /must not mutate/);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('full workflows reject exhausted budgets and stale or absent visual evidence', () => {
|
||||||
|
for (const outcome of ['checkpoint', 'step-budget', 'output-limit', 'error']) {
|
||||||
|
assert.throws(() => assertCompleted({ outcome, steps: 50 }), /did not finish/);
|
||||||
|
}
|
||||||
|
assert.doesNotThrow(() => assertCompleted({ outcome: 'complete', steps: 12 }));
|
||||||
|
const workspace = prepareWorkspace({ files: { 'index.html': '<h1>Test</h1>' } });
|
||||||
|
try {
|
||||||
|
const sourceHash = hashSources(workspace);
|
||||||
|
const edit = { mutatedPaths: ['index.html'] };
|
||||||
|
const shots = ['desktop', 'mobile'].map((viewport) => ({ capture: { target: 'index.html', viewport, sourceHash } }));
|
||||||
|
const check = (toolCalls) => assertFreshCaptures({ toolCalls }, workspace, 'index.html');
|
||||||
|
assert.doesNotThrow(() => check([edit, ...shots]));
|
||||||
|
assert.throws(() => check([edit]), /missing desktop screenshot/);
|
||||||
|
assert.throws(() => check([...shots, edit]), /missing desktop screenshot/);
|
||||||
|
assert.throws(() => check([edit, shots[0]]), /missing mobile screenshot/);
|
||||||
|
fs.writeFileSync(path.join(workspace, 'style.css'), 'h1 { color: red; }');
|
||||||
|
assert.throws(() => check([edit, ...shots]), /missing desktop screenshot/);
|
||||||
|
} finally {
|
||||||
|
cleanupWorkspace(workspace);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('case-study user supplies evidence now instead of promising a future message', async () => {
|
||||||
|
const workspace = prepareWorkspace();
|
||||||
|
try {
|
||||||
|
const { tools, trace } = makeTools(workspace, {}, { answer: () => CASE_STUDY_ANSWER });
|
||||||
|
for (const question of ['What real customer proof do you have?', 'Please paste the promised details.']) {
|
||||||
|
const result = JSON.parse(await tools.ask_user_question.execute({ questions: [{
|
||||||
|
question, options: [{ label: 'I will paste real customer quotes in my next message' }],
|
||||||
|
}] }));
|
||||||
|
assert.equal(result.answers[question], CASE_STUDY_ANSWER);
|
||||||
|
assert.match(result.answers[question], /clearly labeled synthetic case/);
|
||||||
|
}
|
||||||
|
assert.equal(trace.questionAnswers.length, 2);
|
||||||
|
} finally {
|
||||||
|
cleanupWorkspace(workspace);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('new-work requires approval and a brief before code, then documents the finished redesign', () => {
|
||||||
|
const ask = { name: 'ask_user_question' };
|
||||||
|
const brief = { name: 'bash', mutatedPaths: ['.impeccable/surfaces/current-html.md'] };
|
||||||
|
const page = { name: 'write', mutatedPaths: ['current.html'] };
|
||||||
|
const design = { name: 'write', mutatedPaths: ['DESIGN.md'] };
|
||||||
|
const check = (toolCalls) => assertNewWorkLifecycle({ toolCalls }, { target: 'current.html', redesign: true });
|
||||||
|
assert.doesNotThrow(() => check([ask, brief, page, design]));
|
||||||
|
assert.doesNotThrow(() => check([ask, brief, page, design, page, design]));
|
||||||
|
assert.throws(() => check([ask, brief]), /did not produce/);
|
||||||
|
assert.throws(() => check([brief, page, ask, design]), /user answer/);
|
||||||
|
assert.throws(() => check([ask, page, brief, design]), /surface brief before/);
|
||||||
|
assert.throws(() => check([ask, brief, design, page]), /finished build/);
|
||||||
|
assert.throws(() => check([ask, brief, page, design, page]), /finished build/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stages resolved references independently of the source skill', async () => {
|
||||||
|
const workspace = prepareWorkspace();
|
||||||
|
try {
|
||||||
|
const base = path.join(workspace, '.claude/skills/impeccable');
|
||||||
|
assert.equal(fs.lstatSync(base).isSymbolicLink(), false);
|
||||||
|
const { tools } = makeTools(workspace);
|
||||||
|
const critique = await tools.read.execute({ path: '.claude/skills/impeccable/reference/critique.md' });
|
||||||
|
assert.match(critique, /Use the ask_user_question tool\./);
|
||||||
|
assert.doesNotMatch(critique, /\{\{ask_instruction\}\}|\{\{scripts_path\}\}|<codex>/);
|
||||||
|
for (const role of ['finish-reviewer', 'documenter']) {
|
||||||
|
const reference = await tools.read.execute({ path: `.claude/skills/impeccable/reference/degraded/${role}.md` });
|
||||||
|
assert.match(reference, /This harness has no subagent capability/);
|
||||||
|
assert.doesNotMatch(reference, /\{\{scripts_path\}\}|<codex>/);
|
||||||
|
}
|
||||||
|
const shellRead = await tools.bash.execute({ command: 'cat .claude/skills/impeccable/reference/critique.md' });
|
||||||
|
assert.ok(shellRead.includes(critique), 'shell and read tools must see the same resolved reference');
|
||||||
|
assert.match(await tools.write.execute({ path: '.claude/skills/impeccable/reference/critique.md', contents: 'bad' }), /^Error:/);
|
||||||
|
} finally {
|
||||||
|
cleanupWorkspace(workspace);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reference-loading evidence requires content, not a failed read or a filename mention', async () => {
|
||||||
|
const workspace = prepareWorkspace();
|
||||||
|
try {
|
||||||
|
const ref = '.claude/skills/impeccable/reference/polish.md';
|
||||||
|
const denied = makeTools(workspace, {}, {}, { denyBash: true });
|
||||||
|
await denied.tools.bash.execute({ command: `cat ${ref}` });
|
||||||
|
assert.equal(fileLoaded(denied.trace, 'polish.md'), false);
|
||||||
|
await denied.tools.read.execute({ path: 'missing/polish.md' });
|
||||||
|
assert.equal(fileLoaded(denied.trace, 'polish.md'), false);
|
||||||
|
const allowed = makeTools(workspace);
|
||||||
|
await allowed.tools.bash.execute({ command: `printf '%s' '${ref}'` });
|
||||||
|
assert.equal(fileLoaded(allowed.trace, 'polish.md'), false);
|
||||||
|
await allowed.tools.bash.execute({ command: `cat ${ref}` });
|
||||||
|
assert.equal(fileLoaded(allowed.trace, 'polish.md'), true);
|
||||||
|
await denied.tools.read.execute({ path: ref });
|
||||||
|
assert.equal(fileLoaded(denied.trace, 'polish.md'), true);
|
||||||
|
} finally {
|
||||||
|
cleanupWorkspace(workspace);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('headless behavior shells disable unattended decision pages and omit provider credentials', async () => {
|
||||||
|
const workspace = prepareWorkspace();
|
||||||
|
try {
|
||||||
|
const { tools } = makeTools(workspace, { OPENAI_API_KEY: 'synthetic-secret', IMPECCABLE_QUESTION_DISABLED: '0' });
|
||||||
|
const result = await tools.bash.execute({ command: 'node -e \'console.log(JSON.stringify({disabled:process.env.IMPECCABLE_QUESTION_DISABLED,hasKey:!!process.env.OPENAI_API_KEY}))\'' });
|
||||||
|
assert.match(result, /"disabled":"1"/);
|
||||||
|
assert.match(result, /"hasKey":false/);
|
||||||
|
assert.doesNotMatch(result, /synthetic-secret/);
|
||||||
|
if (process.env.IMPECCABLE_BIN) {
|
||||||
|
const question = await tools.bash.execute({ command: '.claude/skills/impeccable/scripts/impeccable serve-question --start --payload nonexistent.json' });
|
||||||
|
assert.match(question, /^exit=2\n/);
|
||||||
|
assert.match(question, /use the structured question tool instead/);
|
||||||
|
assert.equal(fs.existsSync(path.join(workspace, '.impeccable/questions')), false);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
cleanupWorkspace(workspace);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('planning fallback requires an assistant warning between the denial and context reads', () => {
|
it('planning fallback requires an assistant warning between the denial and context reads', () => {
|
||||||
const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };
|
const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };
|
||||||
@@ -47,6 +227,81 @@ it('DeepSeek gets an explicit output ceiling instead of the compatibility SDK de
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('protocol checkpoints stop at successful evidence without claiming task completion', async () => {
|
||||||
|
const workspace = prepareWorkspace();
|
||||||
|
try {
|
||||||
|
const model = new MockLanguageModelV3({ modelId: 'claude-sonnet-5', doGenerate: {
|
||||||
|
content: [{ type: 'tool-call', toolCallId: 'load', toolName: 'read', input: JSON.stringify({ path: '.claude/skills/impeccable/reference/polish.md' }) }],
|
||||||
|
finishReason: { unified: 'tool-calls', raw: 'tool-calls' },
|
||||||
|
usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } }, warnings: [],
|
||||||
|
} });
|
||||||
|
const result = await runTurn({ workspace, model, userPrompt: 'Route only.', maxSteps: 10,
|
||||||
|
stopAfter: (trace) => fileLoaded(trace, 'polish.md') });
|
||||||
|
assert.equal(result.outcome, 'checkpoint');
|
||||||
|
assert.equal(result.steps, 1);
|
||||||
|
assert.equal(model.doGenerateCalls.length, 1);
|
||||||
|
const exhausted = await runTurn({ workspace, model, userPrompt: 'Complete work.', maxSteps: 1 });
|
||||||
|
assert.equal(exhausted.outcome, 'step-budget');
|
||||||
|
} finally { cleanupWorkspace(workspace); }
|
||||||
|
});
|
||||||
|
|
||||||
|
it('protocol notice checkpoints observe intermediate assistant text', async () => {
|
||||||
|
const workspace = prepareWorkspace();
|
||||||
|
try {
|
||||||
|
let calls = 0;
|
||||||
|
const model = new MockLanguageModelV3({ doGenerate: async () => {
|
||||||
|
calls++;
|
||||||
|
return {
|
||||||
|
content: [{ type: 'text', text: 'Impeccable version 99.0.0 is available; may I update it?' },
|
||||||
|
{ type: 'tool-call', toolCallId: 'list', toolName: 'list', input: '{}' }],
|
||||||
|
finishReason: { unified: 'tool-calls', raw: 'tool-calls' },
|
||||||
|
usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } }, warnings: [],
|
||||||
|
};
|
||||||
|
} });
|
||||||
|
const result = await runTurn({ workspace, model, userPrompt: 'Inspect this page.', maxSteps: 10,
|
||||||
|
stopAfter: (trace) => trace.assistantTexts?.some((text) => text.includes('99.0.0')) });
|
||||||
|
assert.equal(calls, 1);
|
||||||
|
assert.equal(result.outcome, 'checkpoint');
|
||||||
|
assert.match(result.trace.assistantTexts[0], /may I update/);
|
||||||
|
} finally {
|
||||||
|
cleanupWorkspace(workspace);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('optional diagnostics retain tool evidence when a provider turn fails', async () => {
|
||||||
|
const workspace = prepareWorkspace({ files: { 'PRODUCT.md': 'Synthetic product context.' } });
|
||||||
|
const previous = process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR;
|
||||||
|
const traceDir = path.join(workspace, 'diagnostics');
|
||||||
|
process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR = traceDir;
|
||||||
|
try {
|
||||||
|
let calls = 0;
|
||||||
|
const model = new MockLanguageModelV3({
|
||||||
|
modelId: 'claude-sonnet-5',
|
||||||
|
doGenerate: async () => {
|
||||||
|
if (calls++ === 0) return {
|
||||||
|
content: [{ type: 'tool-call', toolCallId: 'read-product', toolName: 'read', input: JSON.stringify({ path: 'PRODUCT.md' }) }],
|
||||||
|
finishReason: { unified: 'tool-calls', raw: 'tool-calls' },
|
||||||
|
usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } },
|
||||||
|
warnings: [],
|
||||||
|
};
|
||||||
|
throw new Error('synthetic provider failure');
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await assert.rejects(runTurn({ workspace, model, userPrompt: 'Synthetic diagnostic test.' }), /synthetic provider failure/);
|
||||||
|
const files = fs.readdirSync(traceDir);
|
||||||
|
assert.equal(files.length, 1);
|
||||||
|
const diagnostic = JSON.parse(fs.readFileSync(path.join(traceDir, files[0]), 'utf8'));
|
||||||
|
assert.equal(diagnostic.status, 'failed');
|
||||||
|
assert.match(diagnostic.error, /synthetic provider failure/);
|
||||||
|
assert.equal(diagnostic.trace.toolCalls.length, 1);
|
||||||
|
assert.equal(fileLoaded(diagnostic.trace, 'PRODUCT.md'), true);
|
||||||
|
} finally {
|
||||||
|
if (previous === undefined) delete process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR;
|
||||||
|
else process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR = previous;
|
||||||
|
cleanupWorkspace(workspace);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('loaded-skill metadata resolves to the staged launcher and readable references', async () => {
|
it('loaded-skill metadata resolves to the staged launcher and readable references', async () => {
|
||||||
const workspace = prepareWorkspace();
|
const workspace = prepareWorkspace();
|
||||||
try {
|
try {
|
||||||
@@ -96,18 +351,62 @@ it('context-only routing tools reject shell searches and compound commands befor
|
|||||||
for (const command of [
|
for (const command of [
|
||||||
'find / -name routing.md',
|
'find / -name routing.md',
|
||||||
'.claude/skills/impeccable/scripts/impeccable context; echo bad > index.html',
|
'.claude/skills/impeccable/scripts/impeccable context; echo bad > index.html',
|
||||||
|
'.claude/skills/impeccable/scripts/impeccable context --target index.html; echo bad > index.html',
|
||||||
|
'.claude/skills/impeccable/scripts/impeccable context --target "$(echo bad > index.html)"',
|
||||||
|
'.claude/skills/impeccable/scripts/impeccable context --target=index.html; echo bad > index.html',
|
||||||
|
'.claude/skills/impeccable/scripts/impeccable context --target="$(echo bad > index.html)"',
|
||||||
'echo bad > index.html',
|
'echo bad > index.html',
|
||||||
]) {
|
]) {
|
||||||
assert.match(await tools.bash.execute({ command }), /^Error:/);
|
assert.match(await tools.bash.execute({ command }), /^Error:/);
|
||||||
}
|
}
|
||||||
assert.equal(fs.readFileSync(path.join(workspace, 'index.html'), 'utf8'), 'before');
|
assert.equal(fs.readFileSync(path.join(workspace, 'index.html'), 'utf8'), 'before');
|
||||||
assert.equal(trace.bashCommands.length, 3, 'rejected attempts remain observable');
|
assert.equal(trace.bashCommands.length, 7, 'rejected attempts remain observable');
|
||||||
assert.ok(trace.toolCalls.every((call) => call.mutatedPaths.length === 0));
|
assert.ok(trace.toolCalls.every((call) => call.mutatedPaths.length === 0));
|
||||||
} finally {
|
} finally {
|
||||||
cleanupWorkspace(workspace);
|
cleanupWorkspace(workspace);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('successful-loader controls accept a workspace-relative target', { skip: !process.env.IMPECCABLE_BIN }, async () => {
|
||||||
|
const workspace = prepareWorkspace({ files: { 'index.html': '<html></html>' } });
|
||||||
|
try {
|
||||||
|
const { tools } = makeTools(workspace, {}, {}, { contextOnlyBash: true });
|
||||||
|
assert.match(await tools.bash.execute({ command: '.claude/skills/impeccable/scripts/impeccable context --target index.html' }), /^exit=0\n/);
|
||||||
|
for (const separator of [' ', '=']) {
|
||||||
|
for (const target of ['index.html', 'src/routes/+page.svelte', '"src/routes/+page.svelte"', '"my page.html"', "'my page.html'"]) {
|
||||||
|
assert.match(await tools.bash.execute({ command: `.claude/skills/impeccable/scripts/impeccable context --target${separator}${target}` }), /^exit=0\n/);
|
||||||
|
}
|
||||||
|
assert.match(await tools.bash.execute({ command: `.claude/skills/impeccable/scripts/impeccable context --target${separator}../outside.html` }), /^Error:/);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
cleanupWorkspace(workspace);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('workspace tools reject symlink escapes and preserve staged-skill write protection', async () => {
|
||||||
|
const workspace = prepareWorkspace({ files: { 'local/value.txt': 'local' } });
|
||||||
|
const outside = prepareWorkspace({ files: { 'value.txt': 'outside' } });
|
||||||
|
try {
|
||||||
|
fs.symlinkSync(outside, path.join(workspace, 'escape'), 'junction');
|
||||||
|
fs.symlinkSync(path.join(workspace, 'local'), path.join(workspace, 'alias'), 'junction');
|
||||||
|
fs.symlinkSync(path.join(workspace, '.claude'), path.join(workspace, 'skill-alias'), 'junction');
|
||||||
|
const { tools } = makeTools(workspace, {}, {}, { contextOnlyBash: true });
|
||||||
|
assert.match(await tools.read.execute({ path: 'escape/value.txt' }), /^Error:/);
|
||||||
|
assert.match(await tools.list.execute({ path: 'escape' }), /^Error:/);
|
||||||
|
assert.match(await tools.write.execute({ path: 'escape/new/file.txt', contents: 'bad' }), /^Error:/);
|
||||||
|
assert.match(await tools.bash.execute({ command: '.claude/skills/impeccable/scripts/impeccable context --target escape/value.txt' }), /^Error:/);
|
||||||
|
assert.match(await tools.write.execute({ path: 'skill-alias/skills/impeccable/reference/routing.md', contents: 'bad' }), /^Error:/);
|
||||||
|
assert.equal(await tools.read.execute({ path: 'alias/value.txt' }), 'local');
|
||||||
|
await tools.write.execute({ path: 'alias/nested/new.txt', contents: 'allowed' });
|
||||||
|
assert.equal(fs.readFileSync(path.join(workspace, 'local/nested/new.txt'), 'utf8'), 'allowed');
|
||||||
|
assert.equal(fs.existsSync(path.join(outside, 'new')), false);
|
||||||
|
assert.equal(fs.readFileSync(path.join(outside, 'value.txt'), 'utf8'), 'outside');
|
||||||
|
} finally {
|
||||||
|
cleanupWorkspace(workspace);
|
||||||
|
cleanupWorkspace(outside);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('context-only routing tools keep project writes observable but protect the staged skill', async () => {
|
it('context-only routing tools keep project writes observable but protect the staged skill', async () => {
|
||||||
const workspace = prepareWorkspace({ files: { 'index.html': 'before' } });
|
const workspace = prepareWorkspace({ files: { 'index.html': 'before' } });
|
||||||
try {
|
try {
|
||||||
|
|||||||
+262
-16
@@ -2,8 +2,8 @@
|
|||||||
|
|
||||||
LLM-backed scenarios that verify how the impeccable skill drives context,
|
LLM-backed scenarios that verify how the impeccable skill drives context,
|
||||||
command-reference, new-work, and native-platform loading. Each scenario runs
|
command-reference, new-work, and native-platform loading. Each scenario runs
|
||||||
against one current model from each supported provider (Anthropic, OpenAI,
|
against the default Anthropic, OpenAI, and Google models. DeepSeek remains
|
||||||
Google, DeepSeek).
|
available through `IMPECCABLE_SKILL_BEHAVIOR_MODELS`.
|
||||||
|
|
||||||
These are the tests you re-run when you refactor anything in SKILL.md's
|
These are the tests you re-run when you refactor anything in SKILL.md's
|
||||||
`## Setup` section. They fail when the agent stops following the loading
|
`## Setup` section. They fail when the agent stops following the loading
|
||||||
@@ -25,7 +25,7 @@ skipped, not failed.
|
|||||||
Also requires the engine binary (`bun run fetch:engine`, or `IMPECCABLE_BIN`).
|
Also requires the engine binary (`bun run fetch:engine`, or `IMPECCABLE_BIN`).
|
||||||
The staged skill dir ships the launcher (`scripts/impeccable`); the harness
|
The staged skill dir ships the launcher (`scripts/impeccable`); the harness
|
||||||
exports `IMPECCABLE_BIN` into every bash call the agent makes, so the launcher
|
exports `IMPECCABLE_BIN` into every bash call the agent makes, so the launcher
|
||||||
resolves the binary in both symlink and copy mode without a download. Without a
|
resolves the binary in the generated fixture without a download. Without a
|
||||||
binary the suites skip.
|
binary the suites skip.
|
||||||
|
|
||||||
To run a single scenario against one model:
|
To run a single scenario against one model:
|
||||||
@@ -37,15 +37,175 @@ IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-sonnet-5 IMPECCABLE_SKILL_BEHAVIOR_VERBO
|
|||||||
|
|
||||||
## How it works
|
## How it works
|
||||||
|
|
||||||
|
### Protocol versus full completion
|
||||||
|
|
||||||
|
`test:skill-behavior` now runs only `scenarios.test.mjs`. Routing cases stop
|
||||||
|
at the successful reference/context checkpoint they assert, with a ten-step
|
||||||
|
ceiling; shell access is context-only. They do **not** claim that a page was
|
||||||
|
built or reviewed. Editing/fallback controls retain their original assertions.
|
||||||
|
The focused S1/S2/S3/S4/S19 rerun passed 21/21 across the three default models.
|
||||||
|
A broader run exposed the context-only allowlist rejecting Svelte's valid
|
||||||
|
`+page.svelte` target. It was stopped, the allowlist fixed with a failing-then-
|
||||||
|
passing unit test, and S8 passed 3/3 on the focused rerun. Failed file reads
|
||||||
|
do not count as project exploration.
|
||||||
|
Update-notice (S9) and explicit-command (S18) checks also now stop at their
|
||||||
|
actual protocol checkpoints, instead of continuing into unrelated polishing;
|
||||||
|
their focused final reruns each passed 3/3. S9 now explicitly requires the
|
||||||
|
assistant to surface the update, not merely receive its loader directive.
|
||||||
|
|
||||||
|
Full workflows moved to `tests/skill-workflow/full-build.test.mjs`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun run fetch:engine
|
||||||
|
bunx playwright install chromium
|
||||||
|
bun run test:skill-workflow
|
||||||
|
```
|
||||||
|
|
||||||
|
This separately billed suite defaults to Claude only; use
|
||||||
|
`IMPECCABLE_SKILL_BEHAVIOR_MODELS` to explicitly choose another model or sweep.
|
||||||
|
It preflights a local server and Chromium before each provider turn, exposing
|
||||||
|
real desktop/mobile screenshot and PNG viewing tools. Text-only fixtures use
|
||||||
|
system fonts and block external browser requests. No extra skill prose is added.
|
||||||
|
The API harness is not the actual Claude Code host, nor is its shell sandboxed.
|
||||||
|
|
||||||
|
Each workflow has a 50-step/840-second ceiling. Reaching a budget or output
|
||||||
|
limit fails explicitly; routing checkpoints cannot satisfy completion. UI
|
||||||
|
workflows require desktop and mobile captures matching the final local sources after
|
||||||
|
its last edit. Approval/brief-before-code and redesign documentation-at-finish
|
||||||
|
checks remain, as does exactly one context load across the completed turn.
|
||||||
|
CI runs this lane only when its manual `skill_workflow` checkbox is enabled.
|
||||||
|
Ordinary protocol CI now fetches its engine instead of silently skipping for
|
||||||
|
a missing binary. Full-build results must be reported separately from routing.
|
||||||
|
|
||||||
|
### Remaining gaps after the split (2026-09-07)
|
||||||
|
|
||||||
|
One provisioned Claude natural-build run reached a final response in 637 seconds
|
||||||
|
and 37 model steps, with approval, a surface brief, an implemented page, a
|
||||||
|
finish review, corrections, and fresh desktop/mobile screenshots. Its initial
|
||||||
|
completion assertions passed. The final test revision additionally requires
|
||||||
|
the shipped documentation reference; auditing the saved trace against that
|
||||||
|
guard found it missing. **This is not a final full-workflow pass.** Existing
|
||||||
|
DESIGN.md was preserved, but the required documentation pass was skipped.
|
||||||
|
The final guards were tightened during the run; this trace is not represented
|
||||||
|
as a run of those later assertions. No second full build was purchased.
|
||||||
|
|
||||||
|
Claude's remaining protocol batch passed S10–S15 and existing-project S16,
|
||||||
|
but missing-context S16 omitted routing.md and S17 omitted critique.md.
|
||||||
|
Both responses remained read-only. The older S9 timed out during unrelated
|
||||||
|
polishing; the corrected focused S9 above supersedes it. The batch was stopped
|
||||||
|
during the older S18, before another provider sweep. This is incremental
|
||||||
|
evidence, not an all-green final matrix. Release #782 remains on hold.
|
||||||
|
|
||||||
|
The full build reported about 2.95 million input and 53 thousand output tokens
|
||||||
|
across all turns (no cache usage reported). Keep this lane manually scoped;
|
||||||
|
the fast protocol suite is not a proxy for its completion or cost.
|
||||||
|
|
||||||
|
### Documentation handoff follow-up
|
||||||
|
|
||||||
|
`tests/skill-workflow/finish-handoff.test.mjs` isolates a synthetic post-review
|
||||||
|
checkpoint without rebuilding or capturing a page. Its existing-system fixture
|
||||||
|
has no approved system change and a pre-existing missing sidecar; the correct
|
||||||
|
result is to check the build against DESIGN.md, preserve it, and leave unrelated
|
||||||
|
drift alone. The new-world control must write DESIGN.md and its v2 sidecar.
|
||||||
|
|
||||||
|
The unchanged-instructions baseline reproduced the skipped documenter in 31s.
|
||||||
|
The revised handoff and documenter passages are 29 words shorter overall and
|
||||||
|
make a checked no-change result explicit. The first focused extension retest
|
||||||
|
passed in 27s. A repeat checked the source, DESIGN.md, and document.md and made
|
||||||
|
no mutations, but omitted degraded/documenter.md, so the strict reference guard
|
||||||
|
still failed. The new-world control passed in 91s, writing both required files.
|
||||||
|
These small samples support the narrower behavior change, not an all-green
|
||||||
|
workflow claim; the reference-loading miss remains visible. No full build was
|
||||||
|
rerun. Focused Claude routing S3/S4, the ordinary suite, source-first build, and
|
||||||
|
generated-skill authoring validation passed. Release #782 remains held.
|
||||||
|
|
||||||
|
### Outcome assertions (maintainer-approved follow-up)
|
||||||
|
|
||||||
|
S16/S17 now require a completed, useful, read-only answer; S17 distinguishes
|
||||||
|
assessment from implementation and explains that critique is optional before
|
||||||
|
polish. Explicit invented prerequisites remain failures. Missing routing or
|
||||||
|
comparison references are TAP diagnostics based on successful content loads,
|
||||||
|
not failed read attempts. These English-fixture phrase checks are bounded
|
||||||
|
regression checks, not a comprehensive semantic grader.
|
||||||
|
|
||||||
|
The resumed ordinary-extension checkpoint accepts a direct documentation pass
|
||||||
|
without the degraded wrapper only with actual document.md, page, and DESIGN.md
|
||||||
|
reads, a concrete no-change report grounded in the fixture's type/palette/layout,
|
||||||
|
and zero mutations. Seeded files must still be byte-identical and pre-existing
|
||||||
|
sidecar drift must remain untouched. New-world documentation writes, redesign
|
||||||
|
ordering, and full-build completion gates are unchanged.
|
||||||
|
|
||||||
|
Offline re-evaluation of saved Claude traces: three advice responses and two
|
||||||
|
evidenced no-op handoffs pass the new assertions. The original post-review
|
||||||
|
baseline and the 637-second full build still fail for absent documentation
|
||||||
|
evidence. Unit negative controls reject fabricated prerequisites, unsolicited
|
||||||
|
edits/interviews/scans, failed reads, empty or unsupported reports, and exhausted
|
||||||
|
budgets. This is assertion replay, not new model evidence or a rerun of cleaned-up
|
||||||
|
workspaces' filesystem checks. No paid calls or skill prose changes were needed.
|
||||||
|
The earlier reference misses above are now diagnostics, not release blockers by
|
||||||
|
themselves; this does not establish an all-green full-workflow matrix.
|
||||||
|
|
||||||
|
### Bounded release verification (2026-09-08)
|
||||||
|
|
||||||
|
The focused Anthropic live-accept test now verifies the durable session reaches
|
||||||
|
`completed`, not just clean source/DOM. Its agent instruction had recommended
|
||||||
|
`data-impeccable-e2e-variant`, inside the reserved runtime namespace; the test
|
||||||
|
prompt now uses a permanent `data-design-variant` styling hook. The strengthened
|
||||||
|
test passed in 14s without forced completion. The full non-billed suite passed.
|
||||||
|
No runtime or skill source was changed for this correction.
|
||||||
|
|
||||||
|
Claude post-review controls: new world passed in 77s, writing token-bearing
|
||||||
|
DESIGN.md and the v2 sidecar; approved redesign failed in 16s. The latter read the
|
||||||
|
page and old system, then wrote prose-only DESIGN.md without consulting the
|
||||||
|
documentation spec or creating `.impeccable/design.json`, and claimed nothing
|
||||||
|
remained outstanding. This is a missing required artifact, not merely missing
|
||||||
|
reference coverage. The test stays red; no retry was purchased. Release remains
|
||||||
|
held pending disposition. These are synthetic post-review checkpoints, not
|
||||||
|
proof of a complete redesign lifecycle.
|
||||||
|
|
||||||
|
The single full Claude build passed its automated gates in 623s / 31 steps:
|
||||||
|
approval, brief, implemented page, final desktop/mobile captures, and shipped
|
||||||
|
reviewer/documenter wrapper loads. The final response gave an in-thread review
|
||||||
|
and a no-change documentation assessment of the incumbent system. It did not
|
||||||
|
load document.md itself; this is not evidence that every documentation protocol
|
||||||
|
step ran. The run used about 2.11M input / 50K output tokens (no cache reported).
|
||||||
|
No further billed retry was started. The separate missing-artifact redesign
|
||||||
|
failure still blocks treating this batch as an all-green skill release gate.
|
||||||
|
|
||||||
|
### Targeted redesign handoff correction
|
||||||
|
|
||||||
|
The parent handoff now explicitly requires token-bearing DESIGN.md and
|
||||||
|
`.impeccable/design.json` for approved system changes and verifies those outputs
|
||||||
|
before completion. It is nine words shorter; the agent and schema files did not
|
||||||
|
change. One unchanged-assertion Claude retest finished in 65s: it read document.md
|
||||||
|
and wrote both artifacts, but still skipped the degraded wrapper, so that
|
||||||
|
reference assertion failed before the artifact assertions ran.
|
||||||
|
|
||||||
|
Wrapper coverage is now diagnostic for all post-review modes; successful spec
|
||||||
|
and source reads, completed turns, write boundaries, tokens, and the v2 sidecar
|
||||||
|
remain hard gates. Offline evaluation of the saved retest passes these artifact
|
||||||
|
checks; the original prose-only/missing-sidecar trace remains rejected. Negative
|
||||||
|
controls cover absent tokens, malformed/missing sidecars, old schema versions,
|
||||||
|
and absent metadata. This is replay, not a second live pass or a rerun of the
|
||||||
|
cleaned-up workspace's byte-preservation checks.
|
||||||
|
|
||||||
|
Artifact audit: the sidecar component renders correctly in an offline browser.
|
||||||
|
The heading line-height was recorded as 1.3 while the page inherits 1.6 (38.4px
|
||||||
|
at 24px), and generatedAt used a placeholder date. These are remaining output
|
||||||
|
accuracy limitations, distinct from the corrected missing-artifact failure;
|
||||||
|
the shape checks do not establish complete token fidelity. No broad provider or
|
||||||
|
full-build rerun was purchased. Build and generated-skill validation passed.
|
||||||
|
|
||||||
Each scenario:
|
Each scenario:
|
||||||
|
|
||||||
1. `prepareWorkspace()` mints a temp dir, symlinks the canonical skill
|
1. `prepareWorkspace()` uses the production transformer to build current source
|
||||||
into `<workspace>/.claude/skills/impeccable` (so its launcher is at
|
into an independent `<workspace>/.claude/skills/impeccable`. References have
|
||||||
`.claude/skills/impeccable/scripts/impeccable`), and optionally writes
|
resolved placeholders and generated degraded reviewer/documenter files.
|
||||||
`PRODUCT.md` / `DESIGN.md` fixtures.
|
Host-specific blocks are omitted: this is a neutral API harness, not an exact
|
||||||
|
Claude/Codex/Gemini host simulation. It optionally seeds project fixtures.
|
||||||
2. `runTurn()` inlines `SKILL.md` (placeholders neutralized) as the
|
2. `runTurn()` inlines `SKILL.md` (placeholders neutralized) as the
|
||||||
system prompt and runs Vercel AI SDK `generateText` with four
|
system prompt and runs Vercel AI SDK `generateText` with five
|
||||||
workspace-scoped tools: `bash`, `read`, `write`, `list`, and a fake
|
tools: `bash`, `read`, `write`, `list`, and a fake
|
||||||
provider-neutral `ask_user_question` backed by a deterministic simulated user.
|
provider-neutral `ask_user_question` backed by a deterministic simulated user.
|
||||||
3. The tools record every call into a `trace` that the test asserts on.
|
3. The tools record every call into a `trace` that the test asserts on.
|
||||||
4. For scenario 4, a second `runTurn` reuses turn 1's `responseMessages`
|
4. For scenario 4, a second `runTurn` reuses turn 1's `responseMessages`
|
||||||
@@ -53,6 +213,92 @@ Each scenario:
|
|||||||
|
|
||||||
The trace is the source of truth, not the model's free-form reply.
|
The trace is the source of truth, not the model's free-form reply.
|
||||||
|
|
||||||
|
File tools are workspace-scoped; bash is a real host shell, **not a security
|
||||||
|
sandbox**. Use disposable synthetic fixtures. Shell helpers do not inherit
|
||||||
|
provider API keys/auth tokens; model calls still use the parent's keys. The
|
||||||
|
harness always sets `IMPECCABLE_QUESTION_DISABLED=1` for shell calls so real
|
||||||
|
decision pages cannot wait for a nonexistent browser user. The engine returns
|
||||||
|
its genuine structured-question fallback; browser decisions have separate E2E.
|
||||||
|
|
||||||
|
Set `IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR=<directory>` to retain per-turn JSON
|
||||||
|
with model, prompt, tool results, response ordering, usage, and finish reason.
|
||||||
|
Progress and failed turns retain their tool traces too; only completed turns
|
||||||
|
carry the full response sequence and final usage.
|
||||||
|
These are local diagnostic artifacts; inspect before sharing. Successful reads
|
||||||
|
or full reference content in shell output count as loading; filename mentions,
|
||||||
|
denied commands, and failed reads do not.
|
||||||
|
|
||||||
|
Context-only controls permit the real launcher with an optional workspace-relative
|
||||||
|
`--target`; compound commands remain rejected. The target form is part of the
|
||||||
|
skill's Setup contract, not a launcher failure.
|
||||||
|
|
||||||
|
## Release investigation (2026-09-07)
|
||||||
|
|
||||||
|
The initial release sweep reported 68/81 passes. Do not interpret its 13 failed
|
||||||
|
assertions as 13 demonstrated product regressions. The harness staged raw
|
||||||
|
references with unresolved placeholders, omitted generated degraded roles, and
|
||||||
|
allowed unanswered browser decisions. Its redesign assertion was also stale:
|
||||||
|
current `new-work.md` requires a surface brief **before code**, and DESIGN.md
|
||||||
|
**at finish**, from the built world. The corrected lifecycle checks retain
|
||||||
|
approval, brief, implementation, and final documentation requirements; missing
|
||||||
|
artifacts now have their own error instead of being called premature edits.
|
||||||
|
The historical tables below retain their original measurements and methods.
|
||||||
|
|
||||||
|
Focused launcher-fallback verification on the corrected fixture:
|
||||||
|
|
||||||
|
| Default model | Old fallback paragraph | Explicit pre-tool warning paragraph |
|
||||||
|
|---|---:|---:|
|
||||||
|
| `claude-sonnet-5` | 2/3 | 3/3 |
|
||||||
|
| `gpt-5.6-terra` | 3/3 | 3/3 |
|
||||||
|
| `gemini-3.7-flash` | 1/3 | 3/3 |
|
||||||
|
|
||||||
|
The three cases are denied editing, successful-loader control, and denied
|
||||||
|
planning. The old-paragraph failures were warning order, not refused edits.
|
||||||
|
An intermediate candidate run scored 8/9 because the control rejected valid
|
||||||
|
`context --target index.html`; after correcting that allowlist, the full focused
|
||||||
|
rerun passed 9/9. This is one measured run per variant, not a reliability estimate
|
||||||
|
or an all-workflow pass. Broader routing and workflow results remain separate.
|
||||||
|
|
||||||
|
On the resolved fixture, Gemini's workflow run passed 4/5: the completed new
|
||||||
|
page omitted user confirmation. Tightening the existing question paragraph
|
||||||
|
made its focused build lifecycle rerun pass 1/1. OpenAI passed all five workflow
|
||||||
|
cases with the fixture corrections alone. These are incremental measurements,
|
||||||
|
not one full sweep on the final candidate.
|
||||||
|
|
||||||
|
Claude's three-step routing sweep cut off two setup cases before loading
|
||||||
|
`new-work.md`. Both loaded it in bounded six-step diagnostics. Claude now has
|
||||||
|
the same six-step setup allowance as Gemini; reference and edit-order assertions
|
||||||
|
are unchanged. A full-build baseline separately hit the existing 840-second
|
||||||
|
deadline and is not counted as a pass.
|
||||||
|
|
||||||
|
The complete routing-only sweep retained its original budgets and passed 55/57
|
||||||
|
(Claude 17/19, OpenAI 19/19, Gemini 19/19). A six-step Claude rerun passed the two
|
||||||
|
previously clipped cases but exposed a separate context reload on turn two:
|
||||||
|
the model queried the loader again for image-tool availability. That run was
|
||||||
|
stopped after five passes and this failure rather than finishing another billed
|
||||||
|
sweep; the once-per-session assertion remains unchanged.
|
||||||
|
|
||||||
|
A saved Claude build trace used five calls shortening/counting the direction
|
||||||
|
contract, then reached the 22-step cap before implementation. The word target
|
||||||
|
is now approximate, with the six required blocks retained. A subsequent run
|
||||||
|
correctly stopped for missing customer evidence: the default simulated user had
|
||||||
|
selected “I have real details,” then promised them in a future message. The
|
||||||
|
case-study test now supplies a complete, explicitly synthetic brief when asked;
|
||||||
|
fresh-init and other user simulations are unchanged. Neither incomplete build
|
||||||
|
is counted as a pass.
|
||||||
|
|
||||||
|
The corrected-user Claude retest also remained incomplete: it asked, recorded
|
||||||
|
the six-block brief without the earlier word-count loop, then spent the remaining
|
||||||
|
22-step allowance acquiring and inspecting fonts before writing HTML. The
|
||||||
|
26-step redesign run produced the page and desktop/mobile captures but stopped
|
||||||
|
before DESIGN.md. These results do not establish full workflow completion.
|
||||||
|
Further work should separate narrow protocol checks from realistic, provisioned
|
||||||
|
full-build runs rather than keep adding skill prose or relaxing finish gates.
|
||||||
|
The later Claude run passed fresh init and refinement, failed the two bounded
|
||||||
|
build cases, and was stopped during critique's browser-tool discovery. Its
|
||||||
|
unfinished critique case is not a pass; the earlier OpenAI/Gemini critique
|
||||||
|
results remain the completed measurements.
|
||||||
|
|
||||||
## Scenarios
|
## Scenarios
|
||||||
|
|
||||||
| # | Setup | Assertion |
|
| # | Setup | Assertion |
|
||||||
@@ -72,8 +318,8 @@ The trace is the source of truth, not the model's free-form reply.
|
|||||||
| 13 | empty workspace; prompt is `/impeccable teach` | runs `impeccable context` and diverts into `reference/init.md` because `teach` aliases `init` |
|
| 13 | empty workspace; prompt is `/impeccable teach` | runs `impeccable context` and diverts into `reference/init.md` because `teach` aliases `init` |
|
||||||
| 14 | PRODUCT.md with `## Platform: ios` (native iOS app); prompt is `/impeccable craft a tide detail screen` | `impeccable context` runs and emits the contents of `reference/ios.md` directly, placing native conventions in context without a second model-directed read |
|
| 14 | PRODUCT.md with `## Platform: ios` (native iOS app); prompt is `/impeccable craft a tide detail screen` | `impeccable context` runs and emits the contents of `reference/ios.md` directly, placing native conventions in context without a second model-directed read |
|
||||||
| 15 | same iOS fixture; prompt is `/impeccable audit` | agent loads `reference/audit.native.md` (the Commands-table native variant, routed instead of `audit.md`) |
|
| 15 | same iOS fixture; prompt is `/impeccable audit` | agent loads `reference/audit.native.md` (the Commands-table native variant, routed instead of `audit.md`) |
|
||||||
| 16 | existing surface, with and without PRODUCT.md; asks where to start | loads `routing.md`, delivers advice, and does not edit project files, start an interview, archive a critique, or run menu scans |
|
| 16 | existing surface, with and without PRODUCT.md; asks where to start | completes relevant advice without edits, interviews, critique archives, menu scans, or explicit invented refinement prerequisites; reference coverage is diagnostic |
|
||||||
| 17 | existing surface; asks whether critique is required before polish | loads `routing.md` and both command references, then delivers advice without executing the playbooks |
|
| 17 | existing surface; asks whether critique is required before polish | completes read-only advice distinguishing assessment from implementation and explaining critique is optional; reference coverage is diagnostic |
|
||||||
| 18 | existing surface; explicitly requests polish followed by a next-command recommendation | loads `polish.md` rather than substituting workflow advice for the requested work |
|
| 18 | existing surface; explicitly requests polish followed by a next-command recommendation | loads `polish.md` rather than substituting workflow advice for the requested work |
|
||||||
| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, and a denied-launcher planning-only case | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor |
|
| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, and a denied-launcher planning-only case | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor |
|
||||||
|
|
||||||
@@ -188,7 +434,7 @@ evidence, not rejected bash reads. The initial unrestricted run (stopped after
|
|||||||
host-wide search attempts), earlier rejected-read results, and broader suite's
|
host-wide search attempts), earlier rejected-read results, and broader suite's
|
||||||
sandboxed provider DNS errors are excluded from this baseline.
|
sandboxed provider DNS errors are excluded from this baseline.
|
||||||
|
|
||||||
The workflow-contract file adds end-to-end assertions for attended fresh init,
|
The full-build file adds end-to-end assertions for attended fresh init,
|
||||||
an initialized natural build request, replacement-world redesign, scope-preserving bolder
|
an initialized natural build request, replacement-world redesign, scope-preserving bolder
|
||||||
refinement, and critique's closing question. It checks question order and
|
refinement, and critique's closing question. It checks question order and
|
||||||
context/artifact writes rather than only reference-file loading.
|
context/artifact writes rather than only reference-file loading.
|
||||||
@@ -365,12 +611,12 @@ when bisecting one scenario:
|
|||||||
```bash
|
```bash
|
||||||
IMPECCABLE_QUESTION_DISABLED=1 CI=1 IMPECCABLE_SKILL_BEHAVIOR_MODELS=deepseek-v4-flash \
|
IMPECCABLE_QUESTION_DISABLED=1 CI=1 IMPECCABLE_SKILL_BEHAVIOR_MODELS=deepseek-v4-flash \
|
||||||
node --test --test-timeout=300000 --test-force-exit \
|
node --test --test-timeout=300000 --test-force-exit \
|
||||||
--test-name-pattern="bolder refinement" tests/skill-behavior/workflow-contract.test.mjs
|
--test-name-pattern="bolder refinement" tests/skill-workflow/full-build.test.mjs
|
||||||
```
|
```
|
||||||
|
|
||||||
Keep `--test-timeout` at 300000. A tighter cap turns claude-sonnet-5's slower
|
Use the suite's current 900000ms timeout for full workflow cases; the 300000ms
|
||||||
runs into timeouts that look like failures. Set `IMPECCABLE_QUESTION_DISABLED=1`
|
example above is historical. The harness now disables decision pages itself.
|
||||||
and `CI=1` so `impeccable serve-question` cannot open a browser window on the host. Pipe
|
Pipe
|
||||||
to a file rather than `tail`; node prints the failing-test summary at the end,
|
to a file rather than `tail`; node prints the failing-test summary at the end,
|
||||||
and truncating it costs you the per-model attribution.
|
and truncating it costs you the per-model attribution.
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,64 @@
|
|||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
// These are bounded English-fixture checks, not a general semantic grader.
|
||||||
|
// Reference coverage is reported separately: opening a file proves neither
|
||||||
|
// useful advice nor permission to execute it.
|
||||||
|
export function missingReferences(trace, filenames) {
|
||||||
|
return filenames.filter((filename) => !trace.toolCalls.some((call) =>
|
||||||
|
(call.loadedFiles || []).some((file) => file === filename || file.endsWith(`/${filename}`))));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function assertAdviceOnly(trace, text) {
|
||||||
|
assert.ok(text.trim(), 'advice must reach the user, not stop at reference loading');
|
||||||
|
assert.deepEqual(trace.writePaths, [], 'advice must not use the write tool');
|
||||||
|
const mutations = trace.toolCalls.flatMap((call) => call.mutatedPaths ?? [])
|
||||||
|
.filter((file) => !file.startsWith('.impeccable/') || file.startsWith('.impeccable/critique/'));
|
||||||
|
assert.deepEqual(mutations, [], 'advice must not edit project files or archive an unsolicited critique');
|
||||||
|
assert.deepEqual(trace.questionCalls, [], 'advice must not start an init or design interview');
|
||||||
|
assert.ok(!trace.bashCommands.some((command) => command.includes('impeccable detect')), 'workflow advice does not run menu scans');
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizedAdvice(text) {
|
||||||
|
return text.replace(/[`*_]/g, '').replace(/[’]/g, "'").toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function assertWorkflowAdvice(trace, text, { missingContext = false } = {}) {
|
||||||
|
assertAdviceOnly(trace, text);
|
||||||
|
const advice = normalizedAdvice(text);
|
||||||
|
assert.match(advice, /index\.html/, 'advice should address the existing surface');
|
||||||
|
assert.match(advice, missingContext ? /\binit\b/ : /\b(?:critique|audit|polish)\b/, 'advice must recommend a relevant starting point');
|
||||||
|
if (missingContext) assert.match(advice, /\bdocument\b/, 'advice should explain how to record the existing identity');
|
||||||
|
assert.doesNotMatch(advice, /(?:must|need to|have to|required to)\s+(?:run\s+)?(?:\/impeccable\s+)?(?:init|document)\b[^.!?\n]{0,100}\bbefore\s+(?:you\s+can\s+)?(?:run(?:ning)?\s+)?(?:polish(?:ing)?|refin(?:e|ing|ement))\b|(?:polish|refinement)\s+(?:requires|is blocked by|cannot run without)\s+(?:init|document|product\.md|design\.md)/,
|
||||||
|
'setup is not a mandatory prerequisite for narrow refinement');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function assertCommandComparison(trace, text) {
|
||||||
|
assertAdviceOnly(trace, text);
|
||||||
|
const advice = normalizedAdvice(text);
|
||||||
|
assert.match(advice, /critique[^.!?\n]{0,120}(?:review|assess|evaluat|report|findings)/, 'comparison must explain critique as assessment');
|
||||||
|
assert.match(advice, /polish[^.!?\n]{0,120}(?:fix|refin|implement|edit)/, 'comparison must explain polish as implementation');
|
||||||
|
assert.match(advice, /critique\s+(?:is\s+)?(?:isn't|is not|not)\s+(?:required|necessary)|critique[^.!?\n]{0,50}\boptional\b|polish[^.!?\n]{0,100}(?:directly|without\s+(?:a\s+)?critique|independent)/,
|
||||||
|
'comparison must explain that critique is optional before polish');
|
||||||
|
assert.doesNotMatch(advice, /(?:must|need to|have to)\s+(?:run\s+)?critique[^.!?\n]{0,80}before\s+(?:run(?:ning)?\s+)?polish|critique\s+(?:is\s+)?(?:required|mandatory|necessary)\s+before\s+polish|polish\s+(?:requires|cannot run without)\s+(?:a\s+)?critique/,
|
||||||
|
'comparison must not invent a critique prerequisite');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function assertNewWorkLifecycle(trace, { target, redesign = false }) {
|
||||||
|
const calls = trace.toolCalls;
|
||||||
|
const writes = (call, file) => (call.mutatedPaths || []).includes(file);
|
||||||
|
const implementation = calls.findIndex((call) => writes(call, target));
|
||||||
|
const question = calls.findIndex((call) => call.name === 'ask_user_question');
|
||||||
|
const brief = calls.findIndex((call) => (call.mutatedPaths || []).some((file) => file.startsWith('.impeccable/surfaces/')));
|
||||||
|
assert.ok(implementation >= 0, `new-work did not produce the requested artifact: ${target}`);
|
||||||
|
assert.ok(question >= 0 && question < implementation, 'implementation must follow a user answer');
|
||||||
|
assert.ok(brief >= 0 && brief < implementation, 'the direction contract must be recorded in a surface brief before implementation');
|
||||||
|
if (redesign) {
|
||||||
|
const lastImplementation = calls.findLastIndex((call) => writes(call, target));
|
||||||
|
const documentation = calls.findLastIndex((call) => writes(call, 'DESIGN.md'));
|
||||||
|
assert.ok(documentation > lastImplementation, 'redesign must record DESIGN.md from the finished build, after the last page edit');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export const LAUNCHER_FAILURE_WARNING = /(?:context|launcher|bash)[^.!?\n]{0,160}(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|did(?:n't| not)|fail|unable)|(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|unable)[^.!?\n]{0,160}(?:context|launcher|bash)/i;
|
export const LAUNCHER_FAILURE_WARNING = /(?:context|launcher|bash)[^.!?\n]{0,160}(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|did(?:n't| not)|fail|unable)|(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|unable)[^.!?\n]{0,160}(?:context|launcher|bash)/i;
|
||||||
|
|
||||||
export function assertPlanningFallbackWarning(responseMessages) {
|
export function assertPlanningFallbackWarning(responseMessages) {
|
||||||
|
|||||||
@@ -5,6 +5,9 @@
|
|||||||
* uses to decide whether to gate on `init`. Plausible enough that the agent
|
* uses to decide whether to gate on `init`. Plausible enough that the agent
|
||||||
* treats them as real context rather than test scaffolding.
|
* treats them as real context rather than test scaffolding.
|
||||||
*/
|
*/
|
||||||
|
// This user can supply a complete synthetic case now, not promise future assets.
|
||||||
|
export const CASE_STUDY_ANSWER = 'Build a standalone index.html for the researchers described in PRODUCT.md, preserving DESIGN.md. Use a clearly labeled synthetic case: a doctoral researcher reconstructs why a literature-review conclusion changed by following linked notes and citations. No real customer names, quotes, metrics, or assets are available; author illustrative content and label it, with no invented commercial claims. The reader should understand the preserved reasoning trail and follow an in-page link to the method. For a composition or concept choice, use the first direction you presented. No additional material is coming from me.';
|
||||||
|
|
||||||
export const PRODUCT_MD_SAMPLE = `# Acme Notes
|
export const PRODUCT_MD_SAMPLE = `# Acme Notes
|
||||||
|
|
||||||
## Platform
|
## Platform
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
/**
|
/**
|
||||||
* Sandboxed scenario runner for skill-behavior tests.
|
* Synthetic-workspace scenario runner for skill-behavior tests.
|
||||||
*
|
*
|
||||||
* Each scenario:
|
* Each scenario:
|
||||||
* 1. Creates a temp workspace.
|
* 1. Creates a temp workspace.
|
||||||
* 2. Symlinks the real .claude/skills/impeccable into the workspace so
|
* 2. Builds a neutral .claude/skills/impeccable into the workspace so
|
||||||
* the launcher (`scripts/impeccable`) resolves from the canonical path
|
* the launcher (`scripts/impeccable`) resolves from the canonical path
|
||||||
* the skill references, and points it at an engine binary.
|
* the skill references, and points it at an engine binary.
|
||||||
* 3. Optionally writes PRODUCT.md / DESIGN.md fixtures.
|
* 3. Optionally writes PRODUCT.md / DESIGN.md fixtures.
|
||||||
@@ -15,8 +15,8 @@
|
|||||||
* messages (so multi-turn scenarios can append to them).
|
* messages (so multi-turn scenarios can append to them).
|
||||||
*
|
*
|
||||||
* The harness deliberately mirrors the live-mode E2E pattern: real LLM,
|
* The harness deliberately mirrors the live-mode E2E pattern: real LLM,
|
||||||
* no mocks, but tightly bounded execution surface so we observe the routing
|
* no mocked model. File tools are workspace-scoped; bash is a real host shell,
|
||||||
* behavior of the skill without paying for full-fledged design work.
|
* not a security sandbox. Run only against disposable synthetic fixtures.
|
||||||
*/
|
*/
|
||||||
import { generateText, stepCountIs, tool } from 'ai';
|
import { generateText, stepCountIs, tool } from 'ai';
|
||||||
import { z } from 'zod';
|
import { z } from 'zod';
|
||||||
@@ -28,12 +28,35 @@ import { spawn } from 'node:child_process';
|
|||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
import { getProviderOptions } from './providers.mjs';
|
import { getProviderOptions } from './providers.mjs';
|
||||||
import { ENGINE_MISSING_MESSAGE, findEngineBinary } from '../lib/engine-bin.mjs';
|
import { ENGINE_MISSING_MESSAGE, findEngineBinary } from '../lib/engine-bin.mjs';
|
||||||
|
import { readSourceFiles, compileProviderBlocks, replacePlaceholders, stripRuleMarkers } from '../../scripts/lib/utils.js';
|
||||||
|
import { createTransformer } from '../../scripts/lib/transformers/factory.js';
|
||||||
|
|
||||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||||
const REPO_ROOT = path.resolve(__dirname, '..', '..');
|
const REPO_ROOT = path.resolve(__dirname, '..', '..');
|
||||||
const SKILL_SOURCE_DIR = path.join(REPO_ROOT, 'skill');
|
|
||||||
const MAX_BASH_OUTPUT_BYTES = 200_000;
|
const MAX_BASH_OUTPUT_BYTES = 200_000;
|
||||||
|
|
||||||
|
function renderNeutral(content) {
|
||||||
|
return stripRuleMarkers(replacePlaceholders(compileProviderBlocks(content, [])
|
||||||
|
.replaceAll('{{ask_instruction}}', 'Use the ask_user_question tool.')
|
||||||
|
.replaceAll('{{model}}', 'the assistant'), 'dsh'))
|
||||||
|
.replaceAll('{{scripts_path}}', '.claude/skills/impeccable/scripts')
|
||||||
|
.replaceAll('{{command_hint}}', 'command');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use the production builder so fallback reviewer/documenter references exist.
|
||||||
|
// Generic tool names are shared by the API providers; host-specific blocks are
|
||||||
|
// deliberately absent. Exact provider transforms have separate loader tests.
|
||||||
|
const sourceSkills = readSourceFiles(REPO_ROOT).skills.map((skill) => ({
|
||||||
|
...skill,
|
||||||
|
body: renderNeutral(skill.body),
|
||||||
|
references: skill.references.map((ref) => ({ ...ref, content: renderNeutral(ref.content) })),
|
||||||
|
agents: skill.agents.map((agent) => ({ ...agent, body: renderNeutral(agent.body) })),
|
||||||
|
}));
|
||||||
|
const stageSkill = createTransformer({
|
||||||
|
provider: 'skill-behavior', placeholderProvider: 'dsh', providerTags: [],
|
||||||
|
configDir: '.claude', displayName: 'Behavior fixture',
|
||||||
|
});
|
||||||
|
|
||||||
function snapshotWorkspaceFiles(root) {
|
function snapshotWorkspaceFiles(root) {
|
||||||
const snapshot = new Map();
|
const snapshot = new Map();
|
||||||
const walk = (dir, relDir = '') => {
|
const walk = (dir, relDir = '') => {
|
||||||
@@ -66,24 +89,7 @@ function changedPaths(before, after) {
|
|||||||
* is provider-neutral when inlined.
|
* is provider-neutral when inlined.
|
||||||
*/
|
*/
|
||||||
function loadSkillBody() {
|
function loadSkillBody() {
|
||||||
let md = fs.readFileSync(path.join(SKILL_SOURCE_DIR, 'SKILL.src.md'), 'utf8');
|
return sourceSkills[0].body.trim();
|
||||||
// Strip frontmatter.
|
|
||||||
if (md.startsWith('---')) {
|
|
||||||
const end = md.indexOf('\n---', 3);
|
|
||||||
if (end !== -1) md = md.slice(end + 4).trimStart();
|
|
||||||
}
|
|
||||||
// The source uses placeholders that the build step replaces per-provider.
|
|
||||||
// For the test harness we want a single body that works for any provider,
|
|
||||||
// and the scripts the skill references live at .claude/skills/impeccable/
|
|
||||||
// (the workspace symlink), so hard-code those values.
|
|
||||||
md = md
|
|
||||||
.replaceAll('{{model}}', 'the assistant')
|
|
||||||
.replaceAll('{{command_prefix}}', '/')
|
|
||||||
.replaceAll('{{ask_instruction}}', 'Use the ask_user_question tool.')
|
|
||||||
.replaceAll('{{config_file}}', 'AGENTS.md')
|
|
||||||
.replaceAll('{{scripts_path}}', '.claude/skills/impeccable/scripts')
|
|
||||||
.replaceAll('{{command_hint}}', 'command');
|
|
||||||
return md.trim();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// This provider-neutral fixture assumes a loaded skill with a known base
|
// This provider-neutral fixture assumes a loaded skill with a known base
|
||||||
@@ -96,36 +102,26 @@ export const SKILL_BODY = `Base directory for this skill (workspace-relative): .
|
|||||||
/**
|
/**
|
||||||
* Create a temp workspace and prepopulate it.
|
* Create a temp workspace and prepopulate it.
|
||||||
*
|
*
|
||||||
* - `.claude/skills/impeccable` is symlinked at the SOURCE skill dir (not
|
* - Compile current source into an independent fixture distribution. Shell
|
||||||
* the built `.claude/skills/impeccable/`) so the test exercises whatever
|
* and read tools see the same resolved references, including degraded roles.
|
||||||
* is in `skill/` right now, without needing `bun run build` to refresh
|
|
||||||
* the harness output dirs. The trade-off: reference files surface their
|
|
||||||
* raw `{{placeholders}}`, but the assertions only check tool calls, not
|
|
||||||
* their content.
|
|
||||||
* - `files` lets the test seed PRODUCT.md / DESIGN.md (or anything else).
|
* - `files` lets the test seed PRODUCT.md / DESIGN.md (or anything else).
|
||||||
* - `skillVersion` switches from symlink to a real COPY of the skill dir and
|
* - `skillVersion` adds a `SKILL.md` version. `impeccable context` reads its
|
||||||
* writes a `SKILL.md` carrying that version. `impeccable context` reads its
|
|
||||||
* own version from that sibling file, so this is required for any scenario
|
* own version from that sibling file, so this is required for any scenario
|
||||||
* that exercises the update-check path (the source dir has only SKILL.src.md).
|
* that exercises the update-check path (the source dir has only SKILL.src.md).
|
||||||
*
|
*
|
||||||
* The launcher in the staged scripts dir needs an engine binary. Every bash
|
* The launcher in the staged scripts dir needs an engine binary. Every bash
|
||||||
* call the agent makes gets `IMPECCABLE_BIN` (tests/lib/engine-bin.mjs:
|
* call the agent makes gets `IMPECCABLE_BIN` (tests/lib/engine-bin.mjs:
|
||||||
* `IMPECCABLE_BIN` or `skill/scripts/bin/<os>-<arch>/`), which the launcher
|
* `IMPECCABLE_BIN` or `skill/scripts/bin/<os>-<arch>/`), which the launcher
|
||||||
* honors first, so the symlink and copy modes both work without a download.
|
* honors first, so the staged skill works without a download.
|
||||||
*/
|
*/
|
||||||
export const ENGINE_BIN = findEngineBinary();
|
export const ENGINE_BIN = findEngineBinary();
|
||||||
export { ENGINE_MISSING_MESSAGE };
|
export { ENGINE_MISSING_MESSAGE };
|
||||||
|
|
||||||
export function prepareWorkspace({ files = {}, skillVersion = null } = {}) {
|
export function prepareWorkspace({ files = {}, skillVersion = null } = {}) {
|
||||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-skill-test-'));
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-skill-test-'));
|
||||||
const skillDest = path.join(dir, '.claude', 'skills', 'impeccable');
|
stageSkill(sourceSkills, dir, { skillsVersion: skillVersion || '' });
|
||||||
fs.mkdirSync(path.join(dir, '.claude', 'skills'), { recursive: true });
|
fs.renameSync(path.join(dir, 'skill-behavior', '.claude'), path.join(dir, '.claude'));
|
||||||
if (skillVersion) {
|
fs.rmdirSync(path.join(dir, 'skill-behavior'));
|
||||||
fs.cpSync(SKILL_SOURCE_DIR, skillDest, { recursive: true });
|
|
||||||
fs.writeFileSync(path.join(skillDest, 'SKILL.md'), `---\nname: impeccable\nversion: ${skillVersion}\n---\n\nbody\n`);
|
|
||||||
} else {
|
|
||||||
fs.symlinkSync(SKILL_SOURCE_DIR, skillDest, 'dir');
|
|
||||||
}
|
|
||||||
for (const [name, contents] of Object.entries(files)) {
|
for (const [name, contents] of Object.entries(files)) {
|
||||||
const target = path.join(dir, name);
|
const target = path.join(dir, name);
|
||||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||||
@@ -154,14 +150,43 @@ function safeResolve(root, userPath) {
|
|||||||
if (rel.startsWith('..') || rel.split(path.sep).includes('..')) {
|
if (rel.startsWith('..') || rel.split(path.sep).includes('..')) {
|
||||||
return { error: 'path escapes the workspace' };
|
return { error: 'path escapes the workspace' };
|
||||||
}
|
}
|
||||||
return resolved;
|
try {
|
||||||
|
// New write targets need not exist; validate their nearest existing
|
||||||
|
// ancestor, including dangling links, before appending the missing suffix.
|
||||||
|
let ancestor = resolved;
|
||||||
|
while (!fs.existsSync(ancestor)) {
|
||||||
|
if (fs.lstatSync(ancestor, { throwIfNoEntry: false })?.isSymbolicLink()) {
|
||||||
|
return { error: 'path follows a dangling symlink' };
|
||||||
|
}
|
||||||
|
ancestor = path.dirname(ancestor);
|
||||||
|
}
|
||||||
|
const canonical = path.resolve(fs.realpathSync(ancestor), path.relative(ancestor, resolved));
|
||||||
|
const realRel = path.relative(fs.realpathSync(root), canonical);
|
||||||
|
if (realRel === '..' || realRel.startsWith(`..${path.sep}`) || path.isAbsolute(realRel)) {
|
||||||
|
return { error: 'path escapes the workspace through a symlink' };
|
||||||
|
}
|
||||||
|
return canonical;
|
||||||
|
} catch {
|
||||||
|
return { error: 'path cannot be resolved safely' };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isContextOnlyCommand(workspace, command) {
|
||||||
|
const match = command.trim().match(/^\.claude\/skills\/impeccable\/scripts\/impeccable context(?: --target(?: |=)(?:"([a-zA-Z0-9_./+ -]+)"|'([a-zA-Z0-9_./+ -]+)'|([a-zA-Z0-9_./+-]+)))?$/);
|
||||||
|
if (!match) return false;
|
||||||
|
const target = match[1] ?? match[2] ?? match[3];
|
||||||
|
return target === undefined || (!target.startsWith('-') && typeof safeResolve(workspace, target) === 'string');
|
||||||
}
|
}
|
||||||
|
|
||||||
function execBash(workspace, command, timeoutMs = 20_000, extraEnv = {}) {
|
function execBash(workspace, command, timeoutMs = 20_000, extraEnv = {}) {
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
|
// Model credentials belong to generateText, not to child image helpers.
|
||||||
|
// Real decision pages have browser E2E; this suite has a structured user.
|
||||||
|
const shellEnv = Object.fromEntries(Object.entries({ ...process.env, ...extraEnv })
|
||||||
|
.filter(([name]) => !/(?:^|_)(?:API_KEY|AUTH_TOKEN|ACCESS_TOKEN)$/.test(name)));
|
||||||
const proc = spawn('bash', ['-lc', command], {
|
const proc = spawn('bash', ['-lc', command], {
|
||||||
cwd: workspace,
|
cwd: workspace,
|
||||||
env: { ...process.env, ...(ENGINE_BIN ? { IMPECCABLE_BIN: ENGINE_BIN } : {}), ...extraEnv },
|
env: { ...shellEnv, ...(ENGINE_BIN ? { IMPECCABLE_BIN: ENGINE_BIN } : {}), IMPECCABLE_QUESTION_DISABLED: '1' },
|
||||||
});
|
});
|
||||||
let stdout = '';
|
let stdout = '';
|
||||||
let stderr = '';
|
let stderr = '';
|
||||||
@@ -225,6 +250,10 @@ function defaultSimulatedAnswer(question) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contextOnlyBash = false, denyBash = false } = {}) {
|
export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contextOnlyBash = false, denyBash = false } = {}) {
|
||||||
|
const referenceDir = path.join(workspace, '.claude/skills/impeccable/reference');
|
||||||
|
const references = fs.readdirSync(referenceDir, { recursive: true })
|
||||||
|
.filter((file) => file.endsWith('.md'))
|
||||||
|
.map((file) => ({ file: file.split(path.sep).join('/'), content: fs.readFileSync(path.join(referenceDir, file), 'utf8').trim() }));
|
||||||
const trace = {
|
const trace = {
|
||||||
toolCalls: [],
|
toolCalls: [],
|
||||||
bashCommands: [],
|
bashCommands: [],
|
||||||
@@ -248,7 +277,7 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
|
|||||||
const tools = {
|
const tools = {
|
||||||
bash: tool({
|
bash: tool({
|
||||||
description: contextOnlyBash
|
description: contextOnlyBash
|
||||||
? 'Only `.claude/skills/impeccable/scripts/impeccable context` is allowed here. Use read/list for workspace files and skill references; write remains available for requested edits.'
|
? 'Only `.claude/skills/impeccable/scripts/impeccable context` with an optional `--target <workspace-relative path>` is allowed here. Use read/list for files and references; write remains available for requested edits.'
|
||||||
: 'Run a bash command in the workspace root. Use this to invoke skill commands (e.g. `.claude/skills/impeccable/scripts/impeccable context`).',
|
: 'Run a bash command in the workspace root. Use this to invoke skill commands (e.g. `.claude/skills/impeccable/scripts/impeccable context`).',
|
||||||
inputSchema: z.object({
|
inputSchema: z.object({
|
||||||
command: z.string().describe('The bash command to execute.'),
|
command: z.string().describe('The bash command to execute.'),
|
||||||
@@ -265,14 +294,16 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
|
|||||||
}
|
}
|
||||||
// Routing tests need the real context loader, not a general-purpose
|
// Routing tests need the real context loader, not a general-purpose
|
||||||
// shell on the host. Reject before execution (still record attempts).
|
// shell on the host. Reject before execution (still record attempts).
|
||||||
if (contextOnlyBash && command.trim() !== '.claude/skills/impeccable/scripts/impeccable context') {
|
if (contextOnlyBash && !isContextOnlyCommand(workspace, command)) {
|
||||||
const out = 'Error: only `.claude/skills/impeccable/scripts/impeccable context` is allowed. Use read/list for files; references live at .claude/skills/impeccable/reference/.';
|
const out = 'Error: only `.claude/skills/impeccable/scripts/impeccable context` with an optional workspace-relative `--target` is allowed. Use read/list for files; references live at .claude/skills/impeccable/reference/.';
|
||||||
trace.bashOutputs.push(out);
|
trace.bashOutputs.push(out);
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
const before = snapshotWorkspaceFiles(workspace);
|
const before = snapshotWorkspaceFiles(workspace);
|
||||||
const res = await execBash(workspace, command, 20_000, extraEnv);
|
const res = await execBash(workspace, command, 20_000, extraEnv);
|
||||||
call.mutatedPaths = changedPaths(before, snapshotWorkspaceFiles(workspace));
|
call.mutatedPaths = changedPaths(before, snapshotWorkspaceFiles(workspace));
|
||||||
|
call.loadedFiles = references.filter(({ content }) => content && res.stdout.includes(content))
|
||||||
|
.map(({ file }) => `.claude/skills/impeccable/reference/${file}`);
|
||||||
const head = `exit=${res.exitCode}`;
|
const head = `exit=${res.exitCode}`;
|
||||||
const body = (res.stdout ? `stdout:\n${res.stdout}` : '') + (res.stderr ? `\nstderr:\n${res.stderr}` : '');
|
const body = (res.stdout ? `stdout:\n${res.stdout}` : '') + (res.stderr ? `\nstderr:\n${res.stderr}` : '');
|
||||||
const out = `${head}\n${body}${res.truncated ? '\n[output truncated]' : ''}`;
|
const out = `${head}\n${body}${res.truncated ? '\n[output truncated]' : ''}`;
|
||||||
@@ -295,6 +326,7 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
|
|||||||
if (stat.isDirectory()) return `Path is a directory: ${p}. Use list instead.`;
|
if (stat.isDirectory()) return `Path is a directory: ${p}. Use list instead.`;
|
||||||
const contents = fs.readFileSync(resolved, 'utf8');
|
const contents = fs.readFileSync(resolved, 'utf8');
|
||||||
call.succeeded = true;
|
call.succeeded = true;
|
||||||
|
call.loadedFiles = [p];
|
||||||
return contents;
|
return contents;
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
@@ -308,7 +340,7 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
|
|||||||
const call = record('write', { path: p, contents });
|
const call = record('write', { path: p, contents });
|
||||||
const resolved = safeResolve(workspace, p);
|
const resolved = safeResolve(workspace, p);
|
||||||
if (typeof resolved !== 'string') return `Error: ${resolved.error}`;
|
if (typeof resolved !== 'string') return `Error: ${resolved.error}`;
|
||||||
if ((contextOnlyBash || denyBash) && path.relative(workspace, resolved).split(path.sep)[0] === '.claude') {
|
if (path.relative(fs.realpathSync(workspace), resolved).split(path.sep)[0] === '.claude') {
|
||||||
return 'Error: the staged skill is read-only; edits must target project files.';
|
return 'Error: the staged skill is read-only; edits must target project files.';
|
||||||
}
|
}
|
||||||
fs.mkdirSync(path.dirname(resolved), { recursive: true });
|
fs.mkdirSync(path.dirname(resolved), { recursive: true });
|
||||||
@@ -386,12 +418,20 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
|
|||||||
// run is never killed. The timer is unref'd (it must not keep the loop alive
|
// run is never killed. The timer is unref'd (it must not keep the loop alive
|
||||||
// after a healthy turn) and cleared on completion.
|
// after a healthy turn) and cleared on completion.
|
||||||
const TURN_TIMEOUT_MS = Number(process.env.IMPECCABLE_SKILL_BEHAVIOR_TURN_TIMEOUT_MS) || 840_000;
|
const TURN_TIMEOUT_MS = Number(process.env.IMPECCABLE_SKILL_BEHAVIOR_TURN_TIMEOUT_MS) || 840_000;
|
||||||
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8, env = {}, simulatedUser = {}, timeoutMs = TURN_TIMEOUT_MS, contextOnlyBash = false, denyBash = false }) {
|
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8, env = {}, simulatedUser = {}, timeoutMs = TURN_TIMEOUT_MS, contextOnlyBash = false, denyBash = false, stopAfter, additionalTools, environment = '' }) {
|
||||||
const { tools, trace } = makeTools(workspace, env, simulatedUser, { contextOnlyBash, denyBash });
|
const { tools, trace } = makeTools(workspace, env, simulatedUser, { contextOnlyBash, denyBash });
|
||||||
|
if (additionalTools) Object.assign(tools, additionalTools(trace));
|
||||||
const messages = [
|
const messages = [
|
||||||
...priorMessages,
|
...priorMessages,
|
||||||
{ role: 'user', content: userPrompt },
|
{ role: 'user', content: userPrompt },
|
||||||
];
|
];
|
||||||
|
const traceDir = process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR;
|
||||||
|
const tracePath = traceDir && path.join(traceDir, `${path.basename(workspace)}-${crypto.randomUUID()}.json`);
|
||||||
|
const saveTrace = (details) => {
|
||||||
|
if (!tracePath) return;
|
||||||
|
fs.mkdirSync(traceDir, { recursive: true });
|
||||||
|
fs.writeFileSync(tracePath, JSON.stringify({ model: model.modelId, userPrompt, trace, ...details }, null, 2));
|
||||||
|
};
|
||||||
let result;
|
let result;
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
const timer = setTimeout(
|
const timer = setTimeout(
|
||||||
@@ -402,10 +442,14 @@ export async function runTurn({ workspace, model, userPrompt, priorMessages = []
|
|||||||
try {
|
try {
|
||||||
result = await generateText({
|
result = await generateText({
|
||||||
model,
|
model,
|
||||||
system: SKILL_BODY,
|
system: environment ? `${SKILL_BODY}\n\nRuntime environment: ${environment}` : SKILL_BODY,
|
||||||
messages,
|
messages,
|
||||||
tools,
|
tools,
|
||||||
stopWhen: [stepCountIs(maxSteps)],
|
onStepFinish: (step) => {
|
||||||
|
(trace.assistantTexts ??= []).push(step.text ?? '');
|
||||||
|
saveTrace({ status: 'in-progress', lastStepMessages: step.response.messages });
|
||||||
|
},
|
||||||
|
stopWhen: [stepCountIs(maxSteps), ...(stopAfter ? [() => stopAfter(trace)] : [])],
|
||||||
// Real client-side deadline on the provider call: without it a stalled
|
// Real client-side deadline on the provider call: without it a stalled
|
||||||
// stream wedges the whole sweep with no tally.
|
// stream wedges the whole sweep with no tally.
|
||||||
abortSignal: controller.signal,
|
abortSignal: controller.signal,
|
||||||
@@ -420,18 +464,27 @@ export async function runTurn({ workspace, model, userPrompt, priorMessages = []
|
|||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const reason = controller.signal.aborted ? ` (aborted after ${timeoutMs}ms client-side timeout)` : '';
|
const reason = controller.signal.aborted ? ` (aborted after ${timeoutMs}ms client-side timeout)` : '';
|
||||||
|
saveTrace({ status: 'failed', error: `${String(err)}${reason}` });
|
||||||
throw new Error(`LLM behavior turn failed before completing${reason}: ${String(err)}`, { cause: err });
|
throw new Error(`LLM behavior turn failed before completing${reason}: ${String(err)}`, { cause: err });
|
||||||
} finally {
|
} finally {
|
||||||
clearTimeout(timer);
|
clearTimeout(timer);
|
||||||
}
|
}
|
||||||
const generatedResponseMessages = result.responseMessages ?? result.response?.messages ?? [];
|
const generatedResponseMessages = result.responseMessages ?? result.response?.messages ?? [];
|
||||||
const responseMessages = [...messages, ...generatedResponseMessages];
|
const responseMessages = [...messages, ...generatedResponseMessages];
|
||||||
|
const outcome = stopAfter?.(trace) ? 'checkpoint'
|
||||||
|
: result.finishReason === 'length' ? 'output-limit'
|
||||||
|
: result.finishReason === 'tool-calls' && result.steps.length >= maxSteps ? 'step-budget'
|
||||||
|
: result.finishReason === 'stop' ? 'complete' : result.finishReason;
|
||||||
|
saveTrace({ status: 'completed', responseMessages,
|
||||||
|
outcome, finishReason: result.finishReason, steps: result.steps.length, usage: result.totalUsage ?? result.usage });
|
||||||
return {
|
return {
|
||||||
trace,
|
trace,
|
||||||
|
outcome,
|
||||||
|
steps: result.steps.length,
|
||||||
text: result.text ?? '',
|
text: result.text ?? '',
|
||||||
stepTexts: result.steps.map((step) => step.text ?? ''),
|
stepTexts: result.steps.map((step) => step.text ?? ''),
|
||||||
finishReason: result.finishReason,
|
finishReason: result.finishReason,
|
||||||
usage: result.usage,
|
usage: result.totalUsage ?? result.usage,
|
||||||
responseMessages,
|
responseMessages,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -451,8 +504,12 @@ export function readsMatching(trace, substring) {
|
|||||||
* True if the agent loaded a file by Read OR by a bash `cat` (some models
|
* True if the agent loaded a file by Read OR by a bash `cat` (some models
|
||||||
* stream multiple files via bash to save tool calls).
|
* stream multiple files via bash to save tool calls).
|
||||||
*/
|
*/
|
||||||
|
export function callLoadedFile(call, filename) {
|
||||||
|
return (call.loadedFiles || []).some((file) => file === filename || file.endsWith(`/${filename}`));
|
||||||
|
}
|
||||||
|
|
||||||
export function fileLoaded(trace, filename) {
|
export function fileLoaded(trace, filename) {
|
||||||
return readsMatching(trace, filename).length > 0 || bashCommandsMatching(trace, filename).length > 0;
|
return trace.toolCalls.some((call) => callLoadedFile(call, filename));
|
||||||
}
|
}
|
||||||
|
|
||||||
export function summarizeTrace(trace) {
|
export function summarizeTrace(trace) {
|
||||||
|
|||||||
@@ -18,16 +18,18 @@ import path from 'node:path';
|
|||||||
import {
|
import {
|
||||||
prepareWorkspace,
|
prepareWorkspace,
|
||||||
cleanupWorkspace,
|
cleanupWorkspace,
|
||||||
runTurn,
|
runTurn as runHarnessTurn,
|
||||||
bashCommandsMatching,
|
bashCommandsMatching,
|
||||||
readsMatching,
|
readsMatching,
|
||||||
fileLoaded,
|
fileLoaded,
|
||||||
|
callLoadedFile,
|
||||||
summarizeTrace,
|
summarizeTrace,
|
||||||
ENGINE_BIN,
|
ENGINE_BIN,
|
||||||
ENGINE_MISSING_MESSAGE,
|
ENGINE_MISSING_MESSAGE,
|
||||||
} from './harness.mjs';
|
} from './harness.mjs';
|
||||||
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
|
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
|
||||||
import { assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING } from './assertions.mjs';
|
import { assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs';
|
||||||
|
import { assertCompleted } from '../skill-workflow/assertions.mjs';
|
||||||
import {
|
import {
|
||||||
PRODUCT_MD_SAMPLE,
|
PRODUCT_MD_SAMPLE,
|
||||||
PRODUCT_MD_SAMPLE_NO_REGISTER,
|
PRODUCT_MD_SAMPLE_NO_REGISTER,
|
||||||
@@ -39,9 +41,22 @@ import {
|
|||||||
SVELTE_PROJECT_FILES,
|
SVELTE_PROJECT_FILES,
|
||||||
} from './fixtures.mjs';
|
} from './fixtures.mjs';
|
||||||
|
|
||||||
|
// Protocol-only shell access; successful checkpoints end observation, not the task.
|
||||||
|
async function runTurn({ checkpoint, ...options }) {
|
||||||
|
return runHarnessTurn({ contextOnlyBash: true, timeoutMs: 180000, ...options,
|
||||||
|
stopAfter: typeof checkpoint === 'function' ? checkpoint
|
||||||
|
: checkpoint ? (trace) => fileLoaded(trace, checkpoint) : undefined });
|
||||||
|
}
|
||||||
|
|
||||||
const CRAFT_PROMPT = '/impeccable craft a landing page for the project in this workspace';
|
const CRAFT_PROMPT = '/impeccable craft a landing page for the project in this workspace';
|
||||||
|
function projectCodeReads(trace) {
|
||||||
|
return trace.toolCalls.filter((call) => call.name === 'read' && call.succeeded
|
||||||
|
&& /\.(css|svelte|tsx?|jsx?|astro)$/i.test(call.input.path)
|
||||||
|
&& !call.input.path.includes('.claude/skills/')).map((call) => call.input.path);
|
||||||
|
}
|
||||||
const SHAPE_PROMPT = '/impeccable shape a landing page for the project in this workspace';
|
const SHAPE_PROMPT = '/impeccable shape a landing page for the project in this workspace';
|
||||||
const NATURAL_BUILD_PROMPT = 'Build a landing page for the project in this workspace.';
|
const NATURAL_BUILD_PROMPT = 'Build a landing page for the project in this workspace.';
|
||||||
|
const UPDATE_NOTICE = /(?:skill|impeccable).{0,100}(?:update|version)|(?:update|version).{0,100}(?:skill|impeccable)|99\.0\.0/i;
|
||||||
const TEACH_PROMPT = '/impeccable teach';
|
const TEACH_PROMPT = '/impeccable teach';
|
||||||
const PRIMER_PROMPT =
|
const PRIMER_PROMPT =
|
||||||
'Take a quick look at the project. What context should guide later design work? Run the impeccable context loader once if you need to.';
|
'Take a quick look at the project. What context should guide later design work? Run the impeccable context loader once if you need to.';
|
||||||
@@ -57,14 +72,9 @@ function logTrace(label, scenario, model, trace, extras = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function loadedBeforeImplementationWrite(trace, filename) {
|
function loadedBeforeImplementationWrite(trace, filename) {
|
||||||
const needle = filename.toLowerCase();
|
const loadIndex = trace.toolCalls.findIndex((call) => callLoadedFile(call, filename));
|
||||||
const loadIndex = trace.toolCalls.findIndex(({ name, input }) => {
|
|
||||||
if (name === 'read') return input?.path?.toLowerCase().includes(needle);
|
|
||||||
if (name === 'bash') return input?.command?.toLowerCase().includes(needle);
|
|
||||||
return false;
|
|
||||||
});
|
|
||||||
const writeIndex = trace.toolCalls.findIndex(
|
const writeIndex = trace.toolCalls.findIndex(
|
||||||
({ name, input }) => name === 'write' && /\.(html?|css|svelte|jsx?|tsx?)$/i.test(input?.path ?? ''),
|
({ mutatedPaths = [] }) => mutatedPaths.some((file) => /\.(html?|css|svelte|jsx?|tsx?)$/i.test(file)),
|
||||||
);
|
);
|
||||||
return loadIndex >= 0 && (writeIndex < 0 || loadIndex < writeIndex);
|
return loadIndex >= 0 && (writeIndex < 0 || loadIndex < writeIndex);
|
||||||
}
|
}
|
||||||
@@ -81,16 +91,6 @@ function executedUpdateCommands(trace) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function assertAdviceOnly(trace, text) {
|
|
||||||
assert.ok(text.trim(), 'advice must reach the user, not stop at reference loading');
|
|
||||||
assert.deepEqual(trace.writePaths, [], 'advice must not use the write tool');
|
|
||||||
const mutations = trace.toolCalls.flatMap((call) => call.mutatedPaths ?? [])
|
|
||||||
.filter((file) => !file.startsWith('.impeccable/') || file.startsWith('.impeccable/critique/'));
|
|
||||||
assert.deepEqual(mutations, [], 'advice must not edit project files or archive an unsolicited critique');
|
|
||||||
assert.deepEqual(trace.questionCalls, [], 'advice must not start an init or design interview');
|
|
||||||
assert.equal(bashCommandsMatching(trace, 'impeccable detect').length, 0, 'workflow advice does not run menu scans');
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const modelId of resolveModelList()) {
|
for (const modelId of resolveModelList()) {
|
||||||
const provider = detectProvider(modelId);
|
const provider = detectProvider(modelId);
|
||||||
const keyPresent = hasKey(provider);
|
const keyPresent = hasKey(provider);
|
||||||
@@ -105,17 +105,14 @@ for (const modelId of resolveModelList()) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const model = getModel(modelId);
|
const model = getModel(modelId);
|
||||||
// Gemini Flash tends to inspect one file at a time, while the production
|
// Observe a routing decision, with room for setup reads but no full build.
|
||||||
// Anthropic/OpenAI models batch setup reads and then begin implementation.
|
const setupMaxSteps = 10;
|
||||||
// Keep the latter tightly bounded so this routing suite does not turn into
|
|
||||||
// a page-generation benchmark, but leave Gemini enough room to reach the
|
|
||||||
// same required reference.
|
|
||||||
const setupMaxSteps = provider === 'google' ? 6 : 3;
|
|
||||||
|
|
||||||
it('scenario 1: no PRODUCT.md / DESIGN.md', async () => {
|
it('scenario 1: no PRODUCT.md / DESIGN.md', async () => {
|
||||||
const workspace = prepareWorkspace({ files: {} });
|
const workspace = prepareWorkspace({ files: {} });
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'init.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: CRAFT_PROMPT,
|
userPrompt: CRAFT_PROMPT,
|
||||||
@@ -149,6 +146,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'new-work.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: CRAFT_PROMPT,
|
userPrompt: CRAFT_PROMPT,
|
||||||
@@ -177,6 +175,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'new-work.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: CRAFT_PROMPT,
|
userPrompt: CRAFT_PROMPT,
|
||||||
@@ -221,6 +220,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
// Turn 1: prime the conversation so impeccable context gets run and its
|
// Turn 1: prime the conversation so impeccable context gets run and its
|
||||||
// output enters the message history.
|
// output enters the message history.
|
||||||
const turn1 = await runTurn({
|
const turn1 = await runTurn({
|
||||||
|
checkpoint: (trace) => trace.bashOutputs.some((output) => output.startsWith('exit=0\n')),
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: PRIMER_PROMPT,
|
userPrompt: PRIMER_PROMPT,
|
||||||
@@ -236,6 +236,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
// Turn 2: the real ask. The skill says "skip if you've already
|
// Turn 2: the real ask. The skill says "skip if you've already
|
||||||
// loaded it". Verify the agent honors that.
|
// loaded it". Verify the agent honors that.
|
||||||
const turn2 = await runTurn({
|
const turn2 = await runTurn({
|
||||||
|
checkpoint: 'new-work.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: 'Now, /impeccable craft a landing page based on what you saw.',
|
userPrompt: 'Now, /impeccable craft a landing page based on what you saw.',
|
||||||
@@ -261,6 +262,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'new-work.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: CRAFT_PROMPT,
|
userPrompt: CRAFT_PROMPT,
|
||||||
@@ -292,6 +294,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'polish.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable polish index.html',
|
userPrompt: '/impeccable polish index.html',
|
||||||
@@ -318,6 +321,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'audit.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable audit index.html',
|
userPrompt: '/impeccable audit index.html',
|
||||||
@@ -344,6 +348,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: (trace) => projectCodeReads(trace).length > 0,
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable polish src/routes/+page.svelte',
|
userPrompt: '/impeccable polish src/routes/+page.svelte',
|
||||||
@@ -354,9 +359,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
// agent should read at least one project code file (CSS / tokens /
|
// agent should read at least one project code file (CSS / tokens /
|
||||||
// component / page), not just the skill's PRODUCT.md / DESIGN.md
|
// component / page), not just the skill's PRODUCT.md / DESIGN.md
|
||||||
// / reference files.
|
// / reference files.
|
||||||
const projectReads = trace.readPaths.filter((p) =>
|
const projectReads = projectCodeReads(trace);
|
||||||
/\.(css|svelte|tsx?|jsx?|astro)$/i.test(p) && !p.includes('.claude/skills/'),
|
|
||||||
);
|
|
||||||
assert.ok(
|
assert.ok(
|
||||||
projectReads.length >= 1,
|
projectReads.length >= 1,
|
||||||
`agent should read at least one project code file to understand the existing design system.\n` +
|
`agent should read at least one project code file to understand the existing design system.\n` +
|
||||||
@@ -392,6 +395,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
userPrompt: '/impeccable polish index.html',
|
userPrompt: '/impeccable polish index.html',
|
||||||
maxSteps: setupMaxSteps,
|
maxSteps: setupMaxSteps,
|
||||||
env: { IMPECCABLE_UPDATE_CACHE: path.join(workspace, '.impeccable-update.json') },
|
env: { IMPECCABLE_UPDATE_CACHE: path.join(workspace, '.impeccable-update.json') },
|
||||||
|
checkpoint: (trace) => trace.assistantTexts?.some((text) => UPDATE_NOTICE.test(text)),
|
||||||
});
|
});
|
||||||
logTrace('S9', 'update-available', modelId, trace, { textSample: text.slice(0, 400) });
|
logTrace('S9', 'update-available', modelId, trace, { textSample: text.slice(0, 400) });
|
||||||
|
|
||||||
@@ -408,6 +412,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
`bashOutputs: ${JSON.stringify(trace.bashOutputs, null, 2)}`,
|
`bashOutputs: ${JSON.stringify(trace.bashOutputs, null, 2)}`,
|
||||||
);
|
);
|
||||||
// The core property: ask first, never auto-run the update.
|
// The core property: ask first, never auto-run the update.
|
||||||
|
assert.ok(trace.assistantTexts?.some((text) => UPDATE_NOTICE.test(text)), 'the skill update must be surfaced to the user');
|
||||||
const ranUpdate = executedUpdateCommands(trace);
|
const ranUpdate = executedUpdateCommands(trace);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
ranUpdate.length,
|
ranUpdate.length,
|
||||||
@@ -431,6 +436,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'polish.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable polish index.html',
|
userPrompt: '/impeccable polish index.html',
|
||||||
@@ -469,6 +475,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
const workspace = prepareWorkspace({ files: {} });
|
const workspace = prepareWorkspace({ files: {} });
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'init.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: SHAPE_PROMPT,
|
userPrompt: SHAPE_PROMPT,
|
||||||
@@ -494,6 +501,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
const workspace = prepareWorkspace({ files: {} });
|
const workspace = prepareWorkspace({ files: {} });
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'init.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: NATURAL_BUILD_PROMPT,
|
userPrompt: NATURAL_BUILD_PROMPT,
|
||||||
@@ -521,6 +529,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
const workspace = prepareWorkspace({ files: {} });
|
const workspace = prepareWorkspace({ files: {} });
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'init.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: TEACH_PROMPT,
|
userPrompt: TEACH_PROMPT,
|
||||||
@@ -554,6 +563,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'ios.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable craft a tide detail screen for the project in this workspace',
|
userPrompt: '/impeccable craft a tide detail screen for the project in this workspace',
|
||||||
@@ -589,6 +599,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const { trace, text } = await runTurn({
|
||||||
|
checkpoint: 'audit.native.md',
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable audit the app in this workspace',
|
userPrompt: '/impeccable audit the app in this workspace',
|
||||||
@@ -614,40 +625,42 @@ for (const modelId of resolveModelList()) {
|
|||||||
['existing project', WORKFLOW_ADVICE_FILES],
|
['existing project', WORKFLOW_ADVICE_FILES],
|
||||||
['missing product context', { 'index.html': MINIMAL_LANDING_HTML }],
|
['missing product context', { 'index.html': MINIMAL_LANDING_HTML }],
|
||||||
]) {
|
]) {
|
||||||
it(`scenario 16: workflow advice stays read-only (${label})`, async () => {
|
it(`scenario 16: workflow advice stays read-only (${label})`, async (t) => {
|
||||||
const workspace = prepareWorkspace({ files });
|
const workspace = prepareWorkspace({ files });
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const result = await runTurn({
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: "I'm joining this project. Where should I start with Impeccable?",
|
userPrompt: "I'm joining this project. Where should I start with Impeccable?",
|
||||||
maxSteps: 8,
|
maxSteps: 8,
|
||||||
contextOnlyBash: true,
|
contextOnlyBash: true,
|
||||||
});
|
});
|
||||||
|
const { trace, text } = result;
|
||||||
logTrace('S16', label, modelId, trace, { textSample: text.slice(0, 300) });
|
logTrace('S16', label, modelId, trace, { textSample: text.slice(0, 300) });
|
||||||
assert.ok(readsMatching(trace, 'reference/routing.md').length, 'workflow advice loads the shared routing reference');
|
t.diagnostic(`Reference coverage gaps (non-blocking): ${missingReferences(trace, ['reference/routing.md']).join(', ') || 'none'}`);
|
||||||
assertAdviceOnly(trace, text);
|
assertCompleted(result);
|
||||||
|
assertWorkflowAdvice(trace, text, { missingContext: label === 'missing product context' });
|
||||||
} finally {
|
} finally {
|
||||||
cleanupWorkspace(workspace);
|
cleanupWorkspace(workspace);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
it('scenario 17: command comparison reads references without running them', async () => {
|
it('scenario 17: command comparison explains independent commands without running them', async (t) => {
|
||||||
const workspace = prepareWorkspace({ files: WORKFLOW_ADVICE_FILES });
|
const workspace = prepareWorkspace({ files: WORKFLOW_ADVICE_FILES });
|
||||||
try {
|
try {
|
||||||
const { trace, text } = await runTurn({
|
const result = await runTurn({
|
||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: 'Should I use critique or polish on index.html? Is a critique required before polishing?',
|
userPrompt: 'Should I use critique or polish on index.html? Is a critique required before polishing?',
|
||||||
maxSteps: 8,
|
maxSteps: 8,
|
||||||
contextOnlyBash: true,
|
contextOnlyBash: true,
|
||||||
});
|
});
|
||||||
|
const { trace, text } = result;
|
||||||
logTrace('S17', 'command-comparison', modelId, trace, { textSample: text.slice(0, 300) });
|
logTrace('S17', 'command-comparison', modelId, trace, { textSample: text.slice(0, 300) });
|
||||||
assert.ok(readsMatching(trace, 'reference/routing.md').length, 'a command name in a question still routes to advice');
|
t.diagnostic(`Reference coverage gaps (non-blocking): ${missingReferences(trace, ['reference/routing.md', 'reference/critique.md', 'reference/polish.md']).join(', ') || 'none'}`);
|
||||||
assert.ok(readsMatching(trace, 'reference/critique.md').length, 'comparison consults the critique contract');
|
assertCompleted(result);
|
||||||
assert.ok(readsMatching(trace, 'reference/polish.md').length, 'comparison consults the polish contract');
|
assertCommandComparison(trace, text);
|
||||||
assertAdviceOnly(trace, text);
|
|
||||||
} finally {
|
} finally {
|
||||||
cleanupWorkspace(workspace);
|
cleanupWorkspace(workspace);
|
||||||
}
|
}
|
||||||
@@ -730,6 +743,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable polish index.html. Please do the polish pass now; afterward tell me which command would be useful next.',
|
userPrompt: '/impeccable polish index.html. Please do the polish pass now; afterward tell me which command would be useful next.',
|
||||||
|
checkpoint: 'polish.md',
|
||||||
maxSteps: 8,
|
maxSteps: 8,
|
||||||
contextOnlyBash: true,
|
contextOnlyBash: true,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import { it, mock } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { prepareBrowser, imageOutput } from './skill-workflow/browser.mjs';
|
||||||
|
import { chromium } from 'playwright';
|
||||||
|
|
||||||
|
it('fails browser preflight with an actionable error before starting a workflow', async () => {
|
||||||
|
const launch = mock.method(chromium, 'launch', async () => { throw new Error('browser missing'); });
|
||||||
|
try {
|
||||||
|
await assert.rejects(prepareBrowser('/unused'), /playwright install chromium/);
|
||||||
|
} finally {
|
||||||
|
launch.mock.restore();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('prepares real browser captures, interactions, and multimodal image results offline', async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-workflow-browser-'));
|
||||||
|
let browser;
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(path.join(root, 'index.html'), '<!doctype html><title>Fixture</title><button onclick="this.textContent=\'Done\'">Start</button><img src="https://example.invalid/image.png">');
|
||||||
|
browser = await prepareBrowser(root);
|
||||||
|
const trace = { toolCalls: [] };
|
||||||
|
const tools = browser.tools(trace);
|
||||||
|
const capture = await tools.browser_snapshot.execute({ path: './index.html', viewport: 'desktop', click: 'button' });
|
||||||
|
assert.equal(capture.target, 'index.html');
|
||||||
|
assert.match(capture.text, /Done/);
|
||||||
|
assert.ok(fs.existsSync(path.join(root, capture.screenshot)));
|
||||||
|
assert.equal(capture.viewport, 'desktop');
|
||||||
|
assert.equal(trace.toolCalls[0].name, 'browser_snapshot');
|
||||||
|
const output = imageOutput({ output: capture });
|
||||||
|
assert.equal(output.type, 'content');
|
||||||
|
assert.ok(output.value.some((part) => part.mediaType === 'image/png'));
|
||||||
|
const viewed = await tools.view_image.execute({ path: capture.screenshot });
|
||||||
|
assert.equal(viewed.image, capture.image);
|
||||||
|
const captures = await Promise.all(['desktop', 'mobile'].map((viewport) => tools.browser_snapshot.execute({ path: 'index.html', viewport })));
|
||||||
|
assert.deepEqual(captures.map((result) => result.viewport), ['desktop', 'mobile']);
|
||||||
|
assert.notEqual(captures[0].image, captures[1].image, 'parallel viewports must not share mutable page state');
|
||||||
|
assert.ok(browser.blockedRequests.some((url) => url.includes('example.invalid')));
|
||||||
|
await assert.rejects(tools.browser_snapshot.execute({ path: '../outside.html', viewport: 'mobile' }), /workspace/);
|
||||||
|
await assert.rejects(tools.view_image.execute({ path: 'index.html' }), /PNG/);
|
||||||
|
fs.symlinkSync(os.tmpdir(), path.join(root, 'escape'));
|
||||||
|
const response = await fetch(`${browser.origin}/escape/`);
|
||||||
|
assert.equal(response.status, 403);
|
||||||
|
} finally {
|
||||||
|
await browser?.close();
|
||||||
|
fs.rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { sourceHash } from './source-hash.mjs';
|
||||||
|
import { missingReferences } from '../skill-behavior/assertions.mjs';
|
||||||
|
|
||||||
|
export function assertDocumentationArtifacts(design, sidecarText) {
|
||||||
|
const frontmatter = design.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/)?.[1];
|
||||||
|
assert.ok(frontmatter, 'documentation must include machine-readable frontmatter, not prose alone');
|
||||||
|
assert.match(frontmatter, /^colors:\s*\n[ \t]+\S/m, 'documentation must record color tokens');
|
||||||
|
assert.match(frontmatter, /^typography:\s*\n[ \t]+\S/m, 'documentation must record typography tokens');
|
||||||
|
const sidecar = JSON.parse(sidecarText);
|
||||||
|
assert.equal(sidecar.schemaVersion, 2, 'documentation must write the v2 sidecar');
|
||||||
|
for (const key of ['extensions', 'narrative']) {
|
||||||
|
assert.ok(sidecar[key] && typeof sidecar[key] === 'object' && !Array.isArray(sidecar[key])
|
||||||
|
&& Object.keys(sidecar[key]).length, `sidecar must contain ${key} metadata`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// For a resumed, already-reviewed ordinary extension only. New worlds and
|
||||||
|
// redesigns still owe real documentation writes; this is not an escape hatch.
|
||||||
|
export function assertNoChangeDocumentation(result, { target, evidence }) {
|
||||||
|
assertCompleted(result);
|
||||||
|
const { trace, text } = result;
|
||||||
|
assert.deepEqual(missingReferences(trace, ['reference/document.md', target, 'DESIGN.md']), [],
|
||||||
|
'documentation must consult its contract and inspect the actual source and recorded system');
|
||||||
|
assert.deepEqual(trace.toolCalls.flatMap((call) => call.mutatedPaths || []), [],
|
||||||
|
'the resumed no-change check must not mutate project files');
|
||||||
|
assert.match(text, /no (?:system |visual.system |documentation )?changes|unchanged|no rewrite/i,
|
||||||
|
'documentation must explicitly report a no-change outcome');
|
||||||
|
for (const filename of [target, 'DESIGN.md']) {
|
||||||
|
assert.ok(text.includes(filename), `documentation must identify the checked ${filename}`);
|
||||||
|
}
|
||||||
|
for (const fact of evidence) {
|
||||||
|
assert.match(text, fact, 'no-change documentation must report evidence from the fixture, not an unsupported completion claim');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function assertCompleted(result) {
|
||||||
|
assert.equal(result.outcome, 'complete', `workflow did not finish: ${result.outcome} after ${result.steps} steps`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function assertFreshCaptures(trace, workspace, target) {
|
||||||
|
const calls = trace.toolCalls;
|
||||||
|
const lastEdit = calls.findLastIndex((call) => (call.mutatedPaths || []).includes(target));
|
||||||
|
const hash = sourceHash(workspace);
|
||||||
|
for (const viewport of ['desktop', 'mobile']) {
|
||||||
|
assert.ok(calls.some((call, index) => index > lastEdit && call.capture?.target === target
|
||||||
|
&& call.capture.viewport === viewport && call.capture.sourceHash === hash),
|
||||||
|
`missing ${viewport} screenshot of the final ${target}; pre-edit captures do not count`);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import http from 'node:http';
|
||||||
|
import { sourceHash as hashSources } from './source-hash.mjs';
|
||||||
|
import { chromium } from 'playwright';
|
||||||
|
import { tool } from 'ai';
|
||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
const VIEWPORTS = { desktop: { width: 1440, height: 1000 }, mobile: { width: 390, height: 844 } };
|
||||||
|
const TYPES = { '.html': 'text/html', '.css': 'text/css', '.js': 'text/javascript', '.mjs': 'text/javascript', '.svg': 'image/svg+xml', '.png': 'image/png', '.woff2': 'font/woff2' };
|
||||||
|
const PNG = Buffer.from('89504e470d0a1a0a', 'hex');
|
||||||
|
|
||||||
|
function resolveFile(root, name) {
|
||||||
|
if (path.isAbsolute(name)) throw new Error('Use a workspace-relative path');
|
||||||
|
const file = path.resolve(root, name);
|
||||||
|
const rel = path.relative(root, file);
|
||||||
|
if (rel === '..' || rel.startsWith(`..${path.sep}`)) throw new Error('Path escapes workspace');
|
||||||
|
const real = fs.realpathSync(file);
|
||||||
|
const realRel = path.relative(fs.realpathSync(root), real);
|
||||||
|
if (realRel === '..' || realRel.startsWith(`..${path.sep}`)) throw new Error('Path escapes workspace through a symlink');
|
||||||
|
return real;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function imageOutput({ output }) {
|
||||||
|
const { image, ...metadata } = output;
|
||||||
|
return { type: 'content', value: [
|
||||||
|
{ type: 'text', text: JSON.stringify(metadata) },
|
||||||
|
{ type: 'file', mediaType: 'image/png', data: { type: 'data', data: image } },
|
||||||
|
] };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Preflight before any billed call; no runtime installs or browser discovery. */
|
||||||
|
export async function prepareBrowser(root) {
|
||||||
|
let browser;
|
||||||
|
try {
|
||||||
|
browser = await chromium.launch({ headless: true, timeout: 15000 });
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error('Workflow browser preflight failed. Run `bunx playwright install chromium` before billed tests.', { cause: error });
|
||||||
|
}
|
||||||
|
const blockedRequests = [];
|
||||||
|
const server = http.createServer((req, res) => {
|
||||||
|
try {
|
||||||
|
const name = decodeURIComponent(new URL(req.url, 'http://localhost').pathname).replace(/^\//, '') || 'index.html';
|
||||||
|
if (name.split('/').some((part) => part.startsWith('.'))) throw new Error('Private workspace path');
|
||||||
|
const file = resolveFile(root, name);
|
||||||
|
if (!fs.statSync(file).isFile()) throw new Error('Not a file');
|
||||||
|
res.writeHead(200, { 'Content-Type': TYPES[path.extname(file)] || 'application/octet-stream', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(fs.readFileSync(file));
|
||||||
|
} catch (error) {
|
||||||
|
res.writeHead(error.code === 'ENOENT' ? 404 : 403);
|
||||||
|
res.end('Not available');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); });
|
||||||
|
} catch (error) {
|
||||||
|
await browser.close();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
const origin = `http://127.0.0.1:${server.address().port}`;
|
||||||
|
let context;
|
||||||
|
try {
|
||||||
|
context = await browser.newContext({ reducedMotion: 'reduce', serviceWorkers: 'block' });
|
||||||
|
await context.route('**/*', (route) => {
|
||||||
|
const url = route.request().url();
|
||||||
|
if (new URL(url).origin === origin || url.startsWith('data:')) return route.continue();
|
||||||
|
blockedRequests.push(url);
|
||||||
|
return route.abort();
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
await browser.close();
|
||||||
|
await new Promise((resolve) => { server.close(resolve); server.closeAllConnections(); });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
origin, blockedRequests,
|
||||||
|
environment: `Workspace: ${root}. A local server and Chromium are already running. browser_snapshot renders a workspace-relative HTML path at desktop/mobile size, saves a screenshot, and returns the actual image plus DOM text. view_image opens saved PNGs. No browser installation is needed. External browser requests are blocked; this text-only fixture uses system fonts. No image-generation or subagent tools are available.`,
|
||||||
|
tools(trace) {
|
||||||
|
return {
|
||||||
|
browser_snapshot: tool({
|
||||||
|
description: 'Render and inspect an HTML file with the ready Chromium browser. Returns an actual screenshot and visible DOM text; optionally click a CSS selector before capture. Captures save to .impeccable/review/{desktop|mobile}.png.',
|
||||||
|
inputSchema: z.object({ path: z.string(), viewport: z.enum(['desktop', 'mobile']), click: z.string().optional() }),
|
||||||
|
execute: async ({ path: target, viewport, click }) => {
|
||||||
|
const file = resolveFile(root, target);
|
||||||
|
if (!/\.html?$/i.test(file)) throw new Error('Expected an HTML artifact');
|
||||||
|
const relativeTarget = path.relative(fs.realpathSync(root), file).split(path.sep).join('/');
|
||||||
|
const call = { name: 'browser_snapshot', input: { path: target, viewport, click }, mutatedPaths: [] };
|
||||||
|
trace.toolCalls.push(call);
|
||||||
|
const page = await context.newPage();
|
||||||
|
page.setDefaultTimeout(10000);
|
||||||
|
try {
|
||||||
|
const sourceHash = hashSources(root);
|
||||||
|
await page.setViewportSize(VIEWPORTS[viewport]);
|
||||||
|
await page.goto(`${origin}/${relativeTarget.split('/').map(encodeURIComponent).join('/')}`, { waitUntil: 'load', timeout: 15000 });
|
||||||
|
await page.evaluate(() => document.fonts.ready);
|
||||||
|
if (click) await page.locator(click).click();
|
||||||
|
const screenshot = `.impeccable/review/${viewport}.png`;
|
||||||
|
if (fs.existsSync(path.join(root, '.impeccable'))) resolveFile(root, '.impeccable');
|
||||||
|
fs.mkdirSync(path.join(root, '.impeccable/review'), { recursive: true });
|
||||||
|
resolveFile(root, '.impeccable/review');
|
||||||
|
if (fs.existsSync(path.join(root, screenshot))) resolveFile(root, screenshot);
|
||||||
|
const image = await page.screenshot({ path: path.join(root, screenshot), fullPage: true, animations: 'disabled' });
|
||||||
|
call.mutatedPaths = [screenshot];
|
||||||
|
if (sourceHash !== hashSources(root)) throw new Error('Artifact changed during capture; retry');
|
||||||
|
call.capture = { target: relativeTarget, viewport, screenshot, sourceHash };
|
||||||
|
return { ...call.capture, text: (await page.locator('body').innerText()).slice(0, 12000), image: image.toString('base64') };
|
||||||
|
} finally {
|
||||||
|
await page.close();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
toModelOutput: imageOutput,
|
||||||
|
}),
|
||||||
|
view_image: tool({
|
||||||
|
description: 'Inspect an existing workspace PNG as an actual image, not raw file bytes.',
|
||||||
|
inputSchema: z.object({ path: z.string() }),
|
||||||
|
execute: async ({ path: name }) => {
|
||||||
|
const bytes = fs.readFileSync(resolveFile(root, name));
|
||||||
|
if (!bytes.subarray(0, 8).equals(PNG)) throw new Error('Expected a PNG image');
|
||||||
|
trace.toolCalls.push({ name: 'view_image', input: { path: name }, mutatedPaths: [], loadedImages: [name] });
|
||||||
|
return { path: name, image: bytes.toString('base64') };
|
||||||
|
},
|
||||||
|
toModelOutput: imageOutput,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async close() {
|
||||||
|
await browser.close();
|
||||||
|
await new Promise((resolve) => { server.close(resolve); server.closeAllConnections(); });
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
import { it } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { prepareWorkspace, cleanupWorkspace, runTurn, fileLoaded, ENGINE_BIN } from '../skill-behavior/harness.mjs';
|
||||||
|
import { getModel, detectProvider, hasKey } from '../skill-behavior/providers.mjs';
|
||||||
|
import { assertCompleted, assertNoChangeDocumentation, assertDocumentationArtifacts } from './assertions.mjs';
|
||||||
|
import { missingReferences } from '../skill-behavior/assertions.mjs';
|
||||||
|
|
||||||
|
// A synthetic post-review checkpoint, not another full-build simulation.
|
||||||
|
// The page and system agree. A missing sidecar predates this task and is not
|
||||||
|
// permission to repair drift or rewrite the incumbent DESIGN.md.
|
||||||
|
const DESIGN = `# Field Manual
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
An established, plain reading surface. Preserve this identity.
|
||||||
|
|
||||||
|
## Colors
|
||||||
|
White background (#ffffff), near-black text (#222222), blue links (#0645ad).
|
||||||
|
|
||||||
|
## Typography
|
||||||
|
System-ui body at 16px, line-height 1.6. Headings at 24px, weight 700.
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
One column, max-width 65ch, padding 24px; no decorative containers.
|
||||||
|
`;
|
||||||
|
const PAGE = '<!doctype html><html lang="en"><meta charset="utf-8"><title>Keyboard guide</title><style>body{background:#fff;color:#222;font:16px/1.6 system-ui;max-width:65ch;margin:auto;padding:24px}h1{font-size:24px;font-weight:700}a{color:#0645ad}</style><main><h1>Keyboard guide</h1><p>Use Tab to move between controls. Press Enter to activate a link.</p><a href="#top" id="top">Back to top</a></main></html>';
|
||||||
|
const BRIEF = '# Keyboard guide\n\n## Direction contract\nTHESIS: A short reading page.\nOWN-WORLD: Inherit Field Manual.\nSTORY: Read keyboard instructions.\nFIRST VIEWPORT: Title, paragraph, link.\nFORM: Direct, precisely specified page; no seed required.\nFINISH: unreviewed and undocumented is unfinished.\n';
|
||||||
|
|
||||||
|
for (const modelId of (process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS || 'claude-sonnet-5').split(',').map((id) => id.trim()).filter(Boolean)) {
|
||||||
|
for (const mode of ['extension', 'new world', 'redesign']) {
|
||||||
|
const existingSystem = mode !== 'new world';
|
||||||
|
const preserveSystem = mode === 'extension';
|
||||||
|
it(`post-review ${mode} ${preserveSystem ? 'preserves' : 'records'} its system :: ${modelId}`,
|
||||||
|
{ skip: !ENGINE_BIN || !hasKey(detectProvider(modelId)) }, async (t) => {
|
||||||
|
const files = {
|
||||||
|
'PRODUCT.md': '# Field Manual\n\n## Platform\nweb\n\nA reference guide for keyboard users.\n',
|
||||||
|
...(existingSystem ? { 'DESIGN.md': preserveSystem ? DESIGN : '# Old Field Manual\n\nBeige cards, serif body type, orange links.\n' } : {}),
|
||||||
|
'index.html': PAGE,
|
||||||
|
'.impeccable/surfaces/index-html.md': mode === 'redesign'
|
||||||
|
? BRIEF.replace('Inherit Field Manual.', 'Approved replacement: white, blue links, system-ui, single column.') : BRIEF,
|
||||||
|
};
|
||||||
|
const workspace = prepareWorkspace({ files });
|
||||||
|
try {
|
||||||
|
const reference = fs.readFileSync(path.join(workspace, '.claude/skills/impeccable/reference/new-work.md'), 'utf8');
|
||||||
|
const result = await runTurn({
|
||||||
|
workspace, model: getModel(modelId), maxSteps: 12, timeoutMs: 180000, contextOnlyBash: true,
|
||||||
|
environment: 'This is a resumed post-review checkpoint. No subagent or browser tools are available. The review is closed; no further UI edits or screenshots are needed. Read/list/write tools are available.',
|
||||||
|
priorMessages: [
|
||||||
|
{ role: 'user', content: preserveSystem
|
||||||
|
? 'Use /impeccable to add the specified keyboard guide page inside the established Field Manual world. Keep the existing visual system. Do not repair unrelated project drift.'
|
||||||
|
: mode === 'redesign'
|
||||||
|
? 'Use /impeccable to redesign the keyboard guide. I approve replacing the old beige-card/serif/orange world with the plain single-column, system-font, white-background and blue-link identity. Update the system documentation from the finished page.'
|
||||||
|
: 'Use /impeccable to create Field Manual’s first keyboard guide page. The chosen identity is plain, single-column, system fonts, white background and blue links.' },
|
||||||
|
{ role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'load-new-work', toolName: 'read', input: { path: '.claude/skills/impeccable/reference/new-work.md' } }] },
|
||||||
|
{ role: 'tool', content: [{ type: 'tool-result', toolCallId: 'load-new-work', toolName: 'read', output: { type: 'text', value: reference } }] },
|
||||||
|
{ role: 'assistant', content: `Checkpoint: context and PRODUCT.md were loaded. The user confirmed the exact page and identity. The surface brief and index.html are written. Desktop/mobile captures were validated, the detector ran once, and the shipped finish reviewer returned ship with no open findings. ${preserveSystem
|
||||||
|
? 'DESIGN.md was loaded. No durable system changes were requested or introduced. The pre-existing missing .impeccable/design.json was reported but not repaired.'
|
||||||
|
: mode === 'redesign'
|
||||||
|
? 'The approved replacement world is implemented in index.html. DESIGN.md still describes the superseded identity; no design sidecar exists yet.'
|
||||||
|
: 'This is the first completed surface of the approved new world. No DESIGN.md or design sidecar exists yet.'}` },
|
||||||
|
],
|
||||||
|
userPrompt: 'Continue from this checkpoint and finish the task.',
|
||||||
|
});
|
||||||
|
assertCompleted(result);
|
||||||
|
t.diagnostic(`Documentation wrapper coverage gaps (diagnostic; contract and artifacts remain required): ${missingReferences(result.trace, ['degraded/documenter.md']).join(', ') || 'none'}`);
|
||||||
|
assert.ok(fileLoaded(result.trace, 'reference/document.md'), 'must consult the documentation contract');
|
||||||
|
for (const name of existingSystem ? ['index.html', 'DESIGN.md'] : ['index.html']) {
|
||||||
|
assert.ok(fileLoaded(result.trace, name), `documentation must check ${name}, not merely announce a no-op`);
|
||||||
|
}
|
||||||
|
for (const [name, contents] of Object.entries(files)) {
|
||||||
|
if (mode === 'redesign' && name === 'DESIGN.md') continue;
|
||||||
|
assert.equal(fs.readFileSync(path.join(workspace, name), 'utf8'), contents, `${name} must remain unchanged`);
|
||||||
|
}
|
||||||
|
if (preserveSystem) {
|
||||||
|
assertNoChangeDocumentation(result, { target: 'index.html', evidence: [/system-ui/i, /65\s*ch/i, /#0645ad/i] });
|
||||||
|
assert.equal(fs.existsSync(path.join(workspace, '.impeccable/design.json')), false, 'must not repair pre-existing sidecar drift unasked');
|
||||||
|
assert.deepEqual(result.trace.toolCalls.flatMap((call) => call.mutatedPaths || []), [], 'a no-change check must not mutate other project files');
|
||||||
|
} else {
|
||||||
|
const design = fs.readFileSync(path.join(workspace, 'DESIGN.md'), 'utf8');
|
||||||
|
if (mode === 'redesign') assert.notEqual(design, files['DESIGN.md'], 'approved redesign must replace the old system');
|
||||||
|
assertDocumentationArtifacts(design, fs.readFileSync(path.join(workspace, '.impeccable/design.json'), 'utf8'));
|
||||||
|
assert.match(design, /system-ui/);
|
||||||
|
const writes = result.trace.toolCalls.flatMap((call) => call.mutatedPaths || []);
|
||||||
|
assert.ok(writes.includes('DESIGN.md') && writes.includes('.impeccable/design.json'), 'both documentation artifacts must be written');
|
||||||
|
assert.deepEqual(writes.filter((file) => !['DESIGN.md', '.impeccable/design.json'].includes(file)), [], 'documentation must stay inside its write boundary');
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
cleanupWorkspace(workspace);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
+45
-17
@@ -10,14 +10,39 @@ import path from 'node:path';
|
|||||||
import {
|
import {
|
||||||
prepareWorkspace,
|
prepareWorkspace,
|
||||||
cleanupWorkspace,
|
cleanupWorkspace,
|
||||||
runTurn,
|
runTurn as runHarnessTurn,
|
||||||
fileLoaded,
|
fileLoaded,
|
||||||
summarizeTrace,
|
summarizeTrace,
|
||||||
ENGINE_BIN,
|
ENGINE_BIN,
|
||||||
ENGINE_MISSING_MESSAGE,
|
ENGINE_MISSING_MESSAGE,
|
||||||
} from './harness.mjs';
|
} from '../skill-behavior/harness.mjs';
|
||||||
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
|
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from '../skill-behavior/providers.mjs';
|
||||||
import { PRODUCT_MD_SAMPLE, DESIGN_MD_SAMPLE } from './fixtures.mjs';
|
import { assertNewWorkLifecycle } from '../skill-behavior/assertions.mjs';
|
||||||
|
import { PRODUCT_MD_SAMPLE, DESIGN_MD_SAMPLE as ORIGINAL_DESIGN, CASE_STUDY_ANSWER } from '../skill-behavior/fixtures.mjs';
|
||||||
|
import { prepareBrowser } from './browser.mjs';
|
||||||
|
import { assertCompleted, assertFreshCaptures, assertDocumentationArtifacts } from './assertions.mjs';
|
||||||
|
|
||||||
|
const DESIGN_MD_SAMPLE = ORIGINAL_DESIGN.replace(/GT Sectra \(commercial\)/g, 'Georgia (system)').replace(/JetBrains Mono/g, 'monospace').replace(/Inter/g, 'Arial');
|
||||||
|
|
||||||
|
async function runTurn(options) {
|
||||||
|
// Preflight happens before the first provider call. These are text-only
|
||||||
|
// HTML fixtures: no dependencies, font downloads, or browser discovery.
|
||||||
|
const browser = await prepareBrowser(options.workspace);
|
||||||
|
try {
|
||||||
|
const result = await runHarnessTurn({
|
||||||
|
...options, maxSteps: 50, timeoutMs: 840000,
|
||||||
|
userPrompt: `${options.userPrompt}\nUse system fonts and no external assets for this text-only fixture. The browser_snapshot and view_image tools are ready for visual review.`,
|
||||||
|
environment: browser.environment,
|
||||||
|
additionalTools: (trace) => browser.tools(trace),
|
||||||
|
});
|
||||||
|
assertCompleted(result);
|
||||||
|
const contextCalls = result.trace.toolCalls.filter(({ name, input }) => name === 'bash' && /impeccable\s+context\b/.test(input.command));
|
||||||
|
assert.equal(contextCalls.length, 1, 'completed workflow must load context exactly once');
|
||||||
|
return result;
|
||||||
|
} finally {
|
||||||
|
await browser.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const LEGACY_DESIGN = `# Design
|
const LEGACY_DESIGN = `# Design
|
||||||
|
|
||||||
@@ -100,7 +125,9 @@ function workflowTraceMessage(trace) {
|
|||||||
return JSON.stringify(summarizeTrace(trace), null, 2);
|
return JSON.stringify(summarizeTrace(trace), null, 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const modelId of resolveModelList()) {
|
// Full builds are separately opt-in and default to one provider. The existing
|
||||||
|
// model selection variable can explicitly request a cross-provider sweep.
|
||||||
|
for (const modelId of process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS ? resolveModelList() : ['claude-sonnet-5']) {
|
||||||
const provider = detectProvider(modelId);
|
const provider = detectProvider(modelId);
|
||||||
const keyPresent = hasKey(provider);
|
const keyPresent = hasKey(provider);
|
||||||
|
|
||||||
@@ -122,7 +149,6 @@ for (const modelId of resolveModelList()) {
|
|||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable init for a harbor operations product, then finish setup.',
|
userPrompt: '/impeccable init for a harbor operations product, then finish setup.',
|
||||||
maxSteps: 24,
|
|
||||||
});
|
});
|
||||||
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
||||||
const productWrite = firstMutation(trace, /(^|\/)PRODUCT\.md$/i);
|
const productWrite = firstMutation(trace, /(^|\/)PRODUCT\.md$/i);
|
||||||
@@ -147,20 +173,22 @@ for (const modelId of resolveModelList()) {
|
|||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable create a concise evidence-led case-study page. Leave it at index.html.',
|
userPrompt: '/impeccable create a concise evidence-led case-study page. Leave it at index.html.',
|
||||||
maxSteps: 22,
|
simulatedUser: { answer: () => CASE_STUDY_ANSWER },
|
||||||
});
|
});
|
||||||
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
||||||
const implementation = firstMutation(trace, /\.(?:html?|astro|svelte|jsx?|tsx?)$/i);
|
|
||||||
assert.ok(fileLoaded(trace, 'new-work.md'), `new-work.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
assert.ok(fileLoaded(trace, 'new-work.md'), `new-work.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
||||||
assert.ok(question >= 0, `task concept was never put to the user.\n${workflowTraceMessage(trace)}`);
|
assert.ok(question >= 0, `task concept was never put to the user.\n${workflowTraceMessage(trace)}`);
|
||||||
assert.ok(implementation > question, `implementation began before the attended concept checkpoint.\n${workflowTraceMessage(trace)}`);
|
assertNewWorkLifecycle(trace, { target: 'index.html' });
|
||||||
|
assertFreshCaptures(trace, workspace, 'index.html');
|
||||||
|
assert.ok(fileLoaded(trace, 'finish-reviewer.md'), 'new-work must run the shipped finish review');
|
||||||
|
assert.ok(fileLoaded(trace, 'documenter.md'), 'new-work must run the shipped documentation pass');
|
||||||
assert.equal(fs.existsSync(path.join(workspace, 'index.html')), true, 'new-work must still produce the requested artifact');
|
assert.equal(fs.existsSync(path.join(workspace, 'index.html')), true, 'new-work must still produce the requested artifact');
|
||||||
} finally {
|
} finally {
|
||||||
cleanupWorkspace(workspace);
|
cleanupWorkspace(workspace);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it('redesign replaces DESIGN before touching the existing page', async () => {
|
it('redesign approves and records the direction before code, then documents the built world', async () => {
|
||||||
const workspace = prepareWorkspace({
|
const workspace = prepareWorkspace({
|
||||||
files: {
|
files: {
|
||||||
'PRODUCT.md': PRODUCT_MD_SAMPLE,
|
'PRODUCT.md': PRODUCT_MD_SAMPLE,
|
||||||
@@ -173,17 +201,17 @@ for (const modelId of resolveModelList()) {
|
|||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable redesign current.html for this product. Leave the result at current.html.',
|
userPrompt: '/impeccable redesign current.html for this product. Leave the result at current.html.',
|
||||||
maxSteps: 26,
|
|
||||||
});
|
});
|
||||||
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
||||||
const designWrite = firstMutation(trace, /(^|\/)DESIGN\.md$/i);
|
|
||||||
const implementation = firstMutation(trace, /(^|\/)current\.html$/i);
|
|
||||||
assert.ok(fileLoaded(trace, 'new-work.md'), `redesign did not route through new-work.\n${workflowTraceMessage(trace)}`);
|
assert.ok(fileLoaded(trace, 'new-work.md'), `redesign did not route through new-work.\n${workflowTraceMessage(trace)}`);
|
||||||
assert.ok(question >= 0, `replacement world was not put to the user.\n${workflowTraceMessage(trace)}`);
|
assert.ok(question >= 0, `replacement world was not put to the user.\n${workflowTraceMessage(trace)}`);
|
||||||
assert.ok(designWrite > question, `replacement DESIGN.md must follow user choice.\n${workflowTraceMessage(trace)}`);
|
assertNewWorkLifecycle(trace, { target: 'current.html', redesign: true });
|
||||||
assert.ok(implementation > designWrite, `redesign touched the page before replacing DESIGN.md.\n${workflowTraceMessage(trace)}`);
|
assertFreshCaptures(trace, workspace, 'current.html');
|
||||||
|
assert.ok(fileLoaded(trace, 'finish-reviewer.md'), 'redesign must run the shipped finish review');
|
||||||
|
assert.ok(fileLoaded(trace, 'documenter.md'), 'redesign must run the shipped documentation pass');
|
||||||
const design = fs.readFileSync(path.join(workspace, 'DESIGN.md'), 'utf8');
|
const design = fs.readFileSync(path.join(workspace, 'DESIGN.md'), 'utf8');
|
||||||
assert.notEqual(design.trim(), LEGACY_DESIGN.trim(), 'redesign preserved the old visual world verbatim');
|
assert.notEqual(design.trim(), LEGACY_DESIGN.trim(), 'redesign preserved the old visual world verbatim');
|
||||||
|
assertDocumentationArtifacts(design, fs.readFileSync(path.join(workspace, '.impeccable/design.json'), 'utf8'));
|
||||||
} finally {
|
} finally {
|
||||||
cleanupWorkspace(workspace);
|
cleanupWorkspace(workspace);
|
||||||
}
|
}
|
||||||
@@ -202,7 +230,6 @@ for (const modelId of resolveModelList()) {
|
|||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable bolder current.html, only the #case-study section. Keep everything else untouched.',
|
userPrompt: '/impeccable bolder current.html, only the #case-study section. Keep everything else untouched.',
|
||||||
maxSteps: 16,
|
|
||||||
});
|
});
|
||||||
const productWrite = firstMutation(trace, /(^|\/)PRODUCT\.md$/i);
|
const productWrite = firstMutation(trace, /(^|\/)PRODUCT\.md$/i);
|
||||||
const designWrite = firstMutation(trace, /(^|\/)DESIGN\.md$/i);
|
const designWrite = firstMutation(trace, /(^|\/)DESIGN\.md$/i);
|
||||||
@@ -211,6 +238,7 @@ for (const modelId of resolveModelList()) {
|
|||||||
assert.equal(productWrite, -1, `refinement rewrote PRODUCT.md.\n${workflowTraceMessage(trace)}`);
|
assert.equal(productWrite, -1, `refinement rewrote PRODUCT.md.\n${workflowTraceMessage(trace)}`);
|
||||||
assert.equal(designWrite, -1, `refinement rewrote DESIGN.md.\n${workflowTraceMessage(trace)}`);
|
assert.equal(designWrite, -1, `refinement rewrote DESIGN.md.\n${workflowTraceMessage(trace)}`);
|
||||||
assert.ok(implementation >= 0, `refinement did not write current.html.\n${workflowTraceMessage(trace)}`);
|
assert.ok(implementation >= 0, `refinement did not write current.html.\n${workflowTraceMessage(trace)}`);
|
||||||
|
assertFreshCaptures(trace, workspace, 'current.html');
|
||||||
const artifact = fs.readFileSync(path.join(workspace, 'current.html'), 'utf8');
|
const artifact = fs.readFileSync(path.join(workspace, 'current.html'), 'utf8');
|
||||||
assert.match(artifact, /data-untouched="header"/);
|
assert.match(artifact, /data-untouched="header"/);
|
||||||
assert.match(artifact, /data-untouched="footer"/);
|
assert.match(artifact, /data-untouched="footer"/);
|
||||||
@@ -238,9 +266,9 @@ for (const modelId of resolveModelList()) {
|
|||||||
workspace,
|
workspace,
|
||||||
model,
|
model,
|
||||||
userPrompt: '/impeccable critique current.html',
|
userPrompt: '/impeccable critique current.html',
|
||||||
maxSteps: 30,
|
|
||||||
});
|
});
|
||||||
assert.ok(fileLoaded(trace, 'critique.md'), `critique.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
assert.ok(fileLoaded(trace, 'critique.md'), `critique.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
||||||
|
assertFreshCaptures(trace, workspace, 'current.html');
|
||||||
|
|
||||||
const parts = assistantParts(responseMessages);
|
const parts = assistantParts(responseMessages);
|
||||||
const allText = parts.filter((p) => p.kind === 'text').map((p) => p.value).join('\n');
|
const allText = parts.filter((p) => p.kind === 'text').map((p) => p.value).join('\n');
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import crypto from 'node:crypto';
|
||||||
|
|
||||||
|
// Include local styles/scripts/assets too: an unchanged HTML file is not
|
||||||
|
// evidence of a current capture when an external stylesheet changed.
|
||||||
|
export function sourceHash(root) {
|
||||||
|
const hash = crypto.createHash('sha256');
|
||||||
|
function visit(directory, prefix = '') {
|
||||||
|
for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
||||||
|
if (entry.name.startsWith('.') || entry.name === 'node_modules') continue;
|
||||||
|
const relative = `${prefix}${entry.name}`;
|
||||||
|
const file = path.join(directory, entry.name);
|
||||||
|
if (entry.isDirectory()) visit(file, `${relative}/`);
|
||||||
|
else if (entry.isFile() && /\.(html?|css|m?js|svg|png|jpe?g|webp|woff2?)$/i.test(entry.name)) {
|
||||||
|
hash.update(relative).update('\0').update(fs.readFileSync(file)).update('\0');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
visit(root);
|
||||||
|
return hash.digest('hex');
|
||||||
|
}
|
||||||
@@ -12,6 +12,13 @@ import {
|
|||||||
} from '../scripts/test-suites.mjs';
|
} from '../scripts/test-suites.mjs';
|
||||||
|
|
||||||
describe('test suite registry', () => {
|
describe('test suite registry', () => {
|
||||||
|
it('separates protocol checkpoints from opt-in browser-backed completion', () => {
|
||||||
|
assert.deepEqual(suiteFiles(['skill-behavior']), ['tests/skill-behavior/scenarios.test.mjs']);
|
||||||
|
assert.ok(OPT_IN_SUITES.includes('skill-workflow'));
|
||||||
|
assert.equal(SUITES['skill-workflow'].needsPlaywright, true);
|
||||||
|
assert.ok(suiteFiles(['skill-workflow']).includes('tests/skill-workflow/full-build.test.mjs'));
|
||||||
|
assert.equal(DEFAULT_SUITES.includes('skill-workflow'), false);
|
||||||
|
});
|
||||||
it('assigns every test file to a default or opt-in suite', () => {
|
it('assigns every test file to a default or opt-in suite', () => {
|
||||||
const allDiscovered = findTestFiles();
|
const allDiscovered = findTestFiles();
|
||||||
const allRegistered = new Set(suiteFiles([...DEFAULT_SUITES, ...OPT_IN_SUITES]));
|
const allRegistered = new Set(suiteFiles([...DEFAULT_SUITES, ...OPT_IN_SUITES]));
|
||||||
|
|||||||
Reference in New Issue
Block a user