diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d930a36f..d8c84e8e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,11 @@ on: pull_request: branches: [main] workflow_dispatch: + inputs: + skill_workflow: + description: 'Run billed, browser-backed Claude workflow completion tests' + type: boolean + default: false # Nightly full live-e2e matrix. The smoke groups already gate every PR; the # full sweep is too slow for that, so it runs once a day against main. schedule: @@ -665,5 +670,47 @@ jobs: - name: Install dependencies run: bun install + - name: Prepare engine for protocol tests + run: bun run fetch:engine + - name: Run skill behavior tests run: bun run test:skill-behavior + + skill-workflow: + # Expensive, manually opted-in completion diagnostics. This does not + # replace the multi-family skill-behavior protocol suite above. + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' && inputs.skill_workflow + timeout-minutes: 70 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + IMPECCABLE_SKILL_BEHAVIOR_MODELS: claude-sonnet-5 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: latest + - name: Install dependencies + run: bun install --frozen-lockfile + - name: Prepare engine and browser before billing + run: | + test -n "$ANTHROPIC_API_KEY" || { echo 'ANTHROPIC_API_KEY is required'; exit 1; } + bun run fetch:engine + bunx playwright install --with-deps chromium + - name: Run completed skill workflows + env: + IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR: ${{ runner.temp }}/skill-workflow-traces + run: bun run test:skill-workflow + - name: Retain diagnostic traces + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: skill-workflow-traces + path: ${{ runner.temp }}/skill-workflow-traces + retention-days: 7 diff --git a/package.json b/package.json index fe86c503b..68a6c7527 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "test:new-work-e2e": "node scripts/run-tests.mjs new-work-e2e", "test:live-e2e-agent": "node scripts/run-tests.mjs live-e2e-agent", "test:skill-behavior": "node scripts/run-tests.mjs skill-behavior", + "test:skill-workflow": "node scripts/run-tests.mjs skill-workflow", "test:live-svelte-adapter-deepseek": "node scripts/run-tests.mjs live-svelte-adapter-deepseek", "smoke:hooks": "node scripts/smoke-provider-hooks.mjs", "audit": "bun audit --audit-level=moderate", diff --git a/scripts/test-suites.mjs b/scripts/test-suites.mjs index 21a3790de..fa0de35af 100644 --- a/scripts/test-suites.mjs +++ b/scripts/test-suites.mjs @@ -8,6 +8,7 @@ export const OPT_IN_SUITES = [ 'live-e2e-accept-cleanup', 'new-work-e2e', 'skill-behavior', + 'skill-workflow', 'live-svelte-adapter-deepseek', ]; @@ -267,38 +268,44 @@ export const SUITES = { ], }, 'skill-behavior': { - description: 'LLM-backed skill setup behavior scenarios.', + description: 'LLM-backed protocol checkpoints, not full builds.', optIn: true, triggers: [ ...COMMON_INFRA_PATTERNS, /^skill\/SKILL\.src\.md$/, - /^skill\/reference\/(init|document|brand|product|shape|craft|audit|polish|live|routing)\.md$/, + /^skill\/reference\//, /^ENGINE_VERSION$/, /^tests\/skill-behavior\//, ], + commands: [{ + runner: 'node', + timeoutMs: 240000, + wallClockMs: 1_800_000, + files: ['tests/skill-behavior/scenarios.test.mjs'], + }], + }, + 'skill-workflow': { + description: 'Explicitly opt-in completed workflows with a preflighted browser.', + optIn: true, + needsPlaywright: true, + triggers: [ + ...COMMON_INFRA_PATTERNS, + /^skill\//, + /^ENGINE_VERSION$/, + /^tests\/skill-workflow\//, + /^tests\/skill-behavior\//, + ], commands: [ + { runner: 'node', files: ['tests/skill-workflow-browser.test.mjs'] }, + { + runner: 'node', timeoutMs: 240000, wallClockMs: 600000, + files: ['tests/skill-workflow/finish-handoff.test.mjs'], + }, { runner: 'node', - // 300000 was too low to measure what these scenarios assert. The - // workflow-contract turns run 20+ steps against a frontier model, and - // the *correct* path is the slow one: a run that stops to put the - // concept to the user before building was measured at 579s, while the - // runs that skipped that checkpoint and failed the assertion finished - // in 130-200s. At a 300s cap the thorough path is killed and the hasty - // path is graded, so the cap was selecting for the behavior the suite - // exists to forbid. timeoutMs: 900000, - // Overall wall-clock safety cap for the whole sweep: if a provider - // call wedges past every inner guard (the harness's 840s per-turn - // AbortSignal and the 900s per-test timeout), the runner SIGKILLs the - // process group so the sweep still ends with a per-provider tally - // instead of hanging overnight. Sized well above a healthy two-provider - // sweep; override with IMPECCABLE_TEST_WALL_CLOCK_MS to scope it down. wallClockMs: 3_600_000, - files: [ - 'tests/skill-behavior/scenarios.test.mjs', - 'tests/skill-behavior/workflow-contract.test.mjs', - ], + files: ['tests/skill-workflow/full-build.test.mjs'], }, ], }, diff --git a/skill/SKILL.src.md b/skill/SKILL.src.md index 0637f4789..f989e758c 100644 --- a/skill/SKILL.src.md +++ b/skill/SKILL.src.md @@ -22,7 +22,7 @@ Core principles: 2. Load the request's playbook: its Commands-table reference for an explicit/implied sub-command, or [reference/new-work.md](reference/new-work.md) for a new surface or replacement visual world. Inspect target and incumbent visual truth before editing. When the app cannot run, start with committed visual-regression goldens or screenshot fixtures; verify target and freshness against current tokens, CSS, components, or assets, resolve conflicts, and compare theme/variant captures. 3. After resolving analysis and direction, read [reference/craft-floor.md](reference/craft-floor.md) immediately before any UI edit, including small refinements. It carries the quality floor, the absolute bans, and the reflexes no detector catches. Do not load it for planning-only work. -**Launcher unavailable:** If refused, missing, or failed, **first send the user a message** that context loading did not run. Then read existing PRODUCT.md and DESIGN.md without inventing missing context, follow the applicable steps 2–3, and perform the requested work through permitted tools. Launcher failure alone does not block otherwise-permitted edits. +**Launcher unavailable:** On refusal or failure, send a separate message **before the next tool call**: “Context loading did not run; I’ll read the existing project context directly.” Then read existing PRODUCT.md and DESIGN.md without inventing missing context, follow applicable steps 2–3, and continue through permitted tools. This applies to planning and editing; launcher failure alone does not block either. ## How to design diff --git a/skill/agents/impeccable-documenter.md b/skill/agents/impeccable-documenter.md index 53817f6d9..d2c4e107c 100644 --- a/skill/agents/impeccable-documenter.md +++ b/skill/agents/impeccable-documenter.md @@ -16,7 +16,7 @@ nickname-candidates: You record a project's design system after the build is done. Ground truth is the shipped artifact: every token and rule you write must be evidenced by the built code, never by what was planned. Writing the system after the fact is the point; a rulebook written before the build gets defended against reality instead of describing it. -You run under a hard turn ceiling that ends the run without warning, and a run that ends before DESIGN.md is written has recorded nothing. Batch several Reads into each turn, take `reference/document.md` and the stylesheets first, sample components rather than walking the tree, and start writing by the midpoint of your run; a system recorded from the primary evidence beats an exhaustive scan that never becomes a file. +Complete the check within your turn ceiling. Batch Reads, take `reference/document.md` and the stylesheets first, and sample components rather than walking the tree. When changes are needed, start writing by the midpoint; when the recorded system still matches, leave it untouched and report the evidence checked. ## Input Contract @@ -26,10 +26,10 @@ Expect: the project root; the artifact path(s); the direction contract text (THE 1. Read `reference/document.md` in full; it is the operating spec for DESIGN.md's format, token schema, sidecar, and section order. Follow it exactly. 2. Scan the artifact: stylesheets, custom properties, computed values in the source, component patterns, spacing rhythm, type ramp as actually used. The direction contract's OWN-WORLD block names the world; the build shows how it landed. Where they diverge, the build wins and the prose may note the divergence. -3. Write DESIGN.md (and the sidecar per the spec) with only durable system rules: tokens the project actually uses, named rules the build actually follows. Skip one-off values; a token used once is not a system. +3. For a new world or approved system change, write DESIGN.md and its sidecar from durable, reused rules in the build. Ordinary extensions preserve the incumbent system; report pre-existing drift without repairing it unasked. Do not write merely to prove this pass ran. 4. Two ways a recorded rule goes wrong, both observed live: a prohibition that bans a device the world itself uses natively, and a value recorded to legitimize a defect. Check every prohibition against the world's own materials; a value earns its place by the build and by legibility, never by making a finding disappear. 5. Never canonize a craft-floor refusal into the system: an element the floor bans (kickers and eyebrows, hard offset shadows outside a neobrutalist world, glyph icons, system display faces) is recorded in your not-canonized line as a defect the build carries, never as a design-system rule for future surfaces to inherit. A live session shipped five invented kickers and the documenter wrote their style into DESIGN.md; that is how one violation becomes the house style. ## Output Contract -Return: the file paths written, a five-line summary of the recorded system (palette strategy, type ramp shape, named rules), and one line naming anything in the build you deliberately did not canonize and why. No other prose. +Return: paths written, or “No changes” with the source and system files checked; a five-line system summary (palette, type ramp, named rules); and one line naming defects or drift not canonized or repaired, and why. No other prose. diff --git a/skill/reference/new-work.md b/skill/reference/new-work.md index 675aa02ec..b5e8f224d 100644 --- a/skill/reference/new-work.md +++ b/skill/reference/new-work.md @@ -15,7 +15,7 @@ A section, component, feature, or state inside an established surface inherits t ## 2. Ask what will change the work -Ask one round of two or three related questions through the structured question tool when available. Skip settled facts; a precise request may need only a compact confirmation. +Before implementation, get the user's answer through the structured question tool when available. Ask two or three related questions; a precise request needs only a compact confirmation. Skip settled facts, not the confirmation: DESIGN.md settles the visual world, not this surface's purpose or concept. - **Persuade:** who must act, what they should believe, which real proof, content, or assets earn that belief. - **Operate:** the task, information, important states, frequency, constraints. @@ -72,7 +72,7 @@ Your measured rendition prior: warm, bookish, family, and child-facing subjects ## 5. Record the decision -Before code, record the chosen direction as a development-only contract under `## Direction contract` in the relevant surface brief. A direction contract is durable route or artifact strategy, so create or update the brief even when no other surface strategy needs persistence. Keep the contract to six short blocks and 150 words at most. THESIS: the one idea this surface owns and the category-default arrangement it refuses. OWN-WORLD: the palette and component language, specific enough to be recognizable with all content removed. STORY: what the visitor understands, believes, and does. FIRST VIEWPORT: the exact composition, what is where and at what scale, and where the primary action sits. FORM: the chosen form, its position on your ordered list, and the seed key the script printed. Close with one more line, FINISH: the run's exit condition, verbatim "unreviewed and undocumented is unfinished; this build ends with the finish review, the verdict, DESIGN.md, and every shipping raster carrying its provenance". The surface brief is the reminder later agents reload across edits and sessions: a page that looks complete with the FINISH line undischarged is not done, it is abandoned at the finish line. If a block reads like a mood, the direction is not decided yet; the finishing review audits the render against this contract. +Before code, record the chosen direction as a development-only contract under `## Direction contract` in the relevant surface brief. A direction contract is durable route or artifact strategy, so create or update the brief even when no other surface strategy needs persistence. Use six short blocks, roughly 150 words; do not spend tool calls counting words. THESIS: the one idea this surface owns and the category-default arrangement it refuses. OWN-WORLD: the palette and component language, specific enough to be recognizable with all content removed. STORY: what the visitor understands, believes, and does. FIRST VIEWPORT: the exact composition, what is where and at what scale, and where the primary action sits. FORM: the chosen form, its position on your ordered list, and the seed key the script printed. Close with one more line, FINISH: the run's exit condition, verbatim "unreviewed and undocumented is unfinished; this build ends with the finish review, the verdict, DESIGN.md, and every shipping raster carrying its provenance". The surface brief is the reminder later agents reload across edits and sessions: a page that looks complete with the FINISH line undischarged is not done, it is abandoned at the finish line. If a block reads like a mood, the direction is not decided yet; the finishing review audits the render against this contract. Never copy the direction contract into implementation source or any browser-delivered artifact. This includes HTML or framework comments, hidden DOM, `