skill: simplify context loading and inline register directive

Replaces load-context.mjs's JSON output with a tight markdown block from
the renamed context.mjs. The script now extracts PRODUCT.md's `## Register`
field and appends a `NEXT STEP:` directive naming the matching reference
(brand.md / product.md), which moved Gemini from skipping the register
load entirely to honoring it. Drops the `.impeccable.md` auto-migration;
makes IMPECCABLE_CONTEXT_DIR a lazy escape hatch consulted only when the
default paths come up empty.

Setup is now four bullets in one list. The DESIGN.md nudge is gone; in
its place, a "familiarize with the existing design system" step that
calls out CSS / tokens / running app as authoritative sources alongside
DESIGN.md. The standalone `### Register` H3 stays for the cascade rules
(task cue → surface → register field).

New LLM-backed test suite at tests/skill-behavior/ runs five scenarios
against claude-haiku-4-5, gpt-5.4-mini, and gemini-3.1-flash-lite via
Vercel AI SDK. Captures real tool traces, asserts on context.mjs calls,
brand.md loads, and teach.md fallback. Skips cleanly when API keys are
unset. 13-14/15 pass; only stable failure is the v3.2.0-era gpt-mini S4
"don't re-run" regression. Adds @ai-sdk/google as devDep and the
test:skill-behavior npm script.

Touches em-dashes in skill/SKILL.md and four reference files so
`bun run build:skills` passes its skill-prose validator. teach.md and
document.md drop their "re-run the loader to refresh session cache"
steps since the agent's own write is now the freshest source.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Bakaus
2026-05-20 15:25:31 -07:00
co-authored by Claude Opus 4.7
parent 0d32da57bc
commit 33467e5f0f
19 changed files with 1073 additions and 260 deletions
+9
View File
@@ -13,6 +13,7 @@
},
"devDependencies": {
"@ai-sdk/anthropic": "^3.0.71",
"@ai-sdk/google": "^3.0.75",
"@ai-sdk/openai": "^3.0.53",
"@anthropic-ai/claude-agent-sdk": "^0.2.119",
"@anthropic-ai/sdk": "^0.91.1",
@@ -37,6 +38,8 @@
"@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.104", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ZKX5n74io8VIRlhIMSLWVlvT3sXC8Z7cZ9GHuWBWZDVi96+62AIsWuLGvMfcBA1STYuSoDrp6rIziZmvrTq0TA=="],
"@ai-sdk/google": ["@ai-sdk/google@3.0.75", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-XAm31ftiOrzlb8NjDzT7kw0xw+4lmgFdGFn1QKM73nXFFKyN1kWLESBV75UGNfjXP8X1YJ0YydnMVqO0jaPghw=="],
"@ai-sdk/openai": ["@ai-sdk/openai@3.0.53", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Wld+Rbc05KaUn08uBt06eEuwcgalcIFtIl32Yp+GxuZXUQwOb6YeAuq+C6da4ch6BurFoqEaLemJVwjBb7x+PQ=="],
"@ai-sdk/provider": ["@ai-sdk/provider@3.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ=="],
@@ -1285,6 +1288,10 @@
"zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="],
"@ai-sdk/google/@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="],
"@ai-sdk/google/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.27", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw=="],
"@anthropic-ai/claude-agent-sdk/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.81.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-D4K5PvEV6wPiRtVlVsJHIUhHAmOZ6IT/I9rKlTf84gR7GyyAurPJK7z9BOf/AZqC5d1DhYQGJNKRmV+q8dGhgw=="],
"@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="],
@@ -1329,6 +1336,8 @@
"yauzl/buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="],
"@ai-sdk/google/@ai-sdk/provider-utils/eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="],
"@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="],
"@isaacs/cliui/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
+4 -2
View File
@@ -52,6 +52,7 @@
"deploy": "bun run build && wrangler pages deploy build/",
"test": "bun test tests/build.test.js tests/detect-antipatterns.test.js tests/windows-path-fix.test.js tests/lib/detector-bundle.test.js tests/lib/provider-blocks.test.js tests/lib/transformers/provider-blocks.test.js && node --test tests/critique-storage.test.mjs && node --test tests/detect-antipatterns-fixtures.test.mjs && node --test tests/detect-antipatterns-browser.test.mjs && node --test tests/cleanup-deprecated.test.mjs && node --test tests/impeccable-paths.test.mjs && node --test tests/live-wrap.test.mjs && node --test tests/live-reference.test.mjs && node --test tests/live-accept.test.mjs && node --test tests/live-inject.test.mjs && node --test tests/live-poll.test.mjs && node --test tests/live-server.test.mjs && node --test tests/live-browser-regression.test.mjs && node --test tests/live-session-store.test.mjs && node --test tests/live-browser-session.test.mjs && node --test tests/live-browser-source.test.mjs && node --test tests/live-completion.test.mjs && node --test tests/live-recovery-commands.test.mjs && node --test tests/framework-fixtures.test.mjs",
"test:live-e2e": "node --test --test-timeout=600000 tests/live-e2e.test.mjs",
"test:skill-behavior": "node --test --test-timeout=300000 tests/skill-behavior/scenarios.test.mjs",
"bench:detector": "node scripts/benchmark-detector.mjs",
"bench:detector:browser": "node scripts/benchmark-detector.mjs --browser",
"audit": "bun audit --audit-level=moderate",
@@ -74,15 +75,16 @@
"puppeteer": "^24.42.0"
},
"devDependencies": {
"astro": "^6.2.1",
"@paper-design/shaders": "^0.0.76",
"@ai-sdk/anthropic": "^3.0.71",
"@ai-sdk/google": "^3.0.75",
"@ai-sdk/openai": "^3.0.53",
"@anthropic-ai/claude-agent-sdk": "^0.2.119",
"@anthropic-ai/sdk": "^0.91.1",
"@google/genai": "^1.50.1",
"@paper-design/shaders": "^0.0.76",
"ai": "^6.0.168",
"archiver": "^7.0.1",
"astro": "^6.2.1",
"modern-screenshot": "^4.7.0",
"motion": "^12.38.0",
"playwright": "^1.59.1",
+14 -48
View File
@@ -5,53 +5,19 @@ argument-hint: "[{{command_hint}}] [target]"
user-invocable: true
allowed-tools:
- Bash(npx impeccable *)
license: Apache 2.0. Based on Anthropic's frontend-design skill. See NOTICE.md for attribution.
license: Apache 2.0
---
Designs and iterates production-grade frontend interfaces. Real working code, committed design choices, exceptional craft.
## Setup
Before any design work or file edits:
You MUST do these steps before proceeding:
1. Load context (PRODUCT.md / DESIGN.md) via the loader script.
2. Identify the register and load the matching register reference (brand.md or product.md).
3. **If the user invoked a sub-command (e.g. `craft`, `shape`, `audit`), load its reference file too.** This is non-negotiable: `craft` without `craft.md` loaded means you'll skip the shape-and-confirm step the user expects.
Skipping these produces generic output that ignores the project.
### 1. Context gathering
Two files, case-insensitive. The loader looks at the project root by default and falls back to `.agents/context/` and `docs/` if the root is clean. Override with `IMPECCABLE_CONTEXT_DIR=path/to/dir` (absolute or relative to cwd).
- **PRODUCT.md**: required. Users, brand, tone, anti-references, strategic principles.
- **DESIGN.md**: optional, strongly recommended. Colors, typography, elevation, components.
Load both in one call:
```bash
node {{scripts_path}}/load-context.mjs
```
Consume the full JSON output. Never pipe through `head`, `tail`, `grep`, or `jq`. The output's `contextDir` field tells you where the files were resolved from.
If the output is already in this session's conversation history, don't re-run. Exceptions requiring a fresh load: you just ran `{{command_prefix}}impeccable teach` or `{{command_prefix}}impeccable document` (they rewrite the files), or the user manually edited one.
`{{command_prefix}}impeccable live` already warms context via `live.mjs`. If you've run `live.mjs`, don't also run `load-context.mjs` this session.
If PRODUCT.md is missing, empty, or placeholder (`[TODO]` markers, <200 chars): run `{{command_prefix}}impeccable teach`, then resume the user's original task with the fresh context. If the original task was `{{command_prefix}}impeccable craft`, resume into `{{command_prefix}}impeccable shape` before any implementation work.
If DESIGN.md is missing: nudge once per session (*"Run `{{command_prefix}}impeccable document` for more on-brand output"*), then proceed.
### 2. Register
Every design task is **brand** (marketing, landing, campaign, long-form content, portfolio: design IS the product) or **product** (app UI, admin, dashboard, tool: design SERVES the product).
Identify before designing. Priority: (1) cue in the task itself ("landing page" vs "dashboard"); (2) the surface in focus (the page, file, or route being worked on); (3) `register` field in PRODUCT.md. First match wins.
If PRODUCT.md lacks the `register` field (legacy), infer it once from its "Users" and "Product Purpose" sections, then cache the inferred value for the session. Suggest the user run `{{command_prefix}}impeccable teach` to add the field explicitly.
Load the matching reference: [reference/brand.md](reference/brand.md) or [reference/product.md](reference/product.md). The shared design laws below apply to both.
1. Run `node {{scripts_path}}/context.mjs` once per session. If you've already seen its output in this conversation, do not re-run it. The script either prints the project's PRODUCT.md (and DESIGN.md when present) as a markdown block, or tells you it's missing. Follow whatever it prints. **If it reports `NO_PRODUCT_MD`, stop and follow `reference/teach.md` before doing anything else.**
2. Familiarize yourself with any existing design system, conventions, and components in the code. Don't reinvent the wheel; use what's there when it works, branch out when the UX wins.
3. Read the matching register reference. **This is non-optional; skipping it produces generic output.** If the project is marketing, a landing page, a campaign, long-form content, or a portfolio (design IS the product), read `reference/brand.md`. If it is app UI, admin, a dashboard, or a tool (design SERVES the product), read `reference/product.md`. Pick by first match: (1) task cue ("landing page" vs "dashboard"); (2) surface in focus (the page, file, or route being worked on); (3) `register` field in PRODUCT.md.
4. If the user invoked a sub-command (`craft`, `shape`, `audit`, ...), load its reference too.
## Shared design laws
@@ -60,14 +26,14 @@ Apply to every design, both registers. Match implementation complexity to the ae
### Color
- Use OKLCH. Reduce chroma as lightness approaches 0 or 100; high chroma at extremes looks garish.
- **The cream / sand / beige body bg is the saturated AI default of 2026.** The whole warm-neutral band OKLCH L 0.84-0.97, C < 0.06, hue 40-100 reads as cream/sand/paper/parchment regardless of what you call it. Token names like `--paper`, `--cream`, `--sand`, `--bone`, `--flour`, `--linen`, `--parchment`, `--wheat`, `--biscuit`, `--ivory` are tells in themselves. If the brief is "warm, traditional, family-coastal-Italian" or "magazine-warm" or "editorial-restraint", DO NOT translate that into a near-white warm-tinted bg that's the AI move. Pick: (a) a saturated brand color as the body (terracotta, oxblood, deep ochre, near-black), (b) a true off-white at chroma 0 (or chroma toward the brand's own hue, not toward warmth-by-default), or (c) a darker mid-tone tinted neutral that's clearly the brand's own. "Warmth" in the brand is carried by accent + typography + imagery, not by body bg.
- **The cream / sand / beige body bg is the saturated AI default of 2026.** The whole warm-neutral band (OKLCH L 0.84-0.97, C < 0.06, hue 40-100) reads as cream/sand/paper/parchment regardless of what you call it. Token names like `--paper`, `--cream`, `--sand`, `--bone`, `--flour`, `--linen`, `--parchment`, `--wheat`, `--biscuit`, `--ivory` are tells in themselves. If the brief is "warm, traditional, family-coastal-Italian" or "magazine-warm" or "editorial-restraint", DO NOT translate that into a near-white warm-tinted bg; that's the AI move. Pick: (a) a saturated brand color as the body (terracotta, oxblood, deep ochre, near-black), (b) a true off-white at chroma 0 (or chroma toward the brand's own hue, not toward warmth-by-default), or (c) a darker mid-tone tinted neutral that's clearly the brand's own. "Warmth" in the brand is carried by accent + typography + imagery, not by body bg.
- Pick a **color strategy** before picking colors. Four steps on the commitment axis:
- **Restrained**: tinted neutrals + one accent ≤10%. Product default; brand minimalism.
- **Committed**: one saturated color carries 3060% of the surface. Brand default for identity-driven pages.
- **Full palette**: 34 named roles, each used deliberately. Brand campaigns; product data viz.
- **Drenched**: the surface IS the color. Brand heroes, campaign pages.
- The "one accent ≤10%" rule is Restrained only. Committed / Full palette / Drenched exceed it on purpose. Don't collapse every design to Restrained by reflex.
- **Verify contrast.** Body text must hit ≥4.5:1 against its background; large text (≥18px or bold ≥14px) needs ≥3:1. The most common failure: muted gray body text on a tinted near-white. If the contrast is even close, bump the body color toward the ink end of the ramp light gray "for elegance" is the single biggest reason AI designs feel hard to read.
- **Verify contrast.** Body text must hit ≥4.5:1 against its background; large text (≥18px or bold ≥14px) needs ≥3:1. The most common failure: muted gray body text on a tinted near-white. If the contrast is even close, bump the body color toward the ink end of the ramp; light gray "for elegance" is the single biggest reason AI designs feel hard to read.
### Theme
@@ -84,7 +50,7 @@ Before choosing, write one sentence of physical scene: who uses this, where, und
- Cap font-family count at 3 (display + body + optional mono). More than 3 reads as indecision, not richness. One well-tuned family with weight contrast usually beats three competing typefaces.
- No all-caps body copy. Reserve uppercase for short labels (≤4 words), section eyebrows (used sparingly per the Absolute bans), and badges. Sentences in ALL CAPS are unreadable at body sizes.
- Hero / display heading ceiling: clamp() max ≤ 6rem (~96px). Above that the page is shouting, not designing.
- Display heading letter-spacing floor: ≥ -0.04em. Anything tighter and letters touch cramped, not "designed".
- Display heading letter-spacing floor: ≥ -0.04em. Anything tighter and letters touch; cramped, not "designed".
<codex>
Two hard typographic ceilings you currently miss:
@@ -104,7 +70,7 @@ Two hard typographic ceilings you currently miss:
- Ease out with exponential curves (ease-out-quart / quint / expo). No bounce, no elastic.
<gemini>
**Gemini-specific defect hard ban.** Never animate `<img>` elements on hover. This includes any `transform` on `:hover` of an image, AND `.group:hover .group-hover\:scale` / `.group:hover .group-hover\:rotate` / `.group:hover .group-hover\:translate` patterns from Tailwind that animate a child image via a parent hover. This is your single most common motion tell it adds no information (the image isn't an action target) and reads as "AI animated this because it could". If a card needs hover feedback, animate the card's background, border, or shadow. Never the image, never via the image's parent.
**Gemini-specific defect: hard ban.** Never animate `<img>` elements on hover. This includes any `transform` on `:hover` of an image, AND `.group:hover .group-hover\:scale` / `.group:hover .group-hover\:rotate` / `.group:hover .group-hover\:translate` patterns from Tailwind that animate a child image via a parent hover. This is your single most common motion tell; it adds no information (the image isn't an action target) and reads as "AI animated this because it could". If a card needs hover feedback, animate the card's background, border, or shadow. Never the image, never via the image's parent.
</gemini>
### Absolute bans
@@ -117,15 +83,15 @@ Match-and-refuse. If you're about to write any of these, rewrite the element wit
- **The hero-metric template.** Big number, small label, supporting stats, gradient accent. SaaS cliché.
- **Identical card grids.** Same-sized cards with icon + heading + text, repeated endlessly.
- **Modal as first thought.** Modals are usually laziness. Exhaust inline / progressive alternatives first.
- **Tiny uppercase tracked eyebrow above every section.** The 2023-era kicker (small all-caps text with wide tracking, "ABOUT" "PROCESS" "PRICING" above each heading) is now the saturated AI scaffold it appears on 55-95% of generations regardless of brief, which is the definition of a tell. One named kicker as a deliberate brand system is voice; an eyebrow on every section is AI grammar. Choose a different cadence.
- **Tiny uppercase tracked eyebrow above every section.** The 2023-era kicker (small all-caps text with wide tracking, "ABOUT" "PROCESS" "PRICING" above each heading) is now the saturated AI scaffold; it appears on 55-95% of generations regardless of brief, which is the definition of a tell. One named kicker as a deliberate brand system is voice; an eyebrow on every section is AI grammar. Choose a different cadence.
- **Numbered section markers (01 / 02 / 03).** Section labels as numbered display digits is the same trope, one tier deeper. If you find yourself reaching for them, you're scaffolding by reflex.
- **Text that overflows its container.** Long heading words plus large clamp scales plus narrow grids cause headline overflow on tablet/mobile. Test the heading copy at every breakpoint; if it overflows, reduce the clamp max or rewrite the copy. The viewport is part of the design.
<codex>
**Codex-specific defects** (your most-frequent giveaways refuse-and-rewrite):
**Codex-specific defects** (your most-frequent giveaways; refuse-and-rewrite):
- **`border: 1px solid X` + `box-shadow: 0 Npx Mpx ...` with M ≥ 16px** on the same element. The "ghost-card" pattern 1px border plus soft wide drop shadow on buttons and cards. Don't pair them. Pick one (a single solid border at the brand color, OR a defined shadow at no more than 8px blur), never both as decoration.
- **`border-radius: 32px+` on cards / sections / inputs.** You over-round. Cards top out at 1216px; full-pill is fine for tags/buttons. Picking 24/28/32/40px on a card is the codex tell no brand wants "insanely rounded".
- **`border: 1px solid X` + `box-shadow: 0 Npx Mpx ...` with M ≥ 16px** on the same element. The "ghost-card" pattern: 1px border plus soft wide drop shadow on buttons and cards. Don't pair them. Pick one (a single solid border at the brand color, OR a defined shadow at no more than 8px blur), never both as decoration.
- **`border-radius: 32px+` on cards / sections / inputs.** You over-round. Cards top out at 1216px; full-pill is fine for tags/buttons. Picking 24/28/32/40px on a card is the codex tell; no brand wants "insanely rounded".
- **Hand-drawn / sketchy SVG illustrations.** Class names like `loose-sketch`, `*-sketch`, `doodle`, `wavy`; `feTurbulence` / `feDisplacementMap` "paper grain" filters; 5-to-30 path crude scenes meant to depict a tangible subject (an otter, a table-and-fork, an album cover). All of these read as amateurish, not whimsical. If you can't render the scene with real assets, ship no illustration. Don't attempt sketchy SVG as a fallback.
- **`repeating-linear-gradient(...)` stripe backgrounds.** Diagonal stripes in `body:before` or section backgrounds are pure codex decoration. Don't.
- **"X theater" / "actually X" / "not just X, it's Y" copy.** "Productivity theater", "engagement theater", "growth theater": instant AI slop. Choose a specific noun, not a meta-criticism phrase.
+2 -2
View File
@@ -6,7 +6,7 @@ Add motion that conveys state, gives feedback, and clarifies hierarchy. Cut moti
## Register
Brand: motion is part of the voice; one well-rehearsed entrance beats scattered micro-interactions. The saturated AI default is fade-and-rise reveals on every scrolled section that's a tell, not a choreography. Reserve scroll-triggered motion for moments that earn it.
Brand: motion is part of the voice; one well-rehearsed entrance beats scattered micro-interactions. The saturated AI default is fade-and-rise reveals on every scrolled section; that's a tell, not a choreography. Reserve scroll-triggered motion for moments that earn it.
Product: 150250 ms on most transitions. Motion conveys state: feedback, reveal, loading, transitions between views. No page-load choreography; users are in a task and won't wait for it.
@@ -51,7 +51,7 @@ Add motion systematically across these categories:
### Entrance Animations
- **Hero section**: Dramatic entrance for primary content (scale, parallax, or creative effects)
- **Modal/drawer entry**: Smooth slide + fade, backdrop fade, focus management
- **List rhythm**: Sibling stagger is legitimate for cards-in-a-grid or list-items-appearing cap total stagger time per [motion-design.md](motion-design.md). Whole-section fade-on-scroll is not a list and is not legitimate.
- **List rhythm**: Sibling stagger is legitimate for cards-in-a-grid or list-items-appearing; cap total stagger time per [motion-design.md](motion-design.md). Whole-section fade-on-scroll is not a list and is not legitimate.
### Micro-interactions
- **Button feedback**:
+1 -1
View File
@@ -43,7 +43,7 @@ The reflex-reject lists apply to **new design choices**. When the existing brand
### Pairing and voice
Distinctive + refined is the goal. The specific shape depends on the brand, not on the brand's category. A category "restaurant", "dev tool", "magazine", "fintech" is not a recipe; treating it as one is the first-order reflex SKILL.md warns against.
Distinctive + refined is the goal. The specific shape depends on the brand, not on the brand's category. A category ("restaurant", "dev tool", "magazine", "fintech") is not a recipe; treating it as one is the first-order reflex SKILL.md warns against.
Two families minimum is the rule *only* when the voice needs it. A single well-chosen family with committed weight/size contrast is stronger than a timid display+body pair.
+1 -1
View File
@@ -24,7 +24,7 @@ Before generating anything, run a brief direction conversation grounded in the s
Ask **2-3 targeted questions** about visual lane, color strategy, atmosphere, and named anchor references. Don't enumerate generic menus; tie each question to the shape brief's answers. Example shape-grounded questions:
- "Brief says 'specimen-page restraint.' Are we closer to a quiet typographic page or a wider editorial spread with hero imagery?"
- "Palette strategy from shape was 'Committed.' Which one color carries the surface a brand-driven pick rather than a default warm-or-cool framing? (And no, the answer isn't a cream/sand body bg that's the saturated AI default.)"
- "Palette strategy from shape was 'Committed.' Which one color carries the surface (a brand-driven pick rather than a default warm-or-cool framing)? (And no, the answer isn't a cream/sand body bg; that's the saturated AI default.)"
**STOP and wait for answers.** These pin the palette before any pixel gets generated. Do not proceed to Step B until the user has responded.
+6 -4
View File
@@ -328,12 +328,13 @@ Pull directly from the DESIGN.md you just wrote:
Do not reword. The panel shows these as secondary collapsible context; the same voice that's in the Markdown carries through.
### Step 5: Confirm, refine, and refresh session cache
### Step 5: Confirm and refine
1. Show the user the full DESIGN.md you wrote. Briefly highlight the non-obvious creative choices (descriptive color names, atmosphere language, named rules).
2. Mention that `.impeccable/design.json` was also written alongside; the live panel will now render this project's actual button/input/nav primitives instead of generic approximations.
3. Offer to refine any section: "Want me to revise a section, add component patterns I missed, or adjust the atmosphere language?"
4. **Refresh the session cache.** Run `node {{scripts_path}}/load-context.mjs` one final time so the newly-written DESIGN.md lands in conversation. Subsequent commands in this session will use the fresh version automatically without re-reading.
Your own write is the freshest source; subsequent commands in this session don't need a reload.
## Seed mode
@@ -394,11 +395,12 @@ Per-section guidance in seed mode:
Seed mode writes a minimal frontmatter with `name` and `description` only; no colors, typography, rounded, spacing, or components yet. Real tokens land on the next Scan-mode run. Skip the `.impeccable/design.json` sidecar in seed mode for the same reason: nothing to render.
### Step 4: Confirm and refresh session cache
### Step 4: Confirm
1. Show the seed DESIGN.md. Call out that it is a seed (the marker is the literal commitment).
2. Tell the user: "Re-run `/impeccable document` once you have some code. That pass will extract real tokens and generate the sidecar."
3. Run `node {{scripts_path}}/load-context.mjs` once so the seed lands in conversation for the rest of the session.
Your own write is the freshest source; no reload needed.
## Style guidelines
+6 -2
View File
@@ -4,12 +4,16 @@ Interactive live variant mode: select elements in the browser, pick a design act
A running dev server with hot module replacement (Vite, Next.js, Bun, etc.), OR a static HTML file open in the browser.
<codex>
Codex: run live helper commands, the app dev server, and any dependency-installing setup with `sandbox_permissions: "require_escalated"` from the start; live mode depends on localhost and package-manager network access that the sandbox blocks.
</codex>
## The contract (read once)
Execute in order. No step skipped, no step reordered.
1. `live.mjs`: boot.
2. Navigate to the URL that serves `pageFile` (infer from `package.json`, docs, terminal output, or an open tab). If you can't infer it confidently, tell the user once to open their dev/preview URL. Never use `serverPort` as that URL; it's the helper, not the app.
2. Open the app URL that serves `pageFile` (infer from `package.json`, docs, terminal output, or an open tab). Never use `serverPort`; it's the helper, not the app. **Cursor:** `browser_navigate` to that URL before polling; do not skip. **Other harnesses:** use the available browser tool; if the URL is uncertain, ask the user once.
3. Poll loop with the default long timeout (600000 ms). After every event or `--reply`, run `live-poll.mjs` again immediately. Never pass a short `--timeout=`.
4. On `generate`: read screenshot if present; load the action's reference; plan three distinct directions; write all variants in one edit; `--reply done`; poll again.
5. On `accept` / `discard`: the poll script runs `live-accept.mjs`, acknowledges the delivered event, and prints `_completionAck`. Plain accepts/discards are terminal immediately; carbonize accepts remain recoverable until you finish cleanup, run `live-complete.mjs --id EVENT_ID`, and only then poll again.
@@ -30,7 +34,7 @@ Chat is overhead. No recap, no tutorial output, no pasting PRODUCT / DESIGN bodi
node {{scripts_path}}/live.mjs
```
Output JSON: `{ ok, serverPort, serverToken, pageFiles, hasProduct, product, productPath, hasDesign, design, designPath, migrated }`. `pageFiles` is the list of HTML entries the live script was injected into. Keep PRODUCT.md and DESIGN.md in mind for variant generation; **DESIGN.md wins on visual decisions; PRODUCT.md wins on strategic/voice decisions.** When DESIGN.md is missing, identity is **not** absent; extract it from CSS variables, computed styles, and sibling components on the page (see Step 4 Phase A). Identity preservation is the default; departure from existing identity requires an explicit trigger from PRODUCT.md anti-references or the user's freeform prompt. If `migrated: true`, the loader auto-renamed legacy `.impeccable.md` to `PRODUCT.md`; mention this once and suggest `/impeccable document` for the matching DESIGN.md.
Output JSON: `{ ok, serverPort, serverToken, pageFiles, hasProduct, product, productPath, hasDesign, design, designPath }`. `pageFiles` is the list of HTML entries the live script was injected into. Keep PRODUCT.md and DESIGN.md in mind for variant generation; **DESIGN.md wins on visual decisions; PRODUCT.md wins on strategic/voice decisions.** When DESIGN.md is missing, identity is **not** absent; extract it from CSS variables, computed styles, and sibling components on the page (see Step 4 Phase A). Identity preservation is the default; departure from existing identity requires an explicit trigger from PRODUCT.md anti-references or the user's freeform prompt.
`serverPort` and `serverToken` belong to the small **Impeccable live helper** HTTP server (serves `/live.js`, SSE, and `/poll`). That port is **not** your dev server and is usually not the URL you open to view the app. The browser page is whatever origin serves one of the `pageFiles` entries (Vite / Next / Bun / tunnel / LAN hostname).
+3 -11
View File
@@ -9,13 +9,7 @@ Every other impeccable command reads these files before doing any work.
## Step 1: Load current state
Run the shared loader first so you know what already exists:
```bash
node {{scripts_path}}/load-context.mjs
```
The output tells you whether PRODUCT.md and/or DESIGN.md already exist. If `migrated: true`, legacy `.impeccable.md` was auto-renamed to `PRODUCT.md`. Mention this once to the user.
Check what already exists. PRODUCT.md and DESIGN.md live at the project root, or under `.agents/context/` or `docs/` (case-insensitive). Read whichever are present with your native file tool.
Decision tree:
- **Neither file exists (empty project or no context yet)**: do Steps 2-4 (write PRODUCT.md), then decide on DESIGN.md based on whether there's code to analyze.
@@ -26,7 +20,7 @@ Decision tree:
Never silently overwrite an existing file. Always confirm first.
If teach was invoked as a setup blocker by another command, such as `{{command_prefix}}impeccable craft landing page`, pause that command here. Complete teach, re-run the loader, then resume the original command with the freshly loaded context. For craft, resume into shape next; teach creates project context, but it is not a substitute for the task-specific shape interview and confirmed design brief.
If teach was invoked as a setup blocker by another command, such as `{{command_prefix}}impeccable craft landing page`, pause that command here. Complete teach, then resume the original command. Your own writes are the freshest source; no reload needed. For craft, resume into shape next; teach creates project context, but it is not a substitute for the task-specific shape interview and confirmed design brief.
## Step 2: Explore the codebase
@@ -148,8 +142,6 @@ Summarize:
- The 3-5 strategic principles from PRODUCT.md that will guide future work
- If DESIGN.md is pending, remind the user how to generate it later
**Critical: re-run the loader to refresh session context.** After writing PRODUCT.md, run `node {{scripts_path}}/load-context.mjs` one final time and let its full JSON output land in conversation. This ensures subsequent commands in this session use the freshly-written PRODUCT.md, not a stale earlier version.
If teach was invoked as a blocker by another impeccable command (e.g. the user ran `/impeccable polish` with no PRODUCT.md), resume that original task now with the fresh context.
If teach was invoked as a blocker by another impeccable command (e.g. the user ran `/impeccable polish` with no PRODUCT.md), resume that original task now. Your own writes are the freshest source; no reload needed.
Optionally {{ask_instruction}} Ask whether they'd like a brief summary of PRODUCT.md appended to {{config_file}} for easier agent reference. If yes, append a short **Design Context** pointer section there.
+125
View File
@@ -0,0 +1,125 @@
/**
* Context loader: prints PRODUCT.md (and DESIGN.md if present) as one
* markdown block on stdout, or exits with empty stdout when no PRODUCT.md
* is found anywhere. The skill keys off "empty stdout" to branch into the
* teach flow.
*
* Path resolution (first match wins):
* 1. cwd, if PRODUCT.md or DESIGN.md is there
* 2. .agents/context/ then docs/
* 3. $IMPECCABLE_CONTEXT_DIR (absolute or cwd-relative) — power-user
* escape hatch, only consulted when defaults are empty
* 4. cwd as a "nothing found" default
*
* `resolveContextDir()` and `loadContext()` are also exported for the
* server-side scripts (live.mjs, live-server.mjs) that need the structured
* shape rather than the markdown block.
*/
import fs from 'node:fs';
import path from 'node:path';
const PRODUCT_NAMES = ['PRODUCT.md', 'Product.md', 'product.md'];
const DESIGN_NAMES = ['DESIGN.md', 'Design.md', 'design.md'];
const FALLBACK_DIRS = ['.agents/context', 'docs'];
export function resolveContextDir(cwd = process.cwd()) {
if (firstExisting(cwd, [...PRODUCT_NAMES, ...DESIGN_NAMES])) {
return cwd;
}
for (const rel of FALLBACK_DIRS) {
const candidate = path.resolve(cwd, rel);
if (firstExisting(candidate, [...PRODUCT_NAMES, ...DESIGN_NAMES])) {
return candidate;
}
}
const envDir = process.env.IMPECCABLE_CONTEXT_DIR;
if (envDir && envDir.trim()) {
const trimmed = envDir.trim();
return path.isAbsolute(trimmed) ? trimmed : path.resolve(cwd, trimmed);
}
return cwd;
}
export function loadContext(cwd = process.cwd()) {
const contextDir = resolveContextDir(cwd);
const productPath = firstExisting(contextDir, PRODUCT_NAMES);
const designPath = firstExisting(contextDir, DESIGN_NAMES);
const product = productPath ? safeRead(productPath) : null;
const design = designPath ? safeRead(designPath) : null;
return {
hasProduct: !!product,
product,
productPath: productPath ? path.relative(cwd, productPath) : null,
hasDesign: !!design,
design,
designPath: designPath ? path.relative(cwd, designPath) : null,
contextDir,
};
}
function firstExisting(dir, names) {
for (const name of names) {
const abs = path.join(dir, name);
if (fs.existsSync(abs)) return abs;
}
return null;
}
function safeRead(p) {
try {
return fs.readFileSync(p, 'utf-8');
} catch {
return null;
}
}
/**
* Pull the register (`brand` or `product`) out of PRODUCT.md by looking
* for a `## Register` section and reading the first non-empty line that
* follows it. Returns null when the file is legacy / register-less.
*/
function extractRegister(product) {
if (!product) return null;
const lines = product.split('\n');
for (let i = 0; i < lines.length; i++) {
if (/^##\s+Register\b/i.test(lines[i].trim())) {
for (let j = i + 1; j < lines.length; j++) {
const next = lines[j].trim();
if (!next) continue;
const word = next.toLowerCase();
if (word === 'brand' || word === 'product') return word;
return null;
}
}
}
return null;
}
function cli() {
const ctx = loadContext(process.cwd());
if (!ctx.hasProduct) {
// Direct stdout message instead of relying on empty output as a signal
// — cheap models miss the empty case more often than the explicit one.
process.stdout.write(
'NO_PRODUCT_MD: This project has no PRODUCT.md yet. ' +
'Stop the current task, load reference/teach.md, and follow its ' +
'instructions to write PRODUCT.md before resuming.\n',
);
process.exit(0);
}
const parts = [`# PRODUCT.md\n\n${ctx.product.trim()}`];
if (ctx.hasDesign) {
parts.push(`# DESIGN.md\n\n${ctx.design.trim()}`);
}
const register = extractRegister(ctx.product);
const next = register
? `NEXT STEP: This project's register is \`${register}\`. You MUST now read \`reference/${register}.md\` before producing any design output.`
: `NEXT STEP: You MUST now read the matching register reference (\`reference/brand.md\` or \`reference/product.md\`) before producing any design output. Pick based on PRODUCT.md above.`;
parts.push(next);
process.stdout.write(parts.join('\n\n---\n\n') + '\n');
}
const _running = process.argv[1];
if (_running?.endsWith('context.mjs') || _running?.endsWith('context.mjs/')) {
cli();
}
+3 -4
View File
@@ -21,7 +21,7 @@ import path from 'node:path';
import net from 'node:net';
import { fileURLToPath } from 'node:url';
import { parseDesignMd } from './design-parser.mjs';
import { resolveContextDir } from './load-context.mjs';
import { resolveContextDir } from './context.mjs';
import { createLiveSessionStore } from './live-session-store.mjs';
import {
getDesignSidecarPath,
@@ -33,7 +33,7 @@ import {
} from './impeccable-paths.mjs';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
// PRODUCT.md / DESIGN.md live wherever load-context.mjs resolves. The generated
// PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated
// DESIGN sidecar is project-local at .impeccable/design.json, with legacy
// DESIGN.json fallback for existing projects.
const CONTEXT_DIR = resolveContextDir(process.cwd());
@@ -187,8 +187,7 @@ function loadBrowserScripts() {
function hasProjectContext() {
// PRODUCT.md carries brand voice / anti-references — that's what determines
// whether variants are brand-aware. DESIGN.md (visual tokens) is a separate
// concern, surfaced by the design panel's own empty state. Legacy
// .impeccable.md is auto-migrated to PRODUCT.md by load-context.mjs.
// concern, surfaced by the design panel's own empty state.
try {
fs.accessSync(path.join(CONTEXT_DIR, 'PRODUCT.md'), fs.constants.R_OK);
return true;
+2 -3
View File
@@ -21,7 +21,7 @@ import { execSync } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { loadContext } from './load-context.mjs';
import { loadContext } from './context.mjs';
import { resolveFiles } from './live-inject.mjs';
import { readLiveServerInfo } from './impeccable-paths.mjs';
@@ -80,7 +80,7 @@ The agent should then:
process.exit(1);
}
// 4. Load PRODUCT.md + DESIGN.md context (auto-migrates legacy .impeccable.md)
// 4. Load PRODUCT.md + DESIGN.md context.
const ctx = loadContext(process.cwd());
// 5. Compute drift-heal: compare resolved inject targets against the
@@ -102,7 +102,6 @@ The agent should then:
hasDesign: ctx.hasDesign,
design: ctx.design,
designPath: ctx.designPath,
migrated: ctx.migrated,
}, null, 2));
}
-141
View File
@@ -1,141 +0,0 @@
/**
* Shared context loader for every impeccable command that needs to know
* "who is this for" and "what does this look like".
*
* Input: project root (process.cwd()).
*
* Output (JSON to stdout):
* {
* hasProduct: boolean, // PRODUCT.md found (or auto-migrated)
* product: string | null, // PRODUCT.md contents
* productPath: string | null, // relative path
* hasDesign: boolean, // DESIGN.md found
* design: string | null, // DESIGN.md contents
* designPath: string | null,
* migrated: boolean, // true if we auto-renamed .impeccable.md -> PRODUCT.md
* contextDir: string, // absolute path of the directory the files were found in
* }
*
* Filename matching is case-insensitive for PRODUCT.md and DESIGN.md. The
* Google DESIGN.md convention is uppercase at repo root; Kiro-style and
* lowercase variants are also matched so users don't get punished for case.
*
* Lookup directory resolution (first match wins):
* 1. process.env.IMPECCABLE_CONTEXT_DIR (absolute or relative to cwd)
* 2. cwd, if PRODUCT.md / DESIGN.md / .impeccable.md is there (back-compat)
* 3. Auto-fallback subdirectories of cwd: .agents/context/, then docs/
* 4. cwd as a default "no context found" location
*
* Legacy `.impeccable.md` -> PRODUCT.md migration only fires at cwd root;
* fallback directories are read-only as far as auto-rename is concerned.
*/
import fs from 'node:fs';
import path from 'node:path';
const PRODUCT_NAMES = ['PRODUCT.md', 'Product.md', 'product.md'];
const DESIGN_NAMES = ['DESIGN.md', 'Design.md', 'design.md'];
const LEGACY_NAMES = ['.impeccable.md'];
const FALLBACK_DIRS = ['.agents/context', 'docs'];
/**
* Resolve the directory that holds PRODUCT.md / DESIGN.md for
* this project. Exported so other scripts (e.g. live-server.mjs) can read the
* design files from the same location the loader uses.
*/
export function resolveContextDir(cwd = process.cwd()) {
// 1. Explicit override
const envDir = process.env.IMPECCABLE_CONTEXT_DIR;
if (envDir && envDir.trim()) {
const trimmed = envDir.trim();
return path.isAbsolute(trimmed) ? trimmed : path.resolve(cwd, trimmed);
}
// 2. cwd wins if any canonical or legacy file is there. We check legacy too
// so the auto-migration path in loadContext stays predictable.
if (firstExisting(cwd, [...PRODUCT_NAMES, ...DESIGN_NAMES, ...LEGACY_NAMES])) {
return cwd;
}
// 3. Auto-fallback subdirs. Match if PRODUCT.md or DESIGN.md is present;
// legacy `.impeccable.md` does not pull the lookup into a fallback dir.
for (const rel of FALLBACK_DIRS) {
const candidate = path.resolve(cwd, rel);
if (firstExisting(candidate, [...PRODUCT_NAMES, ...DESIGN_NAMES])) {
return candidate;
}
}
// 4. Nothing found — keep the historical "default to cwd" behaviour so the
// caller's `hasProduct === false` branch still fires the same way.
return cwd;
}
export function loadContext(cwd = process.cwd()) {
let migrated = false;
const contextDir = resolveContextDir(cwd);
// 1. Look for PRODUCT.md (case-insensitive) in the resolved dir
let productPath = firstExisting(contextDir, PRODUCT_NAMES);
// 2. Legacy: if no PRODUCT.md but .impeccable.md exists at cwd root, rename
// it in place. We only migrate at the root — fallback dirs are read-only
// so we don't surprise users by mutating files under docs/ or .agents/.
if (!productPath && contextDir === cwd) {
const legacyPath = firstExisting(cwd, LEGACY_NAMES);
if (legacyPath) {
const newPath = path.join(cwd, 'PRODUCT.md');
try {
fs.renameSync(legacyPath, newPath);
productPath = newPath;
migrated = true;
} catch {
// Rename failed (permissions, etc.) — fall back to reading legacy in place
productPath = legacyPath;
}
}
}
// 3. DESIGN.md (case-insensitive)
const designPath = firstExisting(contextDir, DESIGN_NAMES);
const product = productPath ? safeRead(productPath) : null;
const design = designPath ? safeRead(designPath) : null;
return {
hasProduct: !!product,
product,
productPath: productPath ? path.relative(cwd, productPath) : null,
hasDesign: !!design,
design,
designPath: designPath ? path.relative(cwd, designPath) : null,
migrated,
contextDir,
};
}
function firstExisting(dir, names) {
for (const name of names) {
const abs = path.join(dir, name);
if (fs.existsSync(abs)) return abs;
}
return null;
}
function safeRead(p) {
try { return fs.readFileSync(p, 'utf-8'); } catch { return null; }
}
// ---------------------------------------------------------------------------
// CLI mode — print the context as JSON
// ---------------------------------------------------------------------------
function cli() {
const result = loadContext(process.cwd());
console.log(JSON.stringify(result, null, 2));
}
const _running = process.argv[1];
if (_running?.endsWith('load-context.mjs') || _running?.endsWith('load-context.mjs/')) {
cli();
}
@@ -2,10 +2,11 @@
* Tests for the shared context loader (PRODUCT.md / DESIGN.md resolver).
* Run with: node --test tests/load-context.test.mjs
*
* Covers the resolution order added for issue #119:
* 1. IMPECCABLE_CONTEXT_DIR env var (absolute or relative)
* 2. cwd, when canonical or legacy files are at the root (back-compat)
* 3. Auto-fallback to .agents/context/ then docs/
* Covers the resolution order:
* 1. cwd, when canonical files are at the root
* 2. Auto-fallback to .agents/context/ then docs/
* 3. IMPECCABLE_CONTEXT_DIR env var as a power-user escape hatch (only
* consulted when the default paths come up empty)
* 4. Default to cwd when nothing is found
*
* Each test runs in its own scratch dir under os.tmpdir() so the suite stays
@@ -18,7 +19,10 @@ import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import { loadContext, resolveContextDir } from '../skill/scripts/load-context.mjs';
import { loadContext, resolveContextDir } from '../skill/scripts/context.mjs';
import { fileURLToPath } from 'node:url';
const SCRIPT_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'skill', 'scripts', 'context.mjs');
let scratch;
let savedEnv;
@@ -53,11 +57,6 @@ describe('resolveContextDir', () => {
assert.equal(resolveContextDir(scratch), scratch);
});
it('returns cwd when only legacy .impeccable.md is at the root', () => {
write('.impeccable.md');
assert.equal(resolveContextDir(scratch), scratch);
});
it('falls back to .agents/context/ when root is clean', () => {
write('.agents/context/PRODUCT.md');
assert.equal(resolveContextDir(scratch), path.join(scratch, '.agents', 'context'));
@@ -80,13 +79,13 @@ describe('resolveContextDir', () => {
assert.equal(resolveContextDir(scratch), scratch);
});
it('honors IMPECCABLE_CONTEXT_DIR with a relative path', () => {
it('uses IMPECCABLE_CONTEXT_DIR as a fallback when defaults are empty (relative path)', () => {
write('design/PRODUCT.md');
process.env.IMPECCABLE_CONTEXT_DIR = 'design';
assert.equal(resolveContextDir(scratch), path.join(scratch, 'design'));
});
it('honors IMPECCABLE_CONTEXT_DIR with an absolute path', () => {
it('uses IMPECCABLE_CONTEXT_DIR as a fallback when defaults are empty (absolute path)', () => {
const elsewhere = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-elsewhere-'));
try {
process.env.IMPECCABLE_CONTEXT_DIR = elsewhere;
@@ -96,11 +95,11 @@ describe('resolveContextDir', () => {
}
});
it('IMPECCABLE_CONTEXT_DIR wins even when files exist at the root', () => {
it('default paths win over IMPECCABLE_CONTEXT_DIR (lazy escape hatch)', () => {
write('PRODUCT.md', 'root');
write('design/PRODUCT.md', 'overridden');
process.env.IMPECCABLE_CONTEXT_DIR = 'design';
assert.equal(resolveContextDir(scratch), path.join(scratch, 'design'));
assert.equal(resolveContextDir(scratch), scratch);
});
it('ignores empty IMPECCABLE_CONTEXT_DIR', () => {
@@ -114,8 +113,8 @@ describe('resolveContextDir', () => {
});
});
describe('loadContext (backward compatibility)', () => {
it('reads PRODUCT.md and DESIGN.md from the root the same way as before', () => {
describe('loadContext', () => {
it('reads PRODUCT.md and DESIGN.md from the root', () => {
write('PRODUCT.md', '# product content\n');
write('DESIGN.md', '# design content\n');
const ctx = loadContext(scratch);
@@ -128,18 +127,6 @@ describe('loadContext (backward compatibility)', () => {
assert.equal(ctx.contextDir, scratch);
});
it('migrates legacy .impeccable.md -> PRODUCT.md at root', () => {
write('.impeccable.md', '# legacy body\n');
const ctx = loadContext(scratch);
assert.equal(ctx.migrated, true);
assert.equal(ctx.hasProduct, true);
assert.match(ctx.product, /legacy body/);
assert.ok(fs.existsSync(path.join(scratch, 'PRODUCT.md')));
assert.ok(!fs.existsSync(path.join(scratch, '.impeccable.md')));
});
});
describe('loadContext (fallback dirs)', () => {
it('reads from .agents/context/ when the root is clean', () => {
write('.agents/context/PRODUCT.md', '# product in agents\n');
write('.agents/context/DESIGN.md', '# design in agents\n');
@@ -151,7 +138,6 @@ describe('loadContext (fallback dirs)', () => {
// productPath/designPath are relative to cwd, not contextDir
assert.equal(ctx.productPath, path.join('.agents', 'context', 'PRODUCT.md'));
assert.equal(ctx.designPath, path.join('.agents', 'context', 'DESIGN.md'));
assert.equal(ctx.migrated, false);
});
it('reads from docs/ when .agents/context/ is empty', () => {
@@ -161,20 +147,10 @@ describe('loadContext (fallback dirs)', () => {
assert.equal(ctx.contextDir, path.join(scratch, 'docs'));
assert.equal(ctx.productPath, path.join('docs', 'PRODUCT.md'));
});
it('does not auto-migrate .impeccable.md inside fallback dirs', () => {
write('docs/.impeccable.md', '# legacy in docs\n');
const ctx = loadContext(scratch);
// .impeccable.md inside a fallback dir doesn't pull the lookup there,
// and we never auto-rename outside the cwd root.
assert.equal(ctx.hasProduct, false);
assert.equal(ctx.migrated, false);
assert.ok(fs.existsSync(path.join(scratch, 'docs', '.impeccable.md')));
});
});
describe('loadContext (IMPECCABLE_CONTEXT_DIR override)', () => {
it('reads from the override path when set', () => {
describe('loadContext (IMPECCABLE_CONTEXT_DIR escape hatch)', () => {
it('reads from the override path when defaults are empty', () => {
write('design/PRODUCT.md', '# overridden product\n');
write('design/DESIGN.md', '# overridden design\n');
process.env.IMPECCABLE_CONTEXT_DIR = 'design';
@@ -185,6 +161,15 @@ describe('loadContext (IMPECCABLE_CONTEXT_DIR override)', () => {
assert.equal(ctx.contextDir, path.join(scratch, 'design'));
});
it('does not override defaults when both exist (lazy escape hatch)', () => {
write('PRODUCT.md', '# root product\n');
write('design/PRODUCT.md', '# overridden product\n');
process.env.IMPECCABLE_CONTEXT_DIR = 'design';
const ctx = loadContext(scratch);
assert.match(ctx.product, /root product/);
assert.equal(ctx.contextDir, scratch);
});
it('reports a missing override directory as no-context, not as a crash', () => {
process.env.IMPECCABLE_CONTEXT_DIR = 'no/such/dir';
const ctx = loadContext(scratch);
@@ -195,3 +180,64 @@ describe('loadContext (IMPECCABLE_CONTEXT_DIR override)', () => {
assert.equal(ctx.contextDir, path.resolve(scratch, 'no/such/dir'));
});
});
describe('context.mjs CLI', () => {
it('emits NO_PRODUCT_MD directive when no PRODUCT.md is found', async () => {
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' });
assert.equal(res.status, 0);
assert.match(res.stdout, /^NO_PRODUCT_MD:/);
assert.match(res.stdout, /reference\/teach\.md/);
});
it('prints a PRODUCT.md markdown block when only PRODUCT.md exists', async () => {
write('PRODUCT.md', '# Acme\n\nbody\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' });
assert.equal(res.status, 0);
assert.match(res.stdout, /^# PRODUCT\.md/);
assert.match(res.stdout, /# Acme/);
assert.equal(res.stdout.includes('# DESIGN.md'), false);
// The NEXT STEP directive is always appended after `---`.
assert.match(res.stdout, /\n---\n\nNEXT STEP:/);
});
it('concatenates PRODUCT.md and DESIGN.md with a --- separator', async () => {
write('PRODUCT.md', '# Acme product\n');
write('DESIGN.md', '# Acme design\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' });
assert.equal(res.status, 0);
assert.match(res.stdout, /^# PRODUCT\.md/);
assert.match(res.stdout, /\n---\n/);
assert.match(res.stdout, /# DESIGN\.md\n\n# Acme design/);
assert.match(res.stdout, /NEXT STEP:/);
});
it('reads from a fallback dir when cwd is clean', async () => {
write('.agents/context/PRODUCT.md', '# fallback product\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' });
assert.equal(res.status, 0);
assert.match(res.stdout, /^# PRODUCT\.md/);
assert.match(res.stdout, /# fallback product/);
});
it('names the register-specific reference when PRODUCT.md declares one', async () => {
write('PRODUCT.md', '# Acme\n\n## Register\n\nbrand\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' });
assert.equal(res.status, 0);
assert.match(res.stdout, /NEXT STEP: This project's register is `brand`\./);
assert.match(res.stdout, /read `reference\/brand\.md`/);
});
it('falls back to a generic register directive when no register field is present', async () => {
write('PRODUCT.md', '# Acme\n\n(no register field)\n');
const { spawnSync } = await import('node:child_process');
const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' });
assert.equal(res.status, 0);
assert.match(res.stdout, /NEXT STEP: You MUST now read the matching register reference/);
assert.match(res.stdout, /reference\/brand\.md.*reference\/product\.md/);
});
});
+71
View File
@@ -0,0 +1,71 @@
# Skill-behavior tests
LLM-backed scenarios that verify how the impeccable skill drives
PRODUCT.md / DESIGN.md loading. Each scenario runs against the cheapest
tier of each major provider (Anthropic, OpenAI, Google) so a full sweep
costs a few cents and finishes in ~2 minutes.
These are the tests you re-run when you refactor anything in SKILL.md's
`## Setup` section. They fail when the agent stops following the loading
contract.
## Run
```bash
bun run test:skill-behavior
IMPECCABLE_SKILL_BEHAVIOR_VERBOSE=1 bun run test:skill-behavior # dump per-scenario traces
IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-haiku-4-5 bun run test:skill-behavior # scope to one model
```
Requires `.env` at repo root with at least one of `ANTHROPIC_API_KEY`,
`OPENAI_API_KEY`, `GOOGLE_CLOUD_API_KEY`. Providers without a key are
skipped, not failed.
## How it works
Each scenario:
1. `prepareWorkspace()` mints a temp dir, symlinks the canonical skill
into `<workspace>/.claude/skills/impeccable`, and optionally writes
`PRODUCT.md` / `DESIGN.md` fixtures.
2. `runTurn()` inlines `SKILL.md` (placeholders neutralized) as the
system prompt and runs Vercel AI SDK `generateText` with four
workspace-scoped tools: `bash`, `read`, `write`, `list`.
3. The tools record every call into a `trace` that the test asserts on.
4. For scenario 4, a second `runTurn` reuses turn 1's `responseMessages`
so the model sees a real multi-turn conversation.
The trace is the source of truth, not the model's free-form reply.
## Scenarios
| # | Setup | Assertion |
|---|---|---|
| 1 | empty workspace | runs `context.mjs` (which prints a `NO_PRODUCT_MD` directive); agent then loads `reference/teach.md` via Read or `cat`; does **not** start writing HTML/CSS |
| 2 | PRODUCT.md only (with `## Register: brand`) | runs `context.mjs` 1-3 times; loads `reference/brand.md` |
| 3 | PRODUCT.md + DESIGN.md (brand register) | runs `context.mjs` 1-3 times; loads `reference/brand.md`; consults the design system (DESIGN.md bundled in output, but CSS / tokens / directory listing also count) |
| 4 | PRODUCT.md + DESIGN.md, context already loaded in turn 1 | turn 2 does **not** re-run `context.mjs`; `reference/brand.md` is loaded across turns 1+2 |
| 5 | PRODUCT.md WITHOUT a `## Register` field; task cue says "landing page" | runs `context.mjs` (which emits a generic register directive); agent loads `reference/brand.md` via task-cue cascade |
## Baseline state (2026-05-20)
Captured after condensing Setup to four bullets and teaching `context.mjs`
to emit a `NEXT STEP:` directive that names the matching register
reference when PRODUCT.md declares one (and a generic cascade prompt when
it doesn't). Use this table when comparing pre/post refactor: a
regression is "more failures than baseline", not "any failures at all".
| Scenario | claude-haiku-4-5 | gpt-5.4-mini | gemini-3.1-flash-lite |
|---|---|---|---|
| 1 (no context) | pass (variance: ~1 in 5 the agent stops after `context.mjs` without loading `teach.md`) | pass | pass |
| 2 (product only) | pass | pass | pass |
| 3 (product + design) | pass | pass | pass |
| 4 (already loaded) | pass | **fail** | pass |
| 5 (no register field, task-cue cascade) | pass | pass | pass |
13-14 / 15 typical. The stable failure is gpt-5.4-mini scenario 4:
it re-runs `context.mjs` on turn 2 despite seeing its output in turn 1's
history. Same known weakness as the v3.2.0 script baseline; Claude and
Gemini honor the "don't re-run" rule. The S1 claude flake is rare
(observed once across many runs) and likely terminates early under
load — re-running typically clears it.
+98
View File
@@ -0,0 +1,98 @@
/**
* Realistic-looking PRODUCT.md / DESIGN.md fixtures.
*
* Long enough to clear the "<200 chars / placeholder" heuristic the loader
* uses to decide whether to gate on `teach`. Plausible enough that the agent
* treats them as real context rather than test scaffolding.
*/
export const PRODUCT_MD_SAMPLE = `# Acme Notes
## Register
brand
## Product Purpose
Acme Notes is a marketing-driven landing page for a research-grade note-taking
tool aimed at independent scientists and graduate students. The site needs to
communicate that the product respects the reader's intelligence — no SaaS
buzzwords, no metric-theater, no "trusted by leading teams" wallpaper.
## Users
Working researchers (PhD students, postdocs, principal investigators) who
already maintain disciplined note-taking systems and are choosing between
ours and rolling their own in a Zettelkasten plugin.
## Brand
Editorial, considered, technical. The product is for people who quote
Knuth. The voice is closer to a long-read magazine than to a startup
landing page.
## Anti-references
- Notion (too consumer / too rounded)
- Obsidian (too community-cottagecore)
- Any SaaS landing page with a hero-metric grid
## Strategic Principles
- Type does most of the work. The hero is words, not chrome.
- One named accent color, used sparingly.
- Never lead with screenshots. Lead with the idea.
`;
/**
* Same project shape as PRODUCT_MD_SAMPLE but with no `## Register` field.
* Exercises the cascade fallback (task cue then surface in focus) in
* scenarios where context.mjs cannot detect the register and the agent
* must follow the SKILL.md priority list to pick brand.md.
*/
export const PRODUCT_MD_SAMPLE_NO_REGISTER = `# Acme Notes
## Product Purpose
Acme Notes is a marketing-driven landing page for a research-grade note-taking
tool aimed at independent scientists and graduate students. The site needs to
communicate that the product respects the reader's intelligence: no SaaS
buzzwords, no metric-theater, no "trusted by leading teams" wallpaper.
## Users
Working researchers (PhD students, postdocs, principal investigators) who
already maintain disciplined note-taking systems and are choosing between
ours and rolling their own in a Zettelkasten plugin.
## Brand
Editorial, considered, technical. The product is for people who quote
Knuth. The voice is closer to a long-read magazine than to a startup
landing page.
## Anti-references
- Notion (too consumer / too rounded)
- Obsidian (too community-cottagecore)
- Any SaaS landing page with a hero-metric grid
## Strategic Principles
- Type does most of the work. The hero is words, not chrome.
- One named accent color, used sparingly.
- Never lead with screenshots. Lead with the idea.
`;
export const DESIGN_MD_SAMPLE = `# Acme Notes — Design System
## Colors
- \`--ink\`: oklch(0.16 0.02 250) — body copy
- \`--paper\`: oklch(0.98 0.01 90) — body background
- \`--accent\`: oklch(0.55 0.18 28) — terracotta, used at <8% surface
## Typography
- Display: GT Sectra (commercial), 700, tracking -0.02em
- Body: Inter, 400, 1.55 line-height, 65ch max
- Mono: JetBrains Mono, 400 (rare, only for callouts)
## Spacing
Multi-modular scale: 4 / 8 / 12 / 24 / 48 / 96 px.
## Elevation
Mostly flat. A single 1px hairline border at oklch(0.16 0.02 250 / 0.08)
separates major regions. No drop shadows under 16px blur.
## Components
- Buttons: text-only by default; a single solid primary in accent for CTAs.
- Cards: avoid; prefer hairlined regions and inline lists.
- Forms: floating labels, no border on the input — underline only.
`;
+296
View File
@@ -0,0 +1,296 @@
/**
* Sandboxed scenario runner for skill-behavior tests.
*
* Each scenario:
* 1. Creates a temp workspace.
* 2. Symlinks the real .claude/skills/impeccable into the workspace so
* scripts (load-context.mjs, etc.) resolve from the canonical path
* the skill references.
* 3. Optionally writes PRODUCT.md / DESIGN.md fixtures.
* 4. Inlines SKILL.md as the system prompt (placeholders stripped to
* neutral values so the same body works for all providers).
* 5. Runs Vercel AI SDK generateText with workspace-scoped tools
* (bash, read, write, list).
* 6. Captures every tool call and returns a trace + the raw response
* messages (so multi-turn scenarios can append to them).
*
* The harness deliberately mirrors the live-mode E2E pattern: real LLM,
* no mocks, but tightly bounded execution surface so we observe the routing
* behavior of the skill without paying for full-fledged design work.
*/
import { generateText, stepCountIs, tool } from 'ai';
import { z } from 'zod';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawn } from 'node:child_process';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(__dirname, '..', '..');
const SKILL_SOURCE_DIR = path.join(REPO_ROOT, 'skill');
const MAX_BASH_OUTPUT_BYTES = 200_000;
/**
* Strip the YAML frontmatter and replace `{{...}}` placeholders so SKILL.md
* is provider-neutral when inlined.
*/
function loadSkillBody() {
let md = fs.readFileSync(path.join(SKILL_SOURCE_DIR, 'SKILL.md'), 'utf8');
// Strip frontmatter.
if (md.startsWith('---')) {
const end = md.indexOf('\n---', 3);
if (end !== -1) md = md.slice(end + 4).trimStart();
}
// The source uses placeholders that the build step replaces per-provider.
// For the test harness we want a single body that works for any provider,
// and the scripts the skill references live at .claude/skills/impeccable/
// (the workspace symlink), so hard-code those values.
md = md
.replaceAll('{{model}}', 'the assistant')
.replaceAll('{{command_prefix}}', '/')
.replaceAll('{{ask_instruction}}', 'Ask the user')
.replaceAll('{{config_file}}', 'AGENTS.md')
.replaceAll('{{scripts_path}}', '.claude/skills/impeccable/scripts')
.replaceAll('{{command_hint}}', 'command');
return md.trim();
}
export const SKILL_BODY = loadSkillBody();
/**
* Create a temp workspace and prepopulate it.
*
* - `.claude/skills/impeccable` is symlinked at the SOURCE skill dir (not
* the built `.claude/skills/impeccable/`) so the test exercises whatever
* is in `skill/` right now, without needing `bun run build` to refresh
* the harness output dirs. The trade-off: reference files surface their
* raw `{{placeholders}}`, but the assertions only check tool calls, not
* their content.
* - `files` lets the test seed PRODUCT.md / DESIGN.md (or anything else).
*/
export function prepareWorkspace({ files = {} } = {}) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-skill-test-'));
fs.mkdirSync(path.join(dir, '.claude', 'skills'), { recursive: true });
fs.symlinkSync(SKILL_SOURCE_DIR, path.join(dir, '.claude', 'skills', 'impeccable'), 'dir');
for (const [name, contents] of Object.entries(files)) {
const target = path.join(dir, name);
fs.mkdirSync(path.dirname(target), { recursive: true });
fs.writeFileSync(target, contents);
}
return dir;
}
export function cleanupWorkspace(dir) {
try {
fs.rmSync(dir, { recursive: true, force: true });
} catch {
// Best effort — temp dirs eventually get reaped by the OS.
}
}
function safeResolve(root, userPath) {
if (typeof userPath !== 'string' || !userPath.length) {
return { error: 'path is required' };
}
if (userPath.startsWith('/') || /^[a-zA-Z]:[\\/]/.test(userPath)) {
return { error: 'absolute paths are not allowed' };
}
const resolved = path.resolve(root, userPath);
const rel = path.relative(root, resolved);
if (rel.startsWith('..') || rel.split(path.sep).includes('..')) {
return { error: 'path escapes the workspace' };
}
return resolved;
}
function execBash(workspace, command, timeoutMs = 20_000) {
return new Promise((resolve) => {
const proc = spawn('bash', ['-lc', command], { cwd: workspace });
let stdout = '';
let stderr = '';
const truncatedFlag = { val: false };
const onChunk = (which) => (chunk) => {
const str = chunk.toString();
if (which === 'out') {
if (stdout.length + str.length > MAX_BASH_OUTPUT_BYTES) {
stdout += str.slice(0, MAX_BASH_OUTPUT_BYTES - stdout.length);
truncatedFlag.val = true;
} else {
stdout += str;
}
} else {
if (stderr.length + str.length > MAX_BASH_OUTPUT_BYTES) {
stderr += str.slice(0, MAX_BASH_OUTPUT_BYTES - stderr.length);
truncatedFlag.val = true;
} else {
stderr += str;
}
}
};
proc.stdout.on('data', onChunk('out'));
proc.stderr.on('data', onChunk('err'));
const timer = setTimeout(() => {
proc.kill('SIGKILL');
resolve({ exitCode: null, stdout, stderr: stderr + '\n[TIMED OUT]', truncated: truncatedFlag.val });
}, timeoutMs);
proc.on('exit', (code) => {
clearTimeout(timer);
resolve({ exitCode: code, stdout, stderr, truncated: truncatedFlag.val });
});
proc.on('error', (err) => {
clearTimeout(timer);
resolve({ exitCode: null, stdout, stderr: stderr + `\n[SPAWN ERROR] ${String(err)}`, truncated: truncatedFlag.val });
});
});
}
/**
* Build the workspace-scoped tool set + the trace it writes into.
* Returns `{ tools, trace }`. The trace mutates in place as the agent runs.
*/
export function makeTools(workspace) {
const trace = {
toolCalls: [],
bashCommands: [],
readPaths: [],
writePaths: [],
listPaths: [],
};
function record(name, input) {
trace.toolCalls.push({ name, input });
if (name === 'bash' && typeof input?.command === 'string') trace.bashCommands.push(input.command);
if (name === 'read' && typeof input?.path === 'string') trace.readPaths.push(input.path);
if (name === 'write' && typeof input?.path === 'string') trace.writePaths.push(input.path);
if (name === 'list' && typeof input?.path === 'string') trace.listPaths.push(input.path);
}
const tools = {
bash: tool({
description:
'Run a bash command in the workspace root. Use this to invoke skill scripts (e.g. `node .claude/skills/impeccable/scripts/load-context.mjs`).',
inputSchema: z.object({
command: z.string().describe('The bash command to execute.'),
}),
execute: async ({ command }) => {
record('bash', { command });
const res = await execBash(workspace, command);
const head = `exit=${res.exitCode}`;
const body = (res.stdout ? `stdout:\n${res.stdout}` : '') + (res.stderr ? `\nstderr:\n${res.stderr}` : '');
return `${head}\n${body}${res.truncated ? '\n[output truncated]' : ''}`;
},
}),
read: tool({
description: 'Read a file from the workspace. Path must be workspace-relative.',
inputSchema: z.object({
path: z.string().describe('Workspace-relative file path.'),
}),
execute: async ({ path: p }) => {
record('read', { path: p });
const resolved = safeResolve(workspace, p);
if (typeof resolved !== 'string') return `Error: ${resolved.error}`;
if (!fs.existsSync(resolved)) return `File not found: ${p}`;
const stat = fs.statSync(resolved);
if (stat.isDirectory()) return `Path is a directory: ${p}. Use list instead.`;
return fs.readFileSync(resolved, 'utf8');
},
}),
write: tool({
description: 'Write or overwrite a file in the workspace. Creates parent directories as needed.',
inputSchema: z.object({
path: z.string().describe('Workspace-relative file path.'),
contents: z.string().describe('Full file contents.'),
}),
execute: async ({ path: p, contents }) => {
record('write', { path: p, contents });
const resolved = safeResolve(workspace, p);
if (typeof resolved !== 'string') return `Error: ${resolved.error}`;
fs.mkdirSync(path.dirname(resolved), { recursive: true });
fs.writeFileSync(resolved, contents);
return `Wrote ${Buffer.byteLength(contents, 'utf8')} bytes to ${p}`;
},
}),
list: tool({
description: 'List a workspace directory. Defaults to the workspace root.',
inputSchema: z.object({
path: z.string().default('.').describe('Workspace-relative directory path.'),
}),
execute: async ({ path: p }) => {
record('list', { path: p });
const resolved = safeResolve(workspace, p);
if (typeof resolved !== 'string') return `Error: ${resolved.error}`;
if (!fs.existsSync(resolved)) return `Not found: ${p}`;
const stat = fs.statSync(resolved);
if (!stat.isDirectory()) return `Not a directory: ${p}`;
const entries = fs.readdirSync(resolved).map((name) => {
const st = fs.statSync(path.join(resolved, name));
return st.isDirectory() ? `${name}/` : name;
});
return entries.length ? entries.join('\n') : '(empty)';
},
}),
};
return { tools, trace };
}
/**
* Run one scenario turn against a model.
*
* `priorMessages` lets multi-turn scenarios chain context from a previous
* call (append `result.response.messages` between turns).
*/
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8 }) {
const { tools, trace } = makeTools(workspace);
const messages = [
...priorMessages,
{ role: 'user', content: userPrompt },
];
let result;
try {
result = await generateText({
model,
system: SKILL_BODY,
messages,
tools,
stopWhen: [stepCountIs(maxSteps)],
});
} catch (err) {
return { trace, error: String(err), text: '', responseMessages: messages, finishReason: 'error' };
}
const responseMessages = [...messages, ...(result.response?.messages ?? [])];
return {
trace,
text: result.text ?? '',
finishReason: result.finishReason,
usage: result.usage,
responseMessages,
};
}
/**
* Heuristic helpers — keep the assertion intent declarative in the test file.
*/
export function bashCommandsMatching(trace, substring) {
return trace.bashCommands.filter((cmd) => cmd.includes(substring));
}
export function readsMatching(trace, substring) {
return trace.readPaths.filter((p) => p.toLowerCase().includes(substring.toLowerCase()));
}
/**
* True if the agent loaded a file by Read OR by a bash `cat` (some models
* stream multiple files via bash to save tool calls).
*/
export function fileLoaded(trace, filename) {
return readsMatching(trace, filename).length > 0 || bashCommandsMatching(trace, filename).length > 0;
}
export function summarizeTrace(trace) {
return {
totalCalls: trace.toolCalls.length,
byName: trace.toolCalls.reduce((acc, c) => ((acc[c.name] = (acc[c.name] ?? 0) + 1), acc), {}),
bashCommands: trace.bashCommands,
readPaths: trace.readPaths,
writePaths: trace.writePaths,
};
}
+91
View File
@@ -0,0 +1,91 @@
/**
* Multi-provider model factory for the skill-behavior test harness.
*
* The user explicitly asked for the cheapest tier of each major provider to
* keep CI cost in the cents-per-run range while still exercising real LLM
* decision-making against the skill body.
*
* Anthropic and OpenAI use the Vercel AI SDK providers. Google uses
* @ai-sdk/google for the same reason — uniform tool-use semantics across all
* three keeps the harness tiny.
*
* .env is loaded from the repo root (copied from impeccable-evals). Tests
* skip cleanly when the matching key is unset rather than failing CI.
*/
import { anthropic } from '@ai-sdk/anthropic';
import { google } from '@ai-sdk/google';
import { openai } from '@ai-sdk/openai';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(__dirname, '..', '..');
function loadEnv() {
const envPath = path.join(REPO_ROOT, '.env');
if (!fs.existsSync(envPath)) return;
const text = fs.readFileSync(envPath, 'utf8');
for (const line of text.split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eq = trimmed.indexOf('=');
if (eq === -1) continue;
const key = trimmed.slice(0, eq).trim();
let value = trimmed.slice(eq + 1).trim();
if (value.startsWith('"') && value.endsWith('"')) value = value.slice(1, -1);
if (value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
if (!process.env[key]) process.env[key] = value;
}
}
loadEnv();
export const PROVIDERS = {
anthropic: { envKey: 'ANTHROPIC_API_KEY', label: 'Anthropic' },
openai: { envKey: 'OPENAI_API_KEY', label: 'OpenAI' },
google: { envKey: 'GOOGLE_CLOUD_API_KEY', label: 'Google' },
};
export function detectProvider(modelId) {
if (modelId.startsWith('claude-')) return 'anthropic';
if (modelId.startsWith('gpt-')) return 'openai';
if (modelId.startsWith('gemini-')) return 'google';
throw new Error(`Unsupported model id: "${modelId}"`);
}
export function hasKey(provider) {
const meta = PROVIDERS[provider];
if (!meta) return false;
return Boolean(process.env[meta.envKey]);
}
export function getModel(modelId) {
const provider = detectProvider(modelId);
if (provider === 'anthropic') return anthropic(modelId);
if (provider === 'openai') return openai(modelId);
if (provider === 'google') {
// The @ai-sdk/google provider reads GOOGLE_GENERATIVE_AI_API_KEY by
// default; the evals .env stores the same value under
// GOOGLE_CLOUD_API_KEY. Mirror it so the SDK picks it up automatically.
if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY && process.env.GOOGLE_CLOUD_API_KEY) {
process.env.GOOGLE_GENERATIVE_AI_API_KEY = process.env.GOOGLE_CLOUD_API_KEY;
}
return google(modelId);
}
throw new Error(`Unsupported provider: ${provider}`);
}
/**
* Default model lineup. Cheapest tier per provider — the test is about
* routing/loading behavior, not design output quality, so cheap is fine.
* Override with IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-foo,gpt-bar.
*/
export const DEFAULT_MODELS = ['claude-haiku-4-5', 'gpt-5.4-mini', 'gemini-3.1-flash-lite'];
export function resolveModelList() {
const override = process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS;
if (override && override.trim()) {
return override.split(',').map((s) => s.trim()).filter(Boolean);
}
return DEFAULT_MODELS;
}
+254
View File
@@ -0,0 +1,254 @@
/**
* Skill-behavior scenarios — verify how the agent loads PRODUCT.md / DESIGN.md
* across a controlled matrix of starting states.
*
* Refactors that touch the Setup section of SKILL.md should keep these
* assertions green. If you change Setup intentionally and the assertions
* flip, that's the test catching the regression you wanted to catch.
*
* Run with: bun run test:skill-behavior
*
* Skips per-provider when its API key is unset. The default model lineup is
* the cheapest tier of each major provider so a full sweep costs a few cents.
*/
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import {
prepareWorkspace,
cleanupWorkspace,
runTurn,
bashCommandsMatching,
readsMatching,
fileLoaded,
summarizeTrace,
} from './harness.mjs';
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
import { PRODUCT_MD_SAMPLE, PRODUCT_MD_SAMPLE_NO_REGISTER, DESIGN_MD_SAMPLE } from './fixtures.mjs';
const CRAFT_PROMPT = '/impeccable craft a landing page for the project in this workspace';
const PRIMER_PROMPT =
'Take a quick look at the project. What register is this? Run the impeccable context loader once if you need to.';
const VERBOSE = process.env.IMPECCABLE_SKILL_BEHAVIOR_VERBOSE === '1';
function logTrace(label, scenario, model, trace, extras = {}) {
if (!VERBOSE) return;
const summary = summarizeTrace(trace);
console.error(
`\n[${label}] ${scenario} (${model})\n${JSON.stringify({ ...summary, ...extras }, null, 2)}\n`,
);
}
for (const modelId of resolveModelList()) {
const provider = detectProvider(modelId);
const keyPresent = hasKey(provider);
describe(`skill behavior :: ${modelId}`, () => {
if (!keyPresent) {
it(`skipped — ${PROVIDERS[provider].envKey} is unset`, { skip: true }, () => {});
return;
}
const model = getModel(modelId);
it('scenario 1: no PRODUCT.md / DESIGN.md', async () => {
const workspace = prepareWorkspace({ files: {} });
try {
const { trace, text } = await runTurn({
workspace,
model,
userPrompt: CRAFT_PROMPT,
maxSteps: 6,
});
logTrace('S1', 'no-context', modelId, trace, { textSample: text.slice(0, 400) });
// Agent runs context.mjs, sees NO_PRODUCT_MD directive, loads
// teach.md and follows it. Accept either Read or bash `cat` for
// the teach.md load — different models pick different tools.
const loadCalls = bashCommandsMatching(trace, 'context.mjs');
assert.ok(
loadCalls.length >= 1,
`expected agent to run context.mjs at least once; got ${loadCalls.length}.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
const teachLoaded =
readsMatching(trace, 'teach.md').length > 0 ||
bashCommandsMatching(trace, 'teach.md').length > 0;
assert.ok(
teachLoaded,
`expected agent to load teach.md (via Read or bash cat) after context.mjs reported NO_PRODUCT_MD.\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
// We do NOT want it to silently barrel into design work.
const wroteHtml = trace.writePaths.some((p) => /\.(html?|css|svelte|jsx?|tsx?)$/i.test(p));
assert.equal(
wroteHtml,
false,
`agent should not write implementation files before resolving missing PRODUCT.md.\n` +
`wrote: ${trace.writePaths.join(', ')}`,
);
} finally {
cleanupWorkspace(workspace);
}
});
it('scenario 2: PRODUCT.md only', async () => {
const workspace = prepareWorkspace({
files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE },
});
try {
const { trace, text } = await runTurn({
workspace,
model,
userPrompt: CRAFT_PROMPT,
maxSteps: 6,
});
logTrace('S2', 'product-only', modelId, trace, { textSample: text.slice(0, 400) });
const loadCalls = bashCommandsMatching(trace, 'context.mjs');
assert.ok(
loadCalls.length >= 1 && loadCalls.length <= 3,
`expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` +
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
);
// Fixture sets `register: brand`. Step 3 of Setup says load the
// matching register reference. Accept Read or bash cat.
assert.ok(
fileLoaded(trace, 'brand.md'),
`agent should load brand.md (PRODUCT.md register is brand).\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
} finally {
cleanupWorkspace(workspace);
}
});
it('scenario 3: PRODUCT.md + DESIGN.md', async () => {
const workspace = prepareWorkspace({
files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE, 'DESIGN.md': DESIGN_MD_SAMPLE },
});
try {
const { trace, text } = await runTurn({
workspace,
model,
userPrompt: CRAFT_PROMPT,
maxSteps: 6,
});
logTrace('S3', 'product-and-design', modelId, trace, { textSample: text.slice(0, 400) });
const loadCalls = bashCommandsMatching(trace, 'context.mjs');
assert.ok(
loadCalls.length >= 1 && loadCalls.length <= 3,
`expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` +
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
);
// Register reference: PRODUCT.md fixture is brand, so brand.md
// should be loaded per Setup step 3.
assert.ok(
fileLoaded(trace, 'brand.md'),
`agent should load brand.md (PRODUCT.md register is brand).\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
// The skill tells the agent to also familiarize with the existing
// design system. DESIGN.md is bundled in context.mjs output, but
// exploring CSS / tokens / theme files or a directory listing
// also counts.
const designSignal =
readsMatching(trace, 'design.md').length > 0 ||
trace.readPaths.some((p) => /\.(css|scss|less|ts|tsx|js|jsx|json|svelte|astro)$/i.test(p)) ||
trace.listPaths.length > 0;
assert.ok(
designSignal,
`agent should consult the design system (DESIGN.md, CSS/tokens, or list project files).\n` +
`readPaths: ${JSON.stringify(trace.readPaths)}, listPaths: ${JSON.stringify(trace.listPaths)}`,
);
} finally {
cleanupWorkspace(workspace);
}
});
it('scenario 4: context already loaded in prior turn', async () => {
const workspace = prepareWorkspace({
files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE, 'DESIGN.md': DESIGN_MD_SAMPLE },
});
try {
// Turn 1: prime the conversation so context.mjs gets run and its
// output enters the message history.
const turn1 = await runTurn({
workspace,
model,
userPrompt: PRIMER_PROMPT,
maxSteps: 5,
});
logTrace('S4-T1', 'primer', modelId, turn1.trace, { textSample: turn1.text.slice(0, 200) });
const turn1Loads = bashCommandsMatching(turn1.trace, 'context.mjs');
assert.ok(
turn1Loads.length >= 1,
`primer turn should have run context.mjs. bash: ${JSON.stringify(turn1.trace.bashCommands, null, 2)}`,
);
// Turn 2: the real ask. The skill says "skip if you've already
// loaded it". Verify the agent honors that.
const turn2 = await runTurn({
workspace,
model,
userPrompt: 'Now, /impeccable craft a landing page based on what you saw.',
priorMessages: turn1.responseMessages,
maxSteps: 5,
});
logTrace('S4-T2', 'follow-up', modelId, turn2.trace, { textSample: turn2.text.slice(0, 400) });
const turn2Loads = bashCommandsMatching(turn2.trace, 'context.mjs');
assert.equal(
turn2Loads.length,
0,
`agent re-ran context.mjs on turn 2 despite it being in prior conversation. ` +
`bashCommands: ${JSON.stringify(turn2.trace.bashCommands, null, 2)}`,
);
// Register reference must land somewhere across the two turns —
// craft work without brand.md (for a brand-register project) means
// Setup step 3 was skipped.
const brandLoadedAcrossTurns =
fileLoaded(turn1.trace, 'brand.md') || fileLoaded(turn2.trace, 'brand.md');
assert.ok(
brandLoadedAcrossTurns,
`agent should load brand.md across turn 1 or turn 2 (project is brand register).\n` +
`turn 1 readPaths: ${JSON.stringify(turn1.trace.readPaths)}, bash: ${JSON.stringify(turn1.trace.bashCommands)}\n` +
`turn 2 readPaths: ${JSON.stringify(turn2.trace.readPaths)}, bash: ${JSON.stringify(turn2.trace.bashCommands)}`,
);
} finally {
cleanupWorkspace(workspace);
}
});
it('scenario 5: PRODUCT.md WITHOUT register field (cascade via task cue)', async () => {
// PRODUCT.md has no `## Register` section, so context.mjs cannot
// detect the register and emits a generic "pick by cascade"
// directive. The agent must infer brand from the user's task cue
// ("landing page") per SKILL.md's priority list (1) task cue,
// (2) surface in focus, (3) register field.
const workspace = prepareWorkspace({
files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_NO_REGISTER },
});
try {
const { trace, text } = await runTurn({
workspace,
model,
userPrompt: CRAFT_PROMPT,
maxSteps: 6,
});
logTrace('S5', 'no-register-field', modelId, trace, { textSample: text.slice(0, 400) });
const loadCalls = bashCommandsMatching(trace, 'context.mjs');
assert.ok(
loadCalls.length >= 1,
`expected context.mjs invocation; got ${loadCalls.length}.\n` +
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
);
// Task cue is "landing page" → brand register → brand.md should load.
assert.ok(
fileLoaded(trace, 'brand.md'),
`agent should load brand.md via task-cue cascade (no register field, "landing page" cue).\n` +
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
);
} finally {
cleanupWorkspace(workspace);
}
});
});
}