diff --git a/bun.lock b/bun.lock index 6f89438fc..7bf1e6d37 100644 --- a/bun.lock +++ b/bun.lock @@ -13,6 +13,7 @@ }, "devDependencies": { "@ai-sdk/anthropic": "^3.0.71", + "@ai-sdk/google": "^3.0.75", "@ai-sdk/openai": "^3.0.53", "@anthropic-ai/claude-agent-sdk": "^0.2.119", "@anthropic-ai/sdk": "^0.91.1", @@ -37,6 +38,8 @@ "@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.104", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ZKX5n74io8VIRlhIMSLWVlvT3sXC8Z7cZ9GHuWBWZDVi96+62AIsWuLGvMfcBA1STYuSoDrp6rIziZmvrTq0TA=="], + "@ai-sdk/google": ["@ai-sdk/google@3.0.75", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-XAm31ftiOrzlb8NjDzT7kw0xw+4lmgFdGFn1QKM73nXFFKyN1kWLESBV75UGNfjXP8X1YJ0YydnMVqO0jaPghw=="], + "@ai-sdk/openai": ["@ai-sdk/openai@3.0.53", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Wld+Rbc05KaUn08uBt06eEuwcgalcIFtIl32Yp+GxuZXUQwOb6YeAuq+C6da4ch6BurFoqEaLemJVwjBb7x+PQ=="], "@ai-sdk/provider": ["@ai-sdk/provider@3.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ=="], @@ -1285,6 +1288,10 @@ "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], + "@ai-sdk/google/@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="], + + "@ai-sdk/google/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.27", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw=="], + "@anthropic-ai/claude-agent-sdk/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.81.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-D4K5PvEV6wPiRtVlVsJHIUhHAmOZ6IT/I9rKlTf84gR7GyyAurPJK7z9BOf/AZqC5d1DhYQGJNKRmV+q8dGhgw=="], "@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], @@ -1329,6 +1336,8 @@ "yauzl/buffer-crc32": ["buffer-crc32@0.2.13", "", {}, "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="], + "@ai-sdk/google/@ai-sdk/provider-utils/eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="], + "@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], "@isaacs/cliui/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], diff --git a/package.json b/package.json index 9067ed3c3..d41537fb1 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "deploy": "bun run build && wrangler pages deploy build/", "test": "bun test tests/build.test.js tests/detect-antipatterns.test.js tests/windows-path-fix.test.js tests/lib/detector-bundle.test.js tests/lib/provider-blocks.test.js tests/lib/transformers/provider-blocks.test.js && node --test tests/critique-storage.test.mjs && node --test tests/detect-antipatterns-fixtures.test.mjs && node --test tests/detect-antipatterns-browser.test.mjs && node --test tests/cleanup-deprecated.test.mjs && node --test tests/impeccable-paths.test.mjs && node --test tests/live-wrap.test.mjs && node --test tests/live-reference.test.mjs && node --test tests/live-accept.test.mjs && node --test tests/live-inject.test.mjs && node --test tests/live-poll.test.mjs && node --test tests/live-server.test.mjs && node --test tests/live-browser-regression.test.mjs && node --test tests/live-session-store.test.mjs && node --test tests/live-browser-session.test.mjs && node --test tests/live-browser-source.test.mjs && node --test tests/live-completion.test.mjs && node --test tests/live-recovery-commands.test.mjs && node --test tests/framework-fixtures.test.mjs", "test:live-e2e": "node --test --test-timeout=600000 tests/live-e2e.test.mjs", + "test:skill-behavior": "node --test --test-timeout=300000 tests/skill-behavior/scenarios.test.mjs", "bench:detector": "node scripts/benchmark-detector.mjs", "bench:detector:browser": "node scripts/benchmark-detector.mjs --browser", "audit": "bun audit --audit-level=moderate", @@ -74,15 +75,16 @@ "puppeteer": "^24.42.0" }, "devDependencies": { - "astro": "^6.2.1", - "@paper-design/shaders": "^0.0.76", "@ai-sdk/anthropic": "^3.0.71", + "@ai-sdk/google": "^3.0.75", "@ai-sdk/openai": "^3.0.53", "@anthropic-ai/claude-agent-sdk": "^0.2.119", "@anthropic-ai/sdk": "^0.91.1", "@google/genai": "^1.50.1", + "@paper-design/shaders": "^0.0.76", "ai": "^6.0.168", "archiver": "^7.0.1", + "astro": "^6.2.1", "modern-screenshot": "^4.7.0", "motion": "^12.38.0", "playwright": "^1.59.1", diff --git a/skill/SKILL.md b/skill/SKILL.md index ef2fb5554..bdd560733 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -5,53 +5,19 @@ argument-hint: "[{{command_hint}}] [target]" user-invocable: true allowed-tools: - Bash(npx impeccable *) -license: Apache 2.0. Based on Anthropic's frontend-design skill. See NOTICE.md for attribution. +license: Apache 2.0 --- Designs and iterates production-grade frontend interfaces. Real working code, committed design choices, exceptional craft. ## Setup -Before any design work or file edits: +You MUST do these steps before proceeding: -1. Load context (PRODUCT.md / DESIGN.md) via the loader script. -2. Identify the register and load the matching register reference (brand.md or product.md). -3. **If the user invoked a sub-command (e.g. `craft`, `shape`, `audit`), load its reference file too.** This is non-negotiable: `craft` without `craft.md` loaded means you'll skip the shape-and-confirm step the user expects. - -Skipping these produces generic output that ignores the project. - -### 1. Context gathering - -Two files, case-insensitive. The loader looks at the project root by default and falls back to `.agents/context/` and `docs/` if the root is clean. Override with `IMPECCABLE_CONTEXT_DIR=path/to/dir` (absolute or relative to cwd). - -- **PRODUCT.md**: required. Users, brand, tone, anti-references, strategic principles. -- **DESIGN.md**: optional, strongly recommended. Colors, typography, elevation, components. - -Load both in one call: - -```bash -node {{scripts_path}}/load-context.mjs -``` - -Consume the full JSON output. Never pipe through `head`, `tail`, `grep`, or `jq`. The output's `contextDir` field tells you where the files were resolved from. - -If the output is already in this session's conversation history, don't re-run. Exceptions requiring a fresh load: you just ran `{{command_prefix}}impeccable teach` or `{{command_prefix}}impeccable document` (they rewrite the files), or the user manually edited one. - -`{{command_prefix}}impeccable live` already warms context via `live.mjs`. If you've run `live.mjs`, don't also run `load-context.mjs` this session. - -If PRODUCT.md is missing, empty, or placeholder (`[TODO]` markers, <200 chars): run `{{command_prefix}}impeccable teach`, then resume the user's original task with the fresh context. If the original task was `{{command_prefix}}impeccable craft`, resume into `{{command_prefix}}impeccable shape` before any implementation work. - -If DESIGN.md is missing: nudge once per session (*"Run `{{command_prefix}}impeccable document` for more on-brand output"*), then proceed. - -### 2. Register - -Every design task is **brand** (marketing, landing, campaign, long-form content, portfolio: design IS the product) or **product** (app UI, admin, dashboard, tool: design SERVES the product). - -Identify before designing. Priority: (1) cue in the task itself ("landing page" vs "dashboard"); (2) the surface in focus (the page, file, or route being worked on); (3) `register` field in PRODUCT.md. First match wins. - -If PRODUCT.md lacks the `register` field (legacy), infer it once from its "Users" and "Product Purpose" sections, then cache the inferred value for the session. Suggest the user run `{{command_prefix}}impeccable teach` to add the field explicitly. - -Load the matching reference: [reference/brand.md](reference/brand.md) or [reference/product.md](reference/product.md). The shared design laws below apply to both. +1. Run `node {{scripts_path}}/context.mjs` once per session. If you've already seen its output in this conversation, do not re-run it. The script either prints the project's PRODUCT.md (and DESIGN.md when present) as a markdown block, or tells you it's missing. Follow whatever it prints. **If it reports `NO_PRODUCT_MD`, stop and follow `reference/teach.md` before doing anything else.** +2. Familiarize yourself with any existing design system, conventions, and components in the code. Don't reinvent the wheel; use what's there when it works, branch out when the UX wins. +3. Read the matching register reference. **This is non-optional; skipping it produces generic output.** If the project is marketing, a landing page, a campaign, long-form content, or a portfolio (design IS the product), read `reference/brand.md`. If it is app UI, admin, a dashboard, or a tool (design SERVES the product), read `reference/product.md`. Pick by first match: (1) task cue ("landing page" vs "dashboard"); (2) surface in focus (the page, file, or route being worked on); (3) `register` field in PRODUCT.md. +4. If the user invoked a sub-command (`craft`, `shape`, `audit`, ...), load its reference too. ## Shared design laws @@ -60,14 +26,14 @@ Apply to every design, both registers. Match implementation complexity to the ae ### Color - Use OKLCH. Reduce chroma as lightness approaches 0 or 100; high chroma at extremes looks garish. -- **The cream / sand / beige body bg is the saturated AI default of 2026.** The whole warm-neutral band — OKLCH L 0.84-0.97, C < 0.06, hue 40-100 — reads as cream/sand/paper/parchment regardless of what you call it. Token names like `--paper`, `--cream`, `--sand`, `--bone`, `--flour`, `--linen`, `--parchment`, `--wheat`, `--biscuit`, `--ivory` are tells in themselves. If the brief is "warm, traditional, family-coastal-Italian" or "magazine-warm" or "editorial-restraint", DO NOT translate that into a near-white warm-tinted bg — that's the AI move. Pick: (a) a saturated brand color as the body (terracotta, oxblood, deep ochre, near-black), (b) a true off-white at chroma 0 (or chroma toward the brand's own hue, not toward warmth-by-default), or (c) a darker mid-tone tinted neutral that's clearly the brand's own. "Warmth" in the brand is carried by accent + typography + imagery, not by body bg. +- **The cream / sand / beige body bg is the saturated AI default of 2026.** The whole warm-neutral band (OKLCH L 0.84-0.97, C < 0.06, hue 40-100) reads as cream/sand/paper/parchment regardless of what you call it. Token names like `--paper`, `--cream`, `--sand`, `--bone`, `--flour`, `--linen`, `--parchment`, `--wheat`, `--biscuit`, `--ivory` are tells in themselves. If the brief is "warm, traditional, family-coastal-Italian" or "magazine-warm" or "editorial-restraint", DO NOT translate that into a near-white warm-tinted bg; that's the AI move. Pick: (a) a saturated brand color as the body (terracotta, oxblood, deep ochre, near-black), (b) a true off-white at chroma 0 (or chroma toward the brand's own hue, not toward warmth-by-default), or (c) a darker mid-tone tinted neutral that's clearly the brand's own. "Warmth" in the brand is carried by accent + typography + imagery, not by body bg. - Pick a **color strategy** before picking colors. Four steps on the commitment axis: - **Restrained**: tinted neutrals + one accent ≤10%. Product default; brand minimalism. - **Committed**: one saturated color carries 30–60% of the surface. Brand default for identity-driven pages. - **Full palette**: 3–4 named roles, each used deliberately. Brand campaigns; product data viz. - **Drenched**: the surface IS the color. Brand heroes, campaign pages. - The "one accent ≤10%" rule is Restrained only. Committed / Full palette / Drenched exceed it on purpose. Don't collapse every design to Restrained by reflex. -- **Verify contrast.** Body text must hit ≥4.5:1 against its background; large text (≥18px or bold ≥14px) needs ≥3:1. The most common failure: muted gray body text on a tinted near-white. If the contrast is even close, bump the body color toward the ink end of the ramp — light gray "for elegance" is the single biggest reason AI designs feel hard to read. +- **Verify contrast.** Body text must hit ≥4.5:1 against its background; large text (≥18px or bold ≥14px) needs ≥3:1. The most common failure: muted gray body text on a tinted near-white. If the contrast is even close, bump the body color toward the ink end of the ramp; light gray "for elegance" is the single biggest reason AI designs feel hard to read. ### Theme @@ -84,7 +50,7 @@ Before choosing, write one sentence of physical scene: who uses this, where, und - Cap font-family count at 3 (display + body + optional mono). More than 3 reads as indecision, not richness. One well-tuned family with weight contrast usually beats three competing typefaces. - No all-caps body copy. Reserve uppercase for short labels (≤4 words), section eyebrows (used sparingly per the Absolute bans), and badges. Sentences in ALL CAPS are unreadable at body sizes. - Hero / display heading ceiling: clamp() max ≤ 6rem (~96px). Above that the page is shouting, not designing. -- Display heading letter-spacing floor: ≥ -0.04em. Anything tighter and letters touch — cramped, not "designed". +- Display heading letter-spacing floor: ≥ -0.04em. Anything tighter and letters touch; cramped, not "designed". Two hard typographic ceilings you currently miss: @@ -104,7 +70,7 @@ Two hard typographic ceilings you currently miss: - Ease out with exponential curves (ease-out-quart / quint / expo). No bounce, no elastic. -**Gemini-specific defect — hard ban.** Never animate `` elements on hover. This includes any `transform` on `:hover` of an image, AND `.group:hover .group-hover\:scale` / `.group:hover .group-hover\:rotate` / `.group:hover .group-hover\:translate` patterns from Tailwind that animate a child image via a parent hover. This is your single most common motion tell — it adds no information (the image isn't an action target) and reads as "AI animated this because it could". If a card needs hover feedback, animate the card's background, border, or shadow. Never the image, never via the image's parent. +**Gemini-specific defect: hard ban.** Never animate `` elements on hover. This includes any `transform` on `:hover` of an image, AND `.group:hover .group-hover\:scale` / `.group:hover .group-hover\:rotate` / `.group:hover .group-hover\:translate` patterns from Tailwind that animate a child image via a parent hover. This is your single most common motion tell; it adds no information (the image isn't an action target) and reads as "AI animated this because it could". If a card needs hover feedback, animate the card's background, border, or shadow. Never the image, never via the image's parent. ### Absolute bans @@ -117,15 +83,15 @@ Match-and-refuse. If you're about to write any of these, rewrite the element wit - **The hero-metric template.** Big number, small label, supporting stats, gradient accent. SaaS cliché. - **Identical card grids.** Same-sized cards with icon + heading + text, repeated endlessly. - **Modal as first thought.** Modals are usually laziness. Exhaust inline / progressive alternatives first. -- **Tiny uppercase tracked eyebrow above every section.** The 2023-era kicker (small all-caps text with wide tracking, "ABOUT" "PROCESS" "PRICING" above each heading) is now the saturated AI scaffold — it appears on 55-95% of generations regardless of brief, which is the definition of a tell. One named kicker as a deliberate brand system is voice; an eyebrow on every section is AI grammar. Choose a different cadence. +- **Tiny uppercase tracked eyebrow above every section.** The 2023-era kicker (small all-caps text with wide tracking, "ABOUT" "PROCESS" "PRICING" above each heading) is now the saturated AI scaffold; it appears on 55-95% of generations regardless of brief, which is the definition of a tell. One named kicker as a deliberate brand system is voice; an eyebrow on every section is AI grammar. Choose a different cadence. - **Numbered section markers (01 / 02 / 03).** Section labels as numbered display digits is the same trope, one tier deeper. If you find yourself reaching for them, you're scaffolding by reflex. - **Text that overflows its container.** Long heading words plus large clamp scales plus narrow grids cause headline overflow on tablet/mobile. Test the heading copy at every breakpoint; if it overflows, reduce the clamp max or rewrite the copy. The viewport is part of the design. -**Codex-specific defects** (your most-frequent giveaways — refuse-and-rewrite): +**Codex-specific defects** (your most-frequent giveaways; refuse-and-rewrite): -- **`border: 1px solid X` + `box-shadow: 0 Npx Mpx ...` with M ≥ 16px** on the same element. The "ghost-card" pattern — 1px border plus soft wide drop shadow on buttons and cards. Don't pair them. Pick one (a single solid border at the brand color, OR a defined shadow at no more than 8px blur), never both as decoration. -- **`border-radius: 32px+` on cards / sections / inputs.** You over-round. Cards top out at 12–16px; full-pill is fine for tags/buttons. Picking 24/28/32/40px on a card is the codex tell — no brand wants "insanely rounded". +- **`border: 1px solid X` + `box-shadow: 0 Npx Mpx ...` with M ≥ 16px** on the same element. The "ghost-card" pattern: 1px border plus soft wide drop shadow on buttons and cards. Don't pair them. Pick one (a single solid border at the brand color, OR a defined shadow at no more than 8px blur), never both as decoration. +- **`border-radius: 32px+` on cards / sections / inputs.** You over-round. Cards top out at 12–16px; full-pill is fine for tags/buttons. Picking 24/28/32/40px on a card is the codex tell; no brand wants "insanely rounded". - **Hand-drawn / sketchy SVG illustrations.** Class names like `loose-sketch`, `*-sketch`, `doodle`, `wavy`; `feTurbulence` / `feDisplacementMap` "paper grain" filters; 5-to-30 path crude scenes meant to depict a tangible subject (an otter, a table-and-fork, an album cover). All of these read as amateurish, not whimsical. If you can't render the scene with real assets, ship no illustration. Don't attempt sketchy SVG as a fallback. - **`repeating-linear-gradient(...)` stripe backgrounds.** Diagonal stripes in `body:before` or section backgrounds are pure codex decoration. Don't. - **"X theater" / "actually X" / "not just X, it's Y" copy.** "Productivity theater", "engagement theater", "growth theater": instant AI slop. Choose a specific noun, not a meta-criticism phrase. diff --git a/skill/reference/animate.md b/skill/reference/animate.md index 176721e2c..b334750f8 100644 --- a/skill/reference/animate.md +++ b/skill/reference/animate.md @@ -6,7 +6,7 @@ Add motion that conveys state, gives feedback, and clarifies hierarchy. Cut moti ## Register -Brand: motion is part of the voice; one well-rehearsed entrance beats scattered micro-interactions. The saturated AI default is fade-and-rise reveals on every scrolled section — that's a tell, not a choreography. Reserve scroll-triggered motion for moments that earn it. +Brand: motion is part of the voice; one well-rehearsed entrance beats scattered micro-interactions. The saturated AI default is fade-and-rise reveals on every scrolled section; that's a tell, not a choreography. Reserve scroll-triggered motion for moments that earn it. Product: 150–250 ms on most transitions. Motion conveys state: feedback, reveal, loading, transitions between views. No page-load choreography; users are in a task and won't wait for it. @@ -51,7 +51,7 @@ Add motion systematically across these categories: ### Entrance Animations - **Hero section**: Dramatic entrance for primary content (scale, parallax, or creative effects) - **Modal/drawer entry**: Smooth slide + fade, backdrop fade, focus management -- **List rhythm**: Sibling stagger is legitimate for cards-in-a-grid or list-items-appearing — cap total stagger time per [motion-design.md](motion-design.md). Whole-section fade-on-scroll is not a list and is not legitimate. +- **List rhythm**: Sibling stagger is legitimate for cards-in-a-grid or list-items-appearing; cap total stagger time per [motion-design.md](motion-design.md). Whole-section fade-on-scroll is not a list and is not legitimate. ### Micro-interactions - **Button feedback**: diff --git a/skill/reference/brand.md b/skill/reference/brand.md index 2d5f79397..b723fce51 100644 --- a/skill/reference/brand.md +++ b/skill/reference/brand.md @@ -43,7 +43,7 @@ The reflex-reject lists apply to **new design choices**. When the existing brand ### Pairing and voice -Distinctive + refined is the goal. The specific shape depends on the brand, not on the brand's category. A category — "restaurant", "dev tool", "magazine", "fintech" — is not a recipe; treating it as one is the first-order reflex SKILL.md warns against. +Distinctive + refined is the goal. The specific shape depends on the brand, not on the brand's category. A category ("restaurant", "dev tool", "magazine", "fintech") is not a recipe; treating it as one is the first-order reflex SKILL.md warns against. Two families minimum is the rule *only* when the voice needs it. A single well-chosen family with committed weight/size contrast is stronger than a timid display+body pair. diff --git a/skill/reference/codex.md b/skill/reference/codex.md index 93865a3c9..923bb899b 100644 --- a/skill/reference/codex.md +++ b/skill/reference/codex.md @@ -24,7 +24,7 @@ Before generating anything, run a brief direction conversation grounded in the s Ask **2-3 targeted questions** about visual lane, color strategy, atmosphere, and named anchor references. Don't enumerate generic menus; tie each question to the shape brief's answers. Example shape-grounded questions: - "Brief says 'specimen-page restraint.' Are we closer to a quiet typographic page or a wider editorial spread with hero imagery?" -- "Palette strategy from shape was 'Committed.' Which one color carries the surface — a brand-driven pick rather than a default warm-or-cool framing? (And no, the answer isn't a cream/sand body bg — that's the saturated AI default.)" +- "Palette strategy from shape was 'Committed.' Which one color carries the surface (a brand-driven pick rather than a default warm-or-cool framing)? (And no, the answer isn't a cream/sand body bg; that's the saturated AI default.)" **STOP and wait for answers.** These pin the palette before any pixel gets generated. Do not proceed to Step B until the user has responded. diff --git a/skill/reference/document.md b/skill/reference/document.md index 6a6ed1443..595590de5 100644 --- a/skill/reference/document.md +++ b/skill/reference/document.md @@ -328,12 +328,13 @@ Pull directly from the DESIGN.md you just wrote: Do not reword. The panel shows these as secondary collapsible context; the same voice that's in the Markdown carries through. -### Step 5: Confirm, refine, and refresh session cache +### Step 5: Confirm and refine 1. Show the user the full DESIGN.md you wrote. Briefly highlight the non-obvious creative choices (descriptive color names, atmosphere language, named rules). 2. Mention that `.impeccable/design.json` was also written alongside; the live panel will now render this project's actual button/input/nav primitives instead of generic approximations. 3. Offer to refine any section: "Want me to revise a section, add component patterns I missed, or adjust the atmosphere language?" -4. **Refresh the session cache.** Run `node {{scripts_path}}/load-context.mjs` one final time so the newly-written DESIGN.md lands in conversation. Subsequent commands in this session will use the fresh version automatically without re-reading. + +Your own write is the freshest source; subsequent commands in this session don't need a reload. ## Seed mode @@ -394,11 +395,12 @@ Per-section guidance in seed mode: Seed mode writes a minimal frontmatter with `name` and `description` only; no colors, typography, rounded, spacing, or components yet. Real tokens land on the next Scan-mode run. Skip the `.impeccable/design.json` sidecar in seed mode for the same reason: nothing to render. -### Step 4: Confirm and refresh session cache +### Step 4: Confirm 1. Show the seed DESIGN.md. Call out that it is a seed (the marker is the literal commitment). 2. Tell the user: "Re-run `/impeccable document` once you have some code. That pass will extract real tokens and generate the sidecar." -3. Run `node {{scripts_path}}/load-context.mjs` once so the seed lands in conversation for the rest of the session. + +Your own write is the freshest source; no reload needed. ## Style guidelines diff --git a/skill/reference/live.md b/skill/reference/live.md index 971dda22e..95a918f40 100644 --- a/skill/reference/live.md +++ b/skill/reference/live.md @@ -4,12 +4,16 @@ Interactive live variant mode: select elements in the browser, pick a design act A running dev server with hot module replacement (Vite, Next.js, Bun, etc.), OR a static HTML file open in the browser. + +Codex: run live helper commands, the app dev server, and any dependency-installing setup with `sandbox_permissions: "require_escalated"` from the start; live mode depends on localhost and package-manager network access that the sandbox blocks. + + ## The contract (read once) Execute in order. No step skipped, no step reordered. 1. `live.mjs`: boot. -2. Navigate to the URL that serves `pageFile` (infer from `package.json`, docs, terminal output, or an open tab). If you can't infer it confidently, tell the user once to open their dev/preview URL. Never use `serverPort` as that URL; it's the helper, not the app. +2. Open the app URL that serves `pageFile` (infer from `package.json`, docs, terminal output, or an open tab). Never use `serverPort`; it's the helper, not the app. **Cursor:** `browser_navigate` to that URL before polling; do not skip. **Other harnesses:** use the available browser tool; if the URL is uncertain, ask the user once. 3. Poll loop with the default long timeout (600000 ms). After every event or `--reply`, run `live-poll.mjs` again immediately. Never pass a short `--timeout=`. 4. On `generate`: read screenshot if present; load the action's reference; plan three distinct directions; write all variants in one edit; `--reply done`; poll again. 5. On `accept` / `discard`: the poll script runs `live-accept.mjs`, acknowledges the delivered event, and prints `_completionAck`. Plain accepts/discards are terminal immediately; carbonize accepts remain recoverable until you finish cleanup, run `live-complete.mjs --id EVENT_ID`, and only then poll again. @@ -30,7 +34,7 @@ Chat is overhead. No recap, no tutorial output, no pasting PRODUCT / DESIGN bodi node {{scripts_path}}/live.mjs ``` -Output JSON: `{ ok, serverPort, serverToken, pageFiles, hasProduct, product, productPath, hasDesign, design, designPath, migrated }`. `pageFiles` is the list of HTML entries the live script was injected into. Keep PRODUCT.md and DESIGN.md in mind for variant generation; **DESIGN.md wins on visual decisions; PRODUCT.md wins on strategic/voice decisions.** When DESIGN.md is missing, identity is **not** absent; extract it from CSS variables, computed styles, and sibling components on the page (see Step 4 Phase A). Identity preservation is the default; departure from existing identity requires an explicit trigger from PRODUCT.md anti-references or the user's freeform prompt. If `migrated: true`, the loader auto-renamed legacy `.impeccable.md` to `PRODUCT.md`; mention this once and suggest `/impeccable document` for the matching DESIGN.md. +Output JSON: `{ ok, serverPort, serverToken, pageFiles, hasProduct, product, productPath, hasDesign, design, designPath }`. `pageFiles` is the list of HTML entries the live script was injected into. Keep PRODUCT.md and DESIGN.md in mind for variant generation; **DESIGN.md wins on visual decisions; PRODUCT.md wins on strategic/voice decisions.** When DESIGN.md is missing, identity is **not** absent; extract it from CSS variables, computed styles, and sibling components on the page (see Step 4 Phase A). Identity preservation is the default; departure from existing identity requires an explicit trigger from PRODUCT.md anti-references or the user's freeform prompt. `serverPort` and `serverToken` belong to the small **Impeccable live helper** HTTP server (serves `/live.js`, SSE, and `/poll`). That port is **not** your dev server and is usually not the URL you open to view the app. The browser page is whatever origin serves one of the `pageFiles` entries (Vite / Next / Bun / tunnel / LAN hostname). diff --git a/skill/reference/teach.md b/skill/reference/teach.md index efe88cb73..6efc220f3 100644 --- a/skill/reference/teach.md +++ b/skill/reference/teach.md @@ -9,13 +9,7 @@ Every other impeccable command reads these files before doing any work. ## Step 1: Load current state -Run the shared loader first so you know what already exists: - -```bash -node {{scripts_path}}/load-context.mjs -``` - -The output tells you whether PRODUCT.md and/or DESIGN.md already exist. If `migrated: true`, legacy `.impeccable.md` was auto-renamed to `PRODUCT.md`. Mention this once to the user. +Check what already exists. PRODUCT.md and DESIGN.md live at the project root, or under `.agents/context/` or `docs/` (case-insensitive). Read whichever are present with your native file tool. Decision tree: - **Neither file exists (empty project or no context yet)**: do Steps 2-4 (write PRODUCT.md), then decide on DESIGN.md based on whether there's code to analyze. @@ -26,7 +20,7 @@ Decision tree: Never silently overwrite an existing file. Always confirm first. -If teach was invoked as a setup blocker by another command, such as `{{command_prefix}}impeccable craft landing page`, pause that command here. Complete teach, re-run the loader, then resume the original command with the freshly loaded context. For craft, resume into shape next; teach creates project context, but it is not a substitute for the task-specific shape interview and confirmed design brief. +If teach was invoked as a setup blocker by another command, such as `{{command_prefix}}impeccable craft landing page`, pause that command here. Complete teach, then resume the original command. Your own writes are the freshest source; no reload needed. For craft, resume into shape next; teach creates project context, but it is not a substitute for the task-specific shape interview and confirmed design brief. ## Step 2: Explore the codebase @@ -148,8 +142,6 @@ Summarize: - The 3-5 strategic principles from PRODUCT.md that will guide future work - If DESIGN.md is pending, remind the user how to generate it later -**Critical: re-run the loader to refresh session context.** After writing PRODUCT.md, run `node {{scripts_path}}/load-context.mjs` one final time and let its full JSON output land in conversation. This ensures subsequent commands in this session use the freshly-written PRODUCT.md, not a stale earlier version. - -If teach was invoked as a blocker by another impeccable command (e.g. the user ran `/impeccable polish` with no PRODUCT.md), resume that original task now with the fresh context. +If teach was invoked as a blocker by another impeccable command (e.g. the user ran `/impeccable polish` with no PRODUCT.md), resume that original task now. Your own writes are the freshest source; no reload needed. Optionally {{ask_instruction}} Ask whether they'd like a brief summary of PRODUCT.md appended to {{config_file}} for easier agent reference. If yes, append a short **Design Context** pointer section there. diff --git a/skill/scripts/context.mjs b/skill/scripts/context.mjs new file mode 100644 index 000000000..766c769ee --- /dev/null +++ b/skill/scripts/context.mjs @@ -0,0 +1,125 @@ +/** + * Context loader: prints PRODUCT.md (and DESIGN.md if present) as one + * markdown block on stdout, or exits with empty stdout when no PRODUCT.md + * is found anywhere. The skill keys off "empty stdout" to branch into the + * teach flow. + * + * Path resolution (first match wins): + * 1. cwd, if PRODUCT.md or DESIGN.md is there + * 2. .agents/context/ then docs/ + * 3. $IMPECCABLE_CONTEXT_DIR (absolute or cwd-relative) — power-user + * escape hatch, only consulted when defaults are empty + * 4. cwd as a "nothing found" default + * + * `resolveContextDir()` and `loadContext()` are also exported for the + * server-side scripts (live.mjs, live-server.mjs) that need the structured + * shape rather than the markdown block. + */ +import fs from 'node:fs'; +import path from 'node:path'; + +const PRODUCT_NAMES = ['PRODUCT.md', 'Product.md', 'product.md']; +const DESIGN_NAMES = ['DESIGN.md', 'Design.md', 'design.md']; +const FALLBACK_DIRS = ['.agents/context', 'docs']; + +export function resolveContextDir(cwd = process.cwd()) { + if (firstExisting(cwd, [...PRODUCT_NAMES, ...DESIGN_NAMES])) { + return cwd; + } + for (const rel of FALLBACK_DIRS) { + const candidate = path.resolve(cwd, rel); + if (firstExisting(candidate, [...PRODUCT_NAMES, ...DESIGN_NAMES])) { + return candidate; + } + } + const envDir = process.env.IMPECCABLE_CONTEXT_DIR; + if (envDir && envDir.trim()) { + const trimmed = envDir.trim(); + return path.isAbsolute(trimmed) ? trimmed : path.resolve(cwd, trimmed); + } + return cwd; +} + +export function loadContext(cwd = process.cwd()) { + const contextDir = resolveContextDir(cwd); + const productPath = firstExisting(contextDir, PRODUCT_NAMES); + const designPath = firstExisting(contextDir, DESIGN_NAMES); + const product = productPath ? safeRead(productPath) : null; + const design = designPath ? safeRead(designPath) : null; + return { + hasProduct: !!product, + product, + productPath: productPath ? path.relative(cwd, productPath) : null, + hasDesign: !!design, + design, + designPath: designPath ? path.relative(cwd, designPath) : null, + contextDir, + }; +} + +function firstExisting(dir, names) { + for (const name of names) { + const abs = path.join(dir, name); + if (fs.existsSync(abs)) return abs; + } + return null; +} + +function safeRead(p) { + try { + return fs.readFileSync(p, 'utf-8'); + } catch { + return null; + } +} + +/** + * Pull the register (`brand` or `product`) out of PRODUCT.md by looking + * for a `## Register` section and reading the first non-empty line that + * follows it. Returns null when the file is legacy / register-less. + */ +function extractRegister(product) { + if (!product) return null; + const lines = product.split('\n'); + for (let i = 0; i < lines.length; i++) { + if (/^##\s+Register\b/i.test(lines[i].trim())) { + for (let j = i + 1; j < lines.length; j++) { + const next = lines[j].trim(); + if (!next) continue; + const word = next.toLowerCase(); + if (word === 'brand' || word === 'product') return word; + return null; + } + } + } + return null; +} + +function cli() { + const ctx = loadContext(process.cwd()); + if (!ctx.hasProduct) { + // Direct stdout message instead of relying on empty output as a signal + // — cheap models miss the empty case more often than the explicit one. + process.stdout.write( + 'NO_PRODUCT_MD: This project has no PRODUCT.md yet. ' + + 'Stop the current task, load reference/teach.md, and follow its ' + + 'instructions to write PRODUCT.md before resuming.\n', + ); + process.exit(0); + } + const parts = [`# PRODUCT.md\n\n${ctx.product.trim()}`]; + if (ctx.hasDesign) { + parts.push(`# DESIGN.md\n\n${ctx.design.trim()}`); + } + const register = extractRegister(ctx.product); + const next = register + ? `NEXT STEP: This project's register is \`${register}\`. You MUST now read \`reference/${register}.md\` before producing any design output.` + : `NEXT STEP: You MUST now read the matching register reference (\`reference/brand.md\` or \`reference/product.md\`) before producing any design output. Pick based on PRODUCT.md above.`; + parts.push(next); + process.stdout.write(parts.join('\n\n---\n\n') + '\n'); +} + +const _running = process.argv[1]; +if (_running?.endsWith('context.mjs') || _running?.endsWith('context.mjs/')) { + cli(); +} diff --git a/skill/scripts/live-server.mjs b/skill/scripts/live-server.mjs index 0eae94b9b..0eb22fcb4 100644 --- a/skill/scripts/live-server.mjs +++ b/skill/scripts/live-server.mjs @@ -21,7 +21,7 @@ import path from 'node:path'; import net from 'node:net'; import { fileURLToPath } from 'node:url'; import { parseDesignMd } from './design-parser.mjs'; -import { resolveContextDir } from './load-context.mjs'; +import { resolveContextDir } from './context.mjs'; import { createLiveSessionStore } from './live-session-store.mjs'; import { getDesignSidecarPath, @@ -33,7 +33,7 @@ import { } from './impeccable-paths.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -// PRODUCT.md / DESIGN.md live wherever load-context.mjs resolves. The generated +// PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated // DESIGN sidecar is project-local at .impeccable/design.json, with legacy // DESIGN.json fallback for existing projects. const CONTEXT_DIR = resolveContextDir(process.cwd()); @@ -187,8 +187,7 @@ function loadBrowserScripts() { function hasProjectContext() { // PRODUCT.md carries brand voice / anti-references — that's what determines // whether variants are brand-aware. DESIGN.md (visual tokens) is a separate - // concern, surfaced by the design panel's own empty state. Legacy - // .impeccable.md is auto-migrated to PRODUCT.md by load-context.mjs. + // concern, surfaced by the design panel's own empty state. try { fs.accessSync(path.join(CONTEXT_DIR, 'PRODUCT.md'), fs.constants.R_OK); return true; diff --git a/skill/scripts/live.mjs b/skill/scripts/live.mjs index cafb0eca9..8acd300ed 100644 --- a/skill/scripts/live.mjs +++ b/skill/scripts/live.mjs @@ -21,7 +21,7 @@ import { execSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { loadContext } from './load-context.mjs'; +import { loadContext } from './context.mjs'; import { resolveFiles } from './live-inject.mjs'; import { readLiveServerInfo } from './impeccable-paths.mjs'; @@ -80,7 +80,7 @@ The agent should then: process.exit(1); } - // 4. Load PRODUCT.md + DESIGN.md context (auto-migrates legacy .impeccable.md) + // 4. Load PRODUCT.md + DESIGN.md context. const ctx = loadContext(process.cwd()); // 5. Compute drift-heal: compare resolved inject targets against the @@ -102,7 +102,6 @@ The agent should then: hasDesign: ctx.hasDesign, design: ctx.design, designPath: ctx.designPath, - migrated: ctx.migrated, }, null, 2)); } diff --git a/skill/scripts/load-context.mjs b/skill/scripts/load-context.mjs deleted file mode 100644 index dc340bf16..000000000 --- a/skill/scripts/load-context.mjs +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Shared context loader for every impeccable command that needs to know - * "who is this for" and "what does this look like". - * - * Input: project root (process.cwd()). - * - * Output (JSON to stdout): - * { - * hasProduct: boolean, // PRODUCT.md found (or auto-migrated) - * product: string | null, // PRODUCT.md contents - * productPath: string | null, // relative path - * hasDesign: boolean, // DESIGN.md found - * design: string | null, // DESIGN.md contents - * designPath: string | null, - * migrated: boolean, // true if we auto-renamed .impeccable.md -> PRODUCT.md - * contextDir: string, // absolute path of the directory the files were found in - * } - * - * Filename matching is case-insensitive for PRODUCT.md and DESIGN.md. The - * Google DESIGN.md convention is uppercase at repo root; Kiro-style and - * lowercase variants are also matched so users don't get punished for case. - * - * Lookup directory resolution (first match wins): - * 1. process.env.IMPECCABLE_CONTEXT_DIR (absolute or relative to cwd) - * 2. cwd, if PRODUCT.md / DESIGN.md / .impeccable.md is there (back-compat) - * 3. Auto-fallback subdirectories of cwd: .agents/context/, then docs/ - * 4. cwd as a default "no context found" location - * - * Legacy `.impeccable.md` -> PRODUCT.md migration only fires at cwd root; - * fallback directories are read-only as far as auto-rename is concerned. - */ - -import fs from 'node:fs'; -import path from 'node:path'; - -const PRODUCT_NAMES = ['PRODUCT.md', 'Product.md', 'product.md']; -const DESIGN_NAMES = ['DESIGN.md', 'Design.md', 'design.md']; -const LEGACY_NAMES = ['.impeccable.md']; -const FALLBACK_DIRS = ['.agents/context', 'docs']; - -/** - * Resolve the directory that holds PRODUCT.md / DESIGN.md for - * this project. Exported so other scripts (e.g. live-server.mjs) can read the - * design files from the same location the loader uses. - */ -export function resolveContextDir(cwd = process.cwd()) { - // 1. Explicit override - const envDir = process.env.IMPECCABLE_CONTEXT_DIR; - if (envDir && envDir.trim()) { - const trimmed = envDir.trim(); - return path.isAbsolute(trimmed) ? trimmed : path.resolve(cwd, trimmed); - } - - // 2. cwd wins if any canonical or legacy file is there. We check legacy too - // so the auto-migration path in loadContext stays predictable. - if (firstExisting(cwd, [...PRODUCT_NAMES, ...DESIGN_NAMES, ...LEGACY_NAMES])) { - return cwd; - } - - // 3. Auto-fallback subdirs. Match if PRODUCT.md or DESIGN.md is present; - // legacy `.impeccable.md` does not pull the lookup into a fallback dir. - for (const rel of FALLBACK_DIRS) { - const candidate = path.resolve(cwd, rel); - if (firstExisting(candidate, [...PRODUCT_NAMES, ...DESIGN_NAMES])) { - return candidate; - } - } - - // 4. Nothing found — keep the historical "default to cwd" behaviour so the - // caller's `hasProduct === false` branch still fires the same way. - return cwd; -} - -export function loadContext(cwd = process.cwd()) { - let migrated = false; - const contextDir = resolveContextDir(cwd); - - // 1. Look for PRODUCT.md (case-insensitive) in the resolved dir - let productPath = firstExisting(contextDir, PRODUCT_NAMES); - - // 2. Legacy: if no PRODUCT.md but .impeccable.md exists at cwd root, rename - // it in place. We only migrate at the root — fallback dirs are read-only - // so we don't surprise users by mutating files under docs/ or .agents/. - if (!productPath && contextDir === cwd) { - const legacyPath = firstExisting(cwd, LEGACY_NAMES); - if (legacyPath) { - const newPath = path.join(cwd, 'PRODUCT.md'); - try { - fs.renameSync(legacyPath, newPath); - productPath = newPath; - migrated = true; - } catch { - // Rename failed (permissions, etc.) — fall back to reading legacy in place - productPath = legacyPath; - } - } - } - - // 3. DESIGN.md (case-insensitive) - const designPath = firstExisting(contextDir, DESIGN_NAMES); - - const product = productPath ? safeRead(productPath) : null; - const design = designPath ? safeRead(designPath) : null; - - return { - hasProduct: !!product, - product, - productPath: productPath ? path.relative(cwd, productPath) : null, - hasDesign: !!design, - design, - designPath: designPath ? path.relative(cwd, designPath) : null, - migrated, - contextDir, - }; -} - -function firstExisting(dir, names) { - for (const name of names) { - const abs = path.join(dir, name); - if (fs.existsSync(abs)) return abs; - } - return null; -} - -function safeRead(p) { - try { return fs.readFileSync(p, 'utf-8'); } catch { return null; } -} - -// --------------------------------------------------------------------------- -// CLI mode — print the context as JSON -// --------------------------------------------------------------------------- - -function cli() { - const result = loadContext(process.cwd()); - console.log(JSON.stringify(result, null, 2)); -} - -const _running = process.argv[1]; -if (_running?.endsWith('load-context.mjs') || _running?.endsWith('load-context.mjs/')) { - cli(); -} diff --git a/tests/load-context.test.mjs b/tests/context.test.mjs similarity index 57% rename from tests/load-context.test.mjs rename to tests/context.test.mjs index 7f3dc168d..30d677c76 100644 --- a/tests/load-context.test.mjs +++ b/tests/context.test.mjs @@ -2,10 +2,11 @@ * Tests for the shared context loader (PRODUCT.md / DESIGN.md resolver). * Run with: node --test tests/load-context.test.mjs * - * Covers the resolution order added for issue #119: - * 1. IMPECCABLE_CONTEXT_DIR env var (absolute or relative) - * 2. cwd, when canonical or legacy files are at the root (back-compat) - * 3. Auto-fallback to .agents/context/ then docs/ + * Covers the resolution order: + * 1. cwd, when canonical files are at the root + * 2. Auto-fallback to .agents/context/ then docs/ + * 3. IMPECCABLE_CONTEXT_DIR env var as a power-user escape hatch (only + * consulted when the default paths come up empty) * 4. Default to cwd when nothing is found * * Each test runs in its own scratch dir under os.tmpdir() so the suite stays @@ -18,7 +19,10 @@ import fs from 'node:fs'; import path from 'node:path'; import os from 'node:os'; -import { loadContext, resolveContextDir } from '../skill/scripts/load-context.mjs'; +import { loadContext, resolveContextDir } from '../skill/scripts/context.mjs'; + +import { fileURLToPath } from 'node:url'; +const SCRIPT_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'skill', 'scripts', 'context.mjs'); let scratch; let savedEnv; @@ -53,11 +57,6 @@ describe('resolveContextDir', () => { assert.equal(resolveContextDir(scratch), scratch); }); - it('returns cwd when only legacy .impeccable.md is at the root', () => { - write('.impeccable.md'); - assert.equal(resolveContextDir(scratch), scratch); - }); - it('falls back to .agents/context/ when root is clean', () => { write('.agents/context/PRODUCT.md'); assert.equal(resolveContextDir(scratch), path.join(scratch, '.agents', 'context')); @@ -80,13 +79,13 @@ describe('resolveContextDir', () => { assert.equal(resolveContextDir(scratch), scratch); }); - it('honors IMPECCABLE_CONTEXT_DIR with a relative path', () => { + it('uses IMPECCABLE_CONTEXT_DIR as a fallback when defaults are empty (relative path)', () => { write('design/PRODUCT.md'); process.env.IMPECCABLE_CONTEXT_DIR = 'design'; assert.equal(resolveContextDir(scratch), path.join(scratch, 'design')); }); - it('honors IMPECCABLE_CONTEXT_DIR with an absolute path', () => { + it('uses IMPECCABLE_CONTEXT_DIR as a fallback when defaults are empty (absolute path)', () => { const elsewhere = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-elsewhere-')); try { process.env.IMPECCABLE_CONTEXT_DIR = elsewhere; @@ -96,11 +95,11 @@ describe('resolveContextDir', () => { } }); - it('IMPECCABLE_CONTEXT_DIR wins even when files exist at the root', () => { + it('default paths win over IMPECCABLE_CONTEXT_DIR (lazy escape hatch)', () => { write('PRODUCT.md', 'root'); write('design/PRODUCT.md', 'overridden'); process.env.IMPECCABLE_CONTEXT_DIR = 'design'; - assert.equal(resolveContextDir(scratch), path.join(scratch, 'design')); + assert.equal(resolveContextDir(scratch), scratch); }); it('ignores empty IMPECCABLE_CONTEXT_DIR', () => { @@ -114,8 +113,8 @@ describe('resolveContextDir', () => { }); }); -describe('loadContext (backward compatibility)', () => { - it('reads PRODUCT.md and DESIGN.md from the root the same way as before', () => { +describe('loadContext', () => { + it('reads PRODUCT.md and DESIGN.md from the root', () => { write('PRODUCT.md', '# product content\n'); write('DESIGN.md', '# design content\n'); const ctx = loadContext(scratch); @@ -128,18 +127,6 @@ describe('loadContext (backward compatibility)', () => { assert.equal(ctx.contextDir, scratch); }); - it('migrates legacy .impeccable.md -> PRODUCT.md at root', () => { - write('.impeccable.md', '# legacy body\n'); - const ctx = loadContext(scratch); - assert.equal(ctx.migrated, true); - assert.equal(ctx.hasProduct, true); - assert.match(ctx.product, /legacy body/); - assert.ok(fs.existsSync(path.join(scratch, 'PRODUCT.md'))); - assert.ok(!fs.existsSync(path.join(scratch, '.impeccable.md'))); - }); -}); - -describe('loadContext (fallback dirs)', () => { it('reads from .agents/context/ when the root is clean', () => { write('.agents/context/PRODUCT.md', '# product in agents\n'); write('.agents/context/DESIGN.md', '# design in agents\n'); @@ -151,7 +138,6 @@ describe('loadContext (fallback dirs)', () => { // productPath/designPath are relative to cwd, not contextDir assert.equal(ctx.productPath, path.join('.agents', 'context', 'PRODUCT.md')); assert.equal(ctx.designPath, path.join('.agents', 'context', 'DESIGN.md')); - assert.equal(ctx.migrated, false); }); it('reads from docs/ when .agents/context/ is empty', () => { @@ -161,20 +147,10 @@ describe('loadContext (fallback dirs)', () => { assert.equal(ctx.contextDir, path.join(scratch, 'docs')); assert.equal(ctx.productPath, path.join('docs', 'PRODUCT.md')); }); - - it('does not auto-migrate .impeccable.md inside fallback dirs', () => { - write('docs/.impeccable.md', '# legacy in docs\n'); - const ctx = loadContext(scratch); - // .impeccable.md inside a fallback dir doesn't pull the lookup there, - // and we never auto-rename outside the cwd root. - assert.equal(ctx.hasProduct, false); - assert.equal(ctx.migrated, false); - assert.ok(fs.existsSync(path.join(scratch, 'docs', '.impeccable.md'))); - }); }); -describe('loadContext (IMPECCABLE_CONTEXT_DIR override)', () => { - it('reads from the override path when set', () => { +describe('loadContext (IMPECCABLE_CONTEXT_DIR escape hatch)', () => { + it('reads from the override path when defaults are empty', () => { write('design/PRODUCT.md', '# overridden product\n'); write('design/DESIGN.md', '# overridden design\n'); process.env.IMPECCABLE_CONTEXT_DIR = 'design'; @@ -185,6 +161,15 @@ describe('loadContext (IMPECCABLE_CONTEXT_DIR override)', () => { assert.equal(ctx.contextDir, path.join(scratch, 'design')); }); + it('does not override defaults when both exist (lazy escape hatch)', () => { + write('PRODUCT.md', '# root product\n'); + write('design/PRODUCT.md', '# overridden product\n'); + process.env.IMPECCABLE_CONTEXT_DIR = 'design'; + const ctx = loadContext(scratch); + assert.match(ctx.product, /root product/); + assert.equal(ctx.contextDir, scratch); + }); + it('reports a missing override directory as no-context, not as a crash', () => { process.env.IMPECCABLE_CONTEXT_DIR = 'no/such/dir'; const ctx = loadContext(scratch); @@ -195,3 +180,64 @@ describe('loadContext (IMPECCABLE_CONTEXT_DIR override)', () => { assert.equal(ctx.contextDir, path.resolve(scratch, 'no/such/dir')); }); }); + +describe('context.mjs CLI', () => { + it('emits NO_PRODUCT_MD directive when no PRODUCT.md is found', async () => { + const { spawnSync } = await import('node:child_process'); + const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' }); + assert.equal(res.status, 0); + assert.match(res.stdout, /^NO_PRODUCT_MD:/); + assert.match(res.stdout, /reference\/teach\.md/); + }); + + it('prints a PRODUCT.md markdown block when only PRODUCT.md exists', async () => { + write('PRODUCT.md', '# Acme\n\nbody\n'); + const { spawnSync } = await import('node:child_process'); + const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' }); + assert.equal(res.status, 0); + assert.match(res.stdout, /^# PRODUCT\.md/); + assert.match(res.stdout, /# Acme/); + assert.equal(res.stdout.includes('# DESIGN.md'), false); + // The NEXT STEP directive is always appended after `---`. + assert.match(res.stdout, /\n---\n\nNEXT STEP:/); + }); + + it('concatenates PRODUCT.md and DESIGN.md with a --- separator', async () => { + write('PRODUCT.md', '# Acme product\n'); + write('DESIGN.md', '# Acme design\n'); + const { spawnSync } = await import('node:child_process'); + const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' }); + assert.equal(res.status, 0); + assert.match(res.stdout, /^# PRODUCT\.md/); + assert.match(res.stdout, /\n---\n/); + assert.match(res.stdout, /# DESIGN\.md\n\n# Acme design/); + assert.match(res.stdout, /NEXT STEP:/); + }); + + it('reads from a fallback dir when cwd is clean', async () => { + write('.agents/context/PRODUCT.md', '# fallback product\n'); + const { spawnSync } = await import('node:child_process'); + const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' }); + assert.equal(res.status, 0); + assert.match(res.stdout, /^# PRODUCT\.md/); + assert.match(res.stdout, /# fallback product/); + }); + + it('names the register-specific reference when PRODUCT.md declares one', async () => { + write('PRODUCT.md', '# Acme\n\n## Register\n\nbrand\n'); + const { spawnSync } = await import('node:child_process'); + const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' }); + assert.equal(res.status, 0); + assert.match(res.stdout, /NEXT STEP: This project's register is `brand`\./); + assert.match(res.stdout, /read `reference\/brand\.md`/); + }); + + it('falls back to a generic register directive when no register field is present', async () => { + write('PRODUCT.md', '# Acme\n\n(no register field)\n'); + const { spawnSync } = await import('node:child_process'); + const res = spawnSync(process.execPath, [SCRIPT_PATH], { cwd: scratch, encoding: 'utf8' }); + assert.equal(res.status, 0); + assert.match(res.stdout, /NEXT STEP: You MUST now read the matching register reference/); + assert.match(res.stdout, /reference\/brand\.md.*reference\/product\.md/); + }); +}); diff --git a/tests/skill-behavior/README.md b/tests/skill-behavior/README.md new file mode 100644 index 000000000..cab227664 --- /dev/null +++ b/tests/skill-behavior/README.md @@ -0,0 +1,71 @@ +# Skill-behavior tests + +LLM-backed scenarios that verify how the impeccable skill drives +PRODUCT.md / DESIGN.md loading. Each scenario runs against the cheapest +tier of each major provider (Anthropic, OpenAI, Google) so a full sweep +costs a few cents and finishes in ~2 minutes. + +These are the tests you re-run when you refactor anything in SKILL.md's +`## Setup` section. They fail when the agent stops following the loading +contract. + +## Run + +```bash +bun run test:skill-behavior +IMPECCABLE_SKILL_BEHAVIOR_VERBOSE=1 bun run test:skill-behavior # dump per-scenario traces +IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-haiku-4-5 bun run test:skill-behavior # scope to one model +``` + +Requires `.env` at repo root with at least one of `ANTHROPIC_API_KEY`, +`OPENAI_API_KEY`, `GOOGLE_CLOUD_API_KEY`. Providers without a key are +skipped, not failed. + +## How it works + +Each scenario: + +1. `prepareWorkspace()` mints a temp dir, symlinks the canonical skill + into `/.claude/skills/impeccable`, and optionally writes + `PRODUCT.md` / `DESIGN.md` fixtures. +2. `runTurn()` inlines `SKILL.md` (placeholders neutralized) as the + system prompt and runs Vercel AI SDK `generateText` with four + workspace-scoped tools: `bash`, `read`, `write`, `list`. +3. The tools record every call into a `trace` that the test asserts on. +4. For scenario 4, a second `runTurn` reuses turn 1's `responseMessages` + so the model sees a real multi-turn conversation. + +The trace is the source of truth, not the model's free-form reply. + +## Scenarios + +| # | Setup | Assertion | +|---|---|---| +| 1 | empty workspace | runs `context.mjs` (which prints a `NO_PRODUCT_MD` directive); agent then loads `reference/teach.md` via Read or `cat`; does **not** start writing HTML/CSS | +| 2 | PRODUCT.md only (with `## Register: brand`) | runs `context.mjs` 1-3 times; loads `reference/brand.md` | +| 3 | PRODUCT.md + DESIGN.md (brand register) | runs `context.mjs` 1-3 times; loads `reference/brand.md`; consults the design system (DESIGN.md bundled in output, but CSS / tokens / directory listing also count) | +| 4 | PRODUCT.md + DESIGN.md, context already loaded in turn 1 | turn 2 does **not** re-run `context.mjs`; `reference/brand.md` is loaded across turns 1+2 | +| 5 | PRODUCT.md WITHOUT a `## Register` field; task cue says "landing page" | runs `context.mjs` (which emits a generic register directive); agent loads `reference/brand.md` via task-cue cascade | + +## Baseline state (2026-05-20) + +Captured after condensing Setup to four bullets and teaching `context.mjs` +to emit a `NEXT STEP:` directive that names the matching register +reference when PRODUCT.md declares one (and a generic cascade prompt when +it doesn't). Use this table when comparing pre/post refactor: a +regression is "more failures than baseline", not "any failures at all". + +| Scenario | claude-haiku-4-5 | gpt-5.4-mini | gemini-3.1-flash-lite | +|---|---|---|---| +| 1 (no context) | pass (variance: ~1 in 5 the agent stops after `context.mjs` without loading `teach.md`) | pass | pass | +| 2 (product only) | pass | pass | pass | +| 3 (product + design) | pass | pass | pass | +| 4 (already loaded) | pass | **fail** | pass | +| 5 (no register field, task-cue cascade) | pass | pass | pass | + +13-14 / 15 typical. The stable failure is gpt-5.4-mini scenario 4: +it re-runs `context.mjs` on turn 2 despite seeing its output in turn 1's +history. Same known weakness as the v3.2.0 script baseline; Claude and +Gemini honor the "don't re-run" rule. The S1 claude flake is rare +(observed once across many runs) and likely terminates early under +load — re-running typically clears it. diff --git a/tests/skill-behavior/fixtures.mjs b/tests/skill-behavior/fixtures.mjs new file mode 100644 index 000000000..dc96e1dc2 --- /dev/null +++ b/tests/skill-behavior/fixtures.mjs @@ -0,0 +1,98 @@ +/** + * Realistic-looking PRODUCT.md / DESIGN.md fixtures. + * + * Long enough to clear the "<200 chars / placeholder" heuristic the loader + * uses to decide whether to gate on `teach`. Plausible enough that the agent + * treats them as real context rather than test scaffolding. + */ +export const PRODUCT_MD_SAMPLE = `# Acme Notes + +## Register +brand + +## Product Purpose +Acme Notes is a marketing-driven landing page for a research-grade note-taking +tool aimed at independent scientists and graduate students. The site needs to +communicate that the product respects the reader's intelligence — no SaaS +buzzwords, no metric-theater, no "trusted by leading teams" wallpaper. + +## Users +Working researchers (PhD students, postdocs, principal investigators) who +already maintain disciplined note-taking systems and are choosing between +ours and rolling their own in a Zettelkasten plugin. + +## Brand +Editorial, considered, technical. The product is for people who quote +Knuth. The voice is closer to a long-read magazine than to a startup +landing page. + +## Anti-references +- Notion (too consumer / too rounded) +- Obsidian (too community-cottagecore) +- Any SaaS landing page with a hero-metric grid + +## Strategic Principles +- Type does most of the work. The hero is words, not chrome. +- One named accent color, used sparingly. +- Never lead with screenshots. Lead with the idea. +`; + +/** + * Same project shape as PRODUCT_MD_SAMPLE but with no `## Register` field. + * Exercises the cascade fallback (task cue then surface in focus) in + * scenarios where context.mjs cannot detect the register and the agent + * must follow the SKILL.md priority list to pick brand.md. + */ +export const PRODUCT_MD_SAMPLE_NO_REGISTER = `# Acme Notes + +## Product Purpose +Acme Notes is a marketing-driven landing page for a research-grade note-taking +tool aimed at independent scientists and graduate students. The site needs to +communicate that the product respects the reader's intelligence: no SaaS +buzzwords, no metric-theater, no "trusted by leading teams" wallpaper. + +## Users +Working researchers (PhD students, postdocs, principal investigators) who +already maintain disciplined note-taking systems and are choosing between +ours and rolling their own in a Zettelkasten plugin. + +## Brand +Editorial, considered, technical. The product is for people who quote +Knuth. The voice is closer to a long-read magazine than to a startup +landing page. + +## Anti-references +- Notion (too consumer / too rounded) +- Obsidian (too community-cottagecore) +- Any SaaS landing page with a hero-metric grid + +## Strategic Principles +- Type does most of the work. The hero is words, not chrome. +- One named accent color, used sparingly. +- Never lead with screenshots. Lead with the idea. +`; + +export const DESIGN_MD_SAMPLE = `# Acme Notes — Design System + +## Colors +- \`--ink\`: oklch(0.16 0.02 250) — body copy +- \`--paper\`: oklch(0.98 0.01 90) — body background +- \`--accent\`: oklch(0.55 0.18 28) — terracotta, used at <8% surface + +## Typography +- Display: GT Sectra (commercial), 700, tracking -0.02em +- Body: Inter, 400, 1.55 line-height, 65ch max +- Mono: JetBrains Mono, 400 (rare, only for callouts) + +## Spacing +Multi-modular scale: 4 / 8 / 12 / 24 / 48 / 96 px. + +## Elevation +Mostly flat. A single 1px hairline border at oklch(0.16 0.02 250 / 0.08) +separates major regions. No drop shadows under 16px blur. + +## Components +- Buttons: text-only by default; a single solid primary in accent for CTAs. +- Cards: avoid; prefer hairlined regions and inline lists. +- Forms: floating labels, no border on the input — underline only. +`; diff --git a/tests/skill-behavior/harness.mjs b/tests/skill-behavior/harness.mjs new file mode 100644 index 000000000..4ba302a0b --- /dev/null +++ b/tests/skill-behavior/harness.mjs @@ -0,0 +1,296 @@ +/** + * Sandboxed scenario runner for skill-behavior tests. + * + * Each scenario: + * 1. Creates a temp workspace. + * 2. Symlinks the real .claude/skills/impeccable into the workspace so + * scripts (load-context.mjs, etc.) resolve from the canonical path + * the skill references. + * 3. Optionally writes PRODUCT.md / DESIGN.md fixtures. + * 4. Inlines SKILL.md as the system prompt (placeholders stripped to + * neutral values so the same body works for all providers). + * 5. Runs Vercel AI SDK generateText with workspace-scoped tools + * (bash, read, write, list). + * 6. Captures every tool call and returns a trace + the raw response + * messages (so multi-turn scenarios can append to them). + * + * The harness deliberately mirrors the live-mode E2E pattern: real LLM, + * no mocks, but tightly bounded execution surface so we observe the routing + * behavior of the skill without paying for full-fledged design work. + */ +import { generateText, stepCountIs, tool } from 'ai'; +import { z } from 'zod'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, '..', '..'); +const SKILL_SOURCE_DIR = path.join(REPO_ROOT, 'skill'); +const MAX_BASH_OUTPUT_BYTES = 200_000; + +/** + * Strip the YAML frontmatter and replace `{{...}}` placeholders so SKILL.md + * is provider-neutral when inlined. + */ +function loadSkillBody() { + let md = fs.readFileSync(path.join(SKILL_SOURCE_DIR, 'SKILL.md'), 'utf8'); + // Strip frontmatter. + if (md.startsWith('---')) { + const end = md.indexOf('\n---', 3); + if (end !== -1) md = md.slice(end + 4).trimStart(); + } + // The source uses placeholders that the build step replaces per-provider. + // For the test harness we want a single body that works for any provider, + // and the scripts the skill references live at .claude/skills/impeccable/ + // (the workspace symlink), so hard-code those values. + md = md + .replaceAll('{{model}}', 'the assistant') + .replaceAll('{{command_prefix}}', '/') + .replaceAll('{{ask_instruction}}', 'Ask the user') + .replaceAll('{{config_file}}', 'AGENTS.md') + .replaceAll('{{scripts_path}}', '.claude/skills/impeccable/scripts') + .replaceAll('{{command_hint}}', 'command'); + return md.trim(); +} + +export const SKILL_BODY = loadSkillBody(); + +/** + * Create a temp workspace and prepopulate it. + * + * - `.claude/skills/impeccable` is symlinked at the SOURCE skill dir (not + * the built `.claude/skills/impeccable/`) so the test exercises whatever + * is in `skill/` right now, without needing `bun run build` to refresh + * the harness output dirs. The trade-off: reference files surface their + * raw `{{placeholders}}`, but the assertions only check tool calls, not + * their content. + * - `files` lets the test seed PRODUCT.md / DESIGN.md (or anything else). + */ +export function prepareWorkspace({ files = {} } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-skill-test-')); + fs.mkdirSync(path.join(dir, '.claude', 'skills'), { recursive: true }); + fs.symlinkSync(SKILL_SOURCE_DIR, path.join(dir, '.claude', 'skills', 'impeccable'), 'dir'); + for (const [name, contents] of Object.entries(files)) { + const target = path.join(dir, name); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, contents); + } + return dir; +} + +export function cleanupWorkspace(dir) { + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // Best effort — temp dirs eventually get reaped by the OS. + } +} + +function safeResolve(root, userPath) { + if (typeof userPath !== 'string' || !userPath.length) { + return { error: 'path is required' }; + } + if (userPath.startsWith('/') || /^[a-zA-Z]:[\\/]/.test(userPath)) { + return { error: 'absolute paths are not allowed' }; + } + const resolved = path.resolve(root, userPath); + const rel = path.relative(root, resolved); + if (rel.startsWith('..') || rel.split(path.sep).includes('..')) { + return { error: 'path escapes the workspace' }; + } + return resolved; +} + +function execBash(workspace, command, timeoutMs = 20_000) { + return new Promise((resolve) => { + const proc = spawn('bash', ['-lc', command], { cwd: workspace }); + let stdout = ''; + let stderr = ''; + const truncatedFlag = { val: false }; + const onChunk = (which) => (chunk) => { + const str = chunk.toString(); + if (which === 'out') { + if (stdout.length + str.length > MAX_BASH_OUTPUT_BYTES) { + stdout += str.slice(0, MAX_BASH_OUTPUT_BYTES - stdout.length); + truncatedFlag.val = true; + } else { + stdout += str; + } + } else { + if (stderr.length + str.length > MAX_BASH_OUTPUT_BYTES) { + stderr += str.slice(0, MAX_BASH_OUTPUT_BYTES - stderr.length); + truncatedFlag.val = true; + } else { + stderr += str; + } + } + }; + proc.stdout.on('data', onChunk('out')); + proc.stderr.on('data', onChunk('err')); + const timer = setTimeout(() => { + proc.kill('SIGKILL'); + resolve({ exitCode: null, stdout, stderr: stderr + '\n[TIMED OUT]', truncated: truncatedFlag.val }); + }, timeoutMs); + proc.on('exit', (code) => { + clearTimeout(timer); + resolve({ exitCode: code, stdout, stderr, truncated: truncatedFlag.val }); + }); + proc.on('error', (err) => { + clearTimeout(timer); + resolve({ exitCode: null, stdout, stderr: stderr + `\n[SPAWN ERROR] ${String(err)}`, truncated: truncatedFlag.val }); + }); + }); +} + +/** + * Build the workspace-scoped tool set + the trace it writes into. + * Returns `{ tools, trace }`. The trace mutates in place as the agent runs. + */ +export function makeTools(workspace) { + const trace = { + toolCalls: [], + bashCommands: [], + readPaths: [], + writePaths: [], + listPaths: [], + }; + function record(name, input) { + trace.toolCalls.push({ name, input }); + if (name === 'bash' && typeof input?.command === 'string') trace.bashCommands.push(input.command); + if (name === 'read' && typeof input?.path === 'string') trace.readPaths.push(input.path); + if (name === 'write' && typeof input?.path === 'string') trace.writePaths.push(input.path); + if (name === 'list' && typeof input?.path === 'string') trace.listPaths.push(input.path); + } + const tools = { + bash: tool({ + description: + 'Run a bash command in the workspace root. Use this to invoke skill scripts (e.g. `node .claude/skills/impeccable/scripts/load-context.mjs`).', + inputSchema: z.object({ + command: z.string().describe('The bash command to execute.'), + }), + execute: async ({ command }) => { + record('bash', { command }); + const res = await execBash(workspace, command); + const head = `exit=${res.exitCode}`; + const body = (res.stdout ? `stdout:\n${res.stdout}` : '') + (res.stderr ? `\nstderr:\n${res.stderr}` : ''); + return `${head}\n${body}${res.truncated ? '\n[output truncated]' : ''}`; + }, + }), + read: tool({ + description: 'Read a file from the workspace. Path must be workspace-relative.', + inputSchema: z.object({ + path: z.string().describe('Workspace-relative file path.'), + }), + execute: async ({ path: p }) => { + record('read', { path: p }); + const resolved = safeResolve(workspace, p); + if (typeof resolved !== 'string') return `Error: ${resolved.error}`; + if (!fs.existsSync(resolved)) return `File not found: ${p}`; + const stat = fs.statSync(resolved); + if (stat.isDirectory()) return `Path is a directory: ${p}. Use list instead.`; + return fs.readFileSync(resolved, 'utf8'); + }, + }), + write: tool({ + description: 'Write or overwrite a file in the workspace. Creates parent directories as needed.', + inputSchema: z.object({ + path: z.string().describe('Workspace-relative file path.'), + contents: z.string().describe('Full file contents.'), + }), + execute: async ({ path: p, contents }) => { + record('write', { path: p, contents }); + const resolved = safeResolve(workspace, p); + if (typeof resolved !== 'string') return `Error: ${resolved.error}`; + fs.mkdirSync(path.dirname(resolved), { recursive: true }); + fs.writeFileSync(resolved, contents); + return `Wrote ${Buffer.byteLength(contents, 'utf8')} bytes to ${p}`; + }, + }), + list: tool({ + description: 'List a workspace directory. Defaults to the workspace root.', + inputSchema: z.object({ + path: z.string().default('.').describe('Workspace-relative directory path.'), + }), + execute: async ({ path: p }) => { + record('list', { path: p }); + const resolved = safeResolve(workspace, p); + if (typeof resolved !== 'string') return `Error: ${resolved.error}`; + if (!fs.existsSync(resolved)) return `Not found: ${p}`; + const stat = fs.statSync(resolved); + if (!stat.isDirectory()) return `Not a directory: ${p}`; + const entries = fs.readdirSync(resolved).map((name) => { + const st = fs.statSync(path.join(resolved, name)); + return st.isDirectory() ? `${name}/` : name; + }); + return entries.length ? entries.join('\n') : '(empty)'; + }, + }), + }; + return { tools, trace }; +} + +/** + * Run one scenario turn against a model. + * + * `priorMessages` lets multi-turn scenarios chain context from a previous + * call (append `result.response.messages` between turns). + */ +export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8 }) { + const { tools, trace } = makeTools(workspace); + const messages = [ + ...priorMessages, + { role: 'user', content: userPrompt }, + ]; + let result; + try { + result = await generateText({ + model, + system: SKILL_BODY, + messages, + tools, + stopWhen: [stepCountIs(maxSteps)], + }); + } catch (err) { + return { trace, error: String(err), text: '', responseMessages: messages, finishReason: 'error' }; + } + const responseMessages = [...messages, ...(result.response?.messages ?? [])]; + return { + trace, + text: result.text ?? '', + finishReason: result.finishReason, + usage: result.usage, + responseMessages, + }; +} + +/** + * Heuristic helpers — keep the assertion intent declarative in the test file. + */ +export function bashCommandsMatching(trace, substring) { + return trace.bashCommands.filter((cmd) => cmd.includes(substring)); +} + +export function readsMatching(trace, substring) { + return trace.readPaths.filter((p) => p.toLowerCase().includes(substring.toLowerCase())); +} + +/** + * True if the agent loaded a file by Read OR by a bash `cat` (some models + * stream multiple files via bash to save tool calls). + */ +export function fileLoaded(trace, filename) { + return readsMatching(trace, filename).length > 0 || bashCommandsMatching(trace, filename).length > 0; +} + +export function summarizeTrace(trace) { + return { + totalCalls: trace.toolCalls.length, + byName: trace.toolCalls.reduce((acc, c) => ((acc[c.name] = (acc[c.name] ?? 0) + 1), acc), {}), + bashCommands: trace.bashCommands, + readPaths: trace.readPaths, + writePaths: trace.writePaths, + }; +} diff --git a/tests/skill-behavior/providers.mjs b/tests/skill-behavior/providers.mjs new file mode 100644 index 000000000..e2cf5e4d7 --- /dev/null +++ b/tests/skill-behavior/providers.mjs @@ -0,0 +1,91 @@ +/** + * Multi-provider model factory for the skill-behavior test harness. + * + * The user explicitly asked for the cheapest tier of each major provider to + * keep CI cost in the cents-per-run range while still exercising real LLM + * decision-making against the skill body. + * + * Anthropic and OpenAI use the Vercel AI SDK providers. Google uses + * @ai-sdk/google for the same reason — uniform tool-use semantics across all + * three keeps the harness tiny. + * + * .env is loaded from the repo root (copied from impeccable-evals). Tests + * skip cleanly when the matching key is unset rather than failing CI. + */ +import { anthropic } from '@ai-sdk/anthropic'; +import { google } from '@ai-sdk/google'; +import { openai } from '@ai-sdk/openai'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, '..', '..'); + +function loadEnv() { + const envPath = path.join(REPO_ROOT, '.env'); + if (!fs.existsSync(envPath)) return; + const text = fs.readFileSync(envPath, 'utf8'); + for (const line of text.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const eq = trimmed.indexOf('='); + if (eq === -1) continue; + const key = trimmed.slice(0, eq).trim(); + let value = trimmed.slice(eq + 1).trim(); + if (value.startsWith('"') && value.endsWith('"')) value = value.slice(1, -1); + if (value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1); + if (!process.env[key]) process.env[key] = value; + } +} +loadEnv(); + +export const PROVIDERS = { + anthropic: { envKey: 'ANTHROPIC_API_KEY', label: 'Anthropic' }, + openai: { envKey: 'OPENAI_API_KEY', label: 'OpenAI' }, + google: { envKey: 'GOOGLE_CLOUD_API_KEY', label: 'Google' }, +}; + +export function detectProvider(modelId) { + if (modelId.startsWith('claude-')) return 'anthropic'; + if (modelId.startsWith('gpt-')) return 'openai'; + if (modelId.startsWith('gemini-')) return 'google'; + throw new Error(`Unsupported model id: "${modelId}"`); +} + +export function hasKey(provider) { + const meta = PROVIDERS[provider]; + if (!meta) return false; + return Boolean(process.env[meta.envKey]); +} + +export function getModel(modelId) { + const provider = detectProvider(modelId); + if (provider === 'anthropic') return anthropic(modelId); + if (provider === 'openai') return openai(modelId); + if (provider === 'google') { + // The @ai-sdk/google provider reads GOOGLE_GENERATIVE_AI_API_KEY by + // default; the evals .env stores the same value under + // GOOGLE_CLOUD_API_KEY. Mirror it so the SDK picks it up automatically. + if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY && process.env.GOOGLE_CLOUD_API_KEY) { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = process.env.GOOGLE_CLOUD_API_KEY; + } + return google(modelId); + } + throw new Error(`Unsupported provider: ${provider}`); +} + +/** + * Default model lineup. Cheapest tier per provider — the test is about + * routing/loading behavior, not design output quality, so cheap is fine. + * Override with IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-foo,gpt-bar. + */ +export const DEFAULT_MODELS = ['claude-haiku-4-5', 'gpt-5.4-mini', 'gemini-3.1-flash-lite']; + +export function resolveModelList() { + const override = process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS; + if (override && override.trim()) { + return override.split(',').map((s) => s.trim()).filter(Boolean); + } + return DEFAULT_MODELS; +} diff --git a/tests/skill-behavior/scenarios.test.mjs b/tests/skill-behavior/scenarios.test.mjs new file mode 100644 index 000000000..84547f46b --- /dev/null +++ b/tests/skill-behavior/scenarios.test.mjs @@ -0,0 +1,254 @@ +/** + * Skill-behavior scenarios — verify how the agent loads PRODUCT.md / DESIGN.md + * across a controlled matrix of starting states. + * + * Refactors that touch the Setup section of SKILL.md should keep these + * assertions green. If you change Setup intentionally and the assertions + * flip, that's the test catching the regression you wanted to catch. + * + * Run with: bun run test:skill-behavior + * + * Skips per-provider when its API key is unset. The default model lineup is + * the cheapest tier of each major provider so a full sweep costs a few cents. + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + prepareWorkspace, + cleanupWorkspace, + runTurn, + bashCommandsMatching, + readsMatching, + fileLoaded, + summarizeTrace, +} from './harness.mjs'; +import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs'; +import { PRODUCT_MD_SAMPLE, PRODUCT_MD_SAMPLE_NO_REGISTER, DESIGN_MD_SAMPLE } from './fixtures.mjs'; + +const CRAFT_PROMPT = '/impeccable craft a landing page for the project in this workspace'; +const PRIMER_PROMPT = + 'Take a quick look at the project. What register is this? Run the impeccable context loader once if you need to.'; + +const VERBOSE = process.env.IMPECCABLE_SKILL_BEHAVIOR_VERBOSE === '1'; + +function logTrace(label, scenario, model, trace, extras = {}) { + if (!VERBOSE) return; + const summary = summarizeTrace(trace); + console.error( + `\n[${label}] ${scenario} (${model})\n${JSON.stringify({ ...summary, ...extras }, null, 2)}\n`, + ); +} + +for (const modelId of resolveModelList()) { + const provider = detectProvider(modelId); + const keyPresent = hasKey(provider); + + describe(`skill behavior :: ${modelId}`, () => { + if (!keyPresent) { + it(`skipped — ${PROVIDERS[provider].envKey} is unset`, { skip: true }, () => {}); + return; + } + const model = getModel(modelId); + + it('scenario 1: no PRODUCT.md / DESIGN.md', async () => { + const workspace = prepareWorkspace({ files: {} }); + try { + const { trace, text } = await runTurn({ + workspace, + model, + userPrompt: CRAFT_PROMPT, + maxSteps: 6, + }); + logTrace('S1', 'no-context', modelId, trace, { textSample: text.slice(0, 400) }); + // Agent runs context.mjs, sees NO_PRODUCT_MD directive, loads + // teach.md and follows it. Accept either Read or bash `cat` for + // the teach.md load — different models pick different tools. + const loadCalls = bashCommandsMatching(trace, 'context.mjs'); + assert.ok( + loadCalls.length >= 1, + `expected agent to run context.mjs at least once; got ${loadCalls.length}.\n` + + `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, + ); + const teachLoaded = + readsMatching(trace, 'teach.md').length > 0 || + bashCommandsMatching(trace, 'teach.md').length > 0; + assert.ok( + teachLoaded, + `expected agent to load teach.md (via Read or bash cat) after context.mjs reported NO_PRODUCT_MD.\n` + + `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, + ); + // We do NOT want it to silently barrel into design work. + const wroteHtml = trace.writePaths.some((p) => /\.(html?|css|svelte|jsx?|tsx?)$/i.test(p)); + assert.equal( + wroteHtml, + false, + `agent should not write implementation files before resolving missing PRODUCT.md.\n` + + `wrote: ${trace.writePaths.join(', ')}`, + ); + } finally { + cleanupWorkspace(workspace); + } + }); + + it('scenario 2: PRODUCT.md only', async () => { + const workspace = prepareWorkspace({ + files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE }, + }); + try { + const { trace, text } = await runTurn({ + workspace, + model, + userPrompt: CRAFT_PROMPT, + maxSteps: 6, + }); + logTrace('S2', 'product-only', modelId, trace, { textSample: text.slice(0, 400) }); + const loadCalls = bashCommandsMatching(trace, 'context.mjs'); + assert.ok( + loadCalls.length >= 1 && loadCalls.length <= 3, + `expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` + + `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`, + ); + // Fixture sets `register: brand`. Step 3 of Setup says load the + // matching register reference. Accept Read or bash cat. + assert.ok( + fileLoaded(trace, 'brand.md'), + `agent should load brand.md (PRODUCT.md register is brand).\n` + + `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, + ); + } finally { + cleanupWorkspace(workspace); + } + }); + + it('scenario 3: PRODUCT.md + DESIGN.md', async () => { + const workspace = prepareWorkspace({ + files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE, 'DESIGN.md': DESIGN_MD_SAMPLE }, + }); + try { + const { trace, text } = await runTurn({ + workspace, + model, + userPrompt: CRAFT_PROMPT, + maxSteps: 6, + }); + logTrace('S3', 'product-and-design', modelId, trace, { textSample: text.slice(0, 400) }); + const loadCalls = bashCommandsMatching(trace, 'context.mjs'); + assert.ok( + loadCalls.length >= 1 && loadCalls.length <= 3, + `expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` + + `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`, + ); + // Register reference: PRODUCT.md fixture is brand, so brand.md + // should be loaded per Setup step 3. + assert.ok( + fileLoaded(trace, 'brand.md'), + `agent should load brand.md (PRODUCT.md register is brand).\n` + + `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, + ); + // The skill tells the agent to also familiarize with the existing + // design system. DESIGN.md is bundled in context.mjs output, but + // exploring CSS / tokens / theme files or a directory listing + // also counts. + const designSignal = + readsMatching(trace, 'design.md').length > 0 || + trace.readPaths.some((p) => /\.(css|scss|less|ts|tsx|js|jsx|json|svelte|astro)$/i.test(p)) || + trace.listPaths.length > 0; + assert.ok( + designSignal, + `agent should consult the design system (DESIGN.md, CSS/tokens, or list project files).\n` + + `readPaths: ${JSON.stringify(trace.readPaths)}, listPaths: ${JSON.stringify(trace.listPaths)}`, + ); + } finally { + cleanupWorkspace(workspace); + } + }); + + it('scenario 4: context already loaded in prior turn', async () => { + const workspace = prepareWorkspace({ + files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE, 'DESIGN.md': DESIGN_MD_SAMPLE }, + }); + try { + // Turn 1: prime the conversation so context.mjs gets run and its + // output enters the message history. + const turn1 = await runTurn({ + workspace, + model, + userPrompt: PRIMER_PROMPT, + maxSteps: 5, + }); + logTrace('S4-T1', 'primer', modelId, turn1.trace, { textSample: turn1.text.slice(0, 200) }); + const turn1Loads = bashCommandsMatching(turn1.trace, 'context.mjs'); + assert.ok( + turn1Loads.length >= 1, + `primer turn should have run context.mjs. bash: ${JSON.stringify(turn1.trace.bashCommands, null, 2)}`, + ); + + // Turn 2: the real ask. The skill says "skip if you've already + // loaded it". Verify the agent honors that. + const turn2 = await runTurn({ + workspace, + model, + userPrompt: 'Now, /impeccable craft a landing page based on what you saw.', + priorMessages: turn1.responseMessages, + maxSteps: 5, + }); + logTrace('S4-T2', 'follow-up', modelId, turn2.trace, { textSample: turn2.text.slice(0, 400) }); + const turn2Loads = bashCommandsMatching(turn2.trace, 'context.mjs'); + assert.equal( + turn2Loads.length, + 0, + `agent re-ran context.mjs on turn 2 despite it being in prior conversation. ` + + `bashCommands: ${JSON.stringify(turn2.trace.bashCommands, null, 2)}`, + ); + // Register reference must land somewhere across the two turns — + // craft work without brand.md (for a brand-register project) means + // Setup step 3 was skipped. + const brandLoadedAcrossTurns = + fileLoaded(turn1.trace, 'brand.md') || fileLoaded(turn2.trace, 'brand.md'); + assert.ok( + brandLoadedAcrossTurns, + `agent should load brand.md across turn 1 or turn 2 (project is brand register).\n` + + `turn 1 readPaths: ${JSON.stringify(turn1.trace.readPaths)}, bash: ${JSON.stringify(turn1.trace.bashCommands)}\n` + + `turn 2 readPaths: ${JSON.stringify(turn2.trace.readPaths)}, bash: ${JSON.stringify(turn2.trace.bashCommands)}`, + ); + } finally { + cleanupWorkspace(workspace); + } + }); + + it('scenario 5: PRODUCT.md WITHOUT register field (cascade via task cue)', async () => { + // PRODUCT.md has no `## Register` section, so context.mjs cannot + // detect the register and emits a generic "pick by cascade" + // directive. The agent must infer brand from the user's task cue + // ("landing page") per SKILL.md's priority list (1) task cue, + // (2) surface in focus, (3) register field. + const workspace = prepareWorkspace({ + files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_NO_REGISTER }, + }); + try { + const { trace, text } = await runTurn({ + workspace, + model, + userPrompt: CRAFT_PROMPT, + maxSteps: 6, + }); + logTrace('S5', 'no-register-field', modelId, trace, { textSample: text.slice(0, 400) }); + const loadCalls = bashCommandsMatching(trace, 'context.mjs'); + assert.ok( + loadCalls.length >= 1, + `expected context.mjs invocation; got ${loadCalls.length}.\n` + + `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`, + ); + // Task cue is "landing page" → brand register → brand.md should load. + assert.ok( + fileLoaded(trace, 'brand.md'), + `agent should load brand.md via task-cue cascade (no register field, "landing page" cue).\n` + + `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, + ); + } finally { + cleanupWorkspace(workspace); + } + }); + }); +}