diff --git a/AGENTS.md b/AGENTS.md index 0ad547a1b..3e344343c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,8 @@ Tests use Bun’s test runner plus Node’s built-in `--test`. Name tests `*.tes For changes to `source/skills/impeccable/scripts/live-*.{mjs,js}`, also run `bun run test:live-e2e` (kept out of the default suite because it does real `npm install` per fixture and boots framework dev servers). Scope to one fixture with `IMPECCABLE_E2E_ONLY=` while iterating; pass `IMPECCABLE_E2E_DEBUG=1` for page-DOM and dev-server-log dumps on failure. Schema and authoring guide for new fixtures live in `tests/framework-fixtures/README.md`. +Set `IMPECCABLE_E2E_AGENT=llm` to swap the deterministic fake agent for a Claude-backed one (`tests/live-e2e/agents/llm-agent.mjs`, default Haiku 4.5, override via `IMPECCABLE_E2E_LLM_MODEL`). Requires `ANTHROPIC_API_KEY`; tests skip cleanly when it's unset. This path hits the API — use it for verification, not CI. + ## Commit & Pull Request Guidelines Recent history favors short, imperative subjects such as `Fix: ...`, `Add ...`, `Improve ...`, or `Bump ...`. Keep commits focused and explain the user-facing impact when it is not obvious. PRs should summarize what changed, list validation performed, and call out regenerated artifacts like `dist/` or `build/`. Include screenshots for visible `public/` changes and mention affected providers when transform behavior changes. diff --git a/CLAUDE.md b/CLAUDE.md index b0864c043..10d7ce4c1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -126,7 +126,9 @@ IMPECCABLE_E2E_DEBUG=1 bun run test:live-e2e # dump page DOM + de **Kept out of the default `bun run test`** because (a) it does real `npm install` per fixture, (b) it boots framework dev servers, (c) wall time is ~2 minutes, and (d) it requires Playwright's browser cache. Run it locally before shipping changes to anything in `source/skills/impeccable/scripts/live-*.{mjs,js}`. -The agent is pluggable via a one-method interface in `tests/live-e2e/agent.mjs`: `generateVariants(event, context) → { scopedCss, variants[] }`. The default fake agent emits canned variants that exercise all three param kinds (`range`, `steps`, `toggle`). A future LLM-backed agent slots in by implementing the same shape; the orchestrator (wrap, write, accept, carbonize) is agent-agnostic. +The agent is pluggable via a one-method interface in `tests/live-e2e/agent.mjs`: `generateVariants(event, context) → { scopedCss, variants[] }`. The default fake agent emits canned variants that exercise all three param kinds (`range`, `steps`, `toggle`). The orchestrator (wrap, write, accept, carbonize) is agent-agnostic. + +**LLM agent (opt-in)**: set `IMPECCABLE_E2E_AGENT=llm` to swap the fake agent for `tests/live-e2e/agents/llm-agent.mjs`, which calls Claude (default Haiku 4.5) via `@anthropic-ai/sdk`. Requires `ANTHROPIC_API_KEY` in env; the test runner skips with a clear message when it's unset. Override the model with `IMPECCABLE_E2E_LLM_MODEL=claude-sonnet-4-6` if Haiku produces unreliable JSON. Caching is on — live.md is the cacheable prefix, and after the first call subsequent fixtures pay only the cache-read rate. Pass rate on a typical sweep is 18/19; the modal fixture's intrinsic state-loss flake is amplified by LLM latency and may need a re-run. **This path hits the API and costs money** — keep it out of CI unless you really want it there. Adding a new fixture is a matter of cloning a directory under `tests/framework-fixtures/`, swapping the source files, and writing a `fixture.json`. See `tests/framework-fixtures/README.md` for the full schema. diff --git a/package.json b/package.json index 44805b2ed..19c04a5b2 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "@ai-sdk/anthropic": "^3.0.69", "@ai-sdk/openai": "^3.0.53", "@anthropic-ai/claude-agent-sdk": "^0.2.110", + "@anthropic-ai/sdk": "^0.91.1", "@google/genai": "^1.50.1", "ai": "^6.0.162", "archiver": "^7.0.1", diff --git a/tests/live-e2e.test.mjs b/tests/live-e2e.test.mjs index 46edecfe9..5e7103063 100644 --- a/tests/live-e2e.test.mjs +++ b/tests/live-e2e.test.mjs @@ -25,6 +25,7 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createFakeAgent } from './live-e2e/agent.mjs'; +import { createLlmAgent } from './live-e2e/agents/llm-agent.mjs'; import { bootFixtureSession, FIXTURES_DIR } from './live-e2e/session.mjs'; import { clickAccept, @@ -102,12 +103,32 @@ for (const { name, fixture } of fixtures) { // the limitation is visible in the run output. const knownLimitation = fixture.runtime.knownLimitation; + // Pick the agent. `IMPECCABLE_E2E_AGENT=llm` opts into the real Claude + // API; everything else uses the deterministic fake. Skip rather than + // fail when LLM is requested but no API key is set so default suite + // runs in unauthenticated environments still pass. + const agentMode = process.env.IMPECCABLE_E2E_AGENT || 'fake'; + let agent; + if (agentMode === 'llm') { + agent = await createLlmAgent({ + model: process.env.IMPECCABLE_E2E_LLM_MODEL, + log: (m) => t.diagnostic('[llm] ' + m), + }); + if (!agent) { + t.skip('IMPECCABLE_E2E_AGENT=llm requires ANTHROPIC_API_KEY'); + return; + } + t.diagnostic(`Using LLM agent (model=${process.env.IMPECCABLE_E2E_LLM_MODEL || 'claude-haiku-4-5'})`); + } else { + agent = createFakeAgent(); + } + t.diagnostic(`Booting fixture ${name}`); const session = await bootFixtureSession({ name, fixture, browser, - agent: createFakeAgent(), + agent, log: (m) => t.diagnostic(m), }); @@ -147,22 +168,33 @@ for (const { name, fixture } of fixtures) { // For fixtures whose picked element lives inside a conditional // render (modal, tab, route), HMR can remount the parent and lose // the open/active state — the wrapper exists in source but isn't - // in the DOM, so MutationObserver never fires. Live mode now + // in the DOM, so MutationObserver never sees it. Live mode now // surfaces a toast asking the user to retrace the path; we mirror // that here by re-running preActions on the first short timeout. + // + // The first-pass timeout has to be long enough to cover the agent's + // generate latency before declaring "state was lost, retrace." A + // fake agent finishes in <100ms; an LLM agent typically lands in + // 3-8s. Scale the gate accordingly. t.diagnostic(`Waiting for CYCLING state with ${expectedCount} variants`); + const firstPassTimeoutMs = agentMode === 'llm' ? 25_000 : 5_000; let cyclingReached = false; if (fixture.runtime.preActions) { try { - await waitForCycling(page, expectedCount, { timeout: 5_000 }); + await waitForCycling(page, expectedCount, { timeout: firstPassTimeoutMs }); cyclingReached = true; } catch { - t.diagnostic('Cycling not reached in 5s — retracing preActions'); + t.diagnostic(`Cycling not reached in ${firstPassTimeoutMs}ms — retracing preActions`); await runPreActions(page, fixture.runtime.preActions); } } try { - if (!cyclingReached) await waitForCycling(page, expectedCount); + if (!cyclingReached) { + // Default 30s; LLM mode bumps to 60s to absorb API latency on + // top of HMR settle time. + const finalTimeoutMs = agentMode === 'llm' ? 60_000 : 30_000; + await waitForCycling(page, expectedCount, { timeout: finalTimeoutMs }); + } } catch (err) { if (process.env.IMPECCABLE_E2E_DEBUG) { const variantCount = await page.evaluate(() => @@ -196,10 +228,15 @@ for (const { name, fixture } of fixtures) { assert.match(after, /@scope \(\[data-impeccable-variant="1"\]\)/, 'scoped CSS for variant 1'); assert.match(after, /@scope \(\[data-impeccable-variant="2"\]\)/, 'scoped CSS for variant 2'); assert.match(after, /@scope \(\[data-impeccable-variant="3"\]\)/, 'scoped CSS for variant 3'); - assert.match(after, /data-impeccable-params=/, 'data-impeccable-params manifest emitted'); - // Sanity-check the param manifest covers all three kinds across the set. - for (const kind of ['range', 'steps', 'toggle']) { - assert.match(after, new RegExp(`"kind"\\s*:\\s*"${kind}"`), `param kind ${kind} present`); + // Param manifest assertions are scoped to fake-agent mode. The fake + // agent deterministically emits one param per variant covering all + // three kinds; the LLM agent is non-deterministic and may legitimately + // emit no params per the live.md spec ("variants are fixed points"). + if (agentMode === 'fake') { + assert.match(after, /data-impeccable-params=/, 'data-impeccable-params manifest emitted'); + for (const kind of ['range', 'steps', 'toggle']) { + assert.match(after, new RegExp(`"kind"\\s*:\\s*"${kind}"`), `param kind ${kind} present`); + } } // 6. Cycle to variant 2 (the bold one in the fake agent) @@ -222,7 +259,15 @@ for (const { name, fixture } of fixtures) { assert.doesNotMatch(final, /impeccable-carbonize-start/, 'carbonize-start marker removed'); assert.doesNotMatch(final, /impeccable-carbonize-end/, 'carbonize-end marker removed'); assert.doesNotMatch(final, /data-impeccable-variant="/, 'no leftover variant scaffolding'); - assert.match(final, /]*(class|className)="hero-title"/, 'accepted h1 survives'); + // Accept the original class as a substring of the className value so + // an LLM agent that adds classes around the original (e.g. + // class="hero-title bold red") still passes — only the literal + // class="hero-title" form would otherwise match. + assert.match( + final, + /]*(class|className)="[^"]*\bhero-title\b[^"]*"/, + 'accepted h1 survives with hero-title class', + ); // 9. DOM-side: at least one matching element, none inside any wrapper. await page.waitForFunction( diff --git a/tests/live-e2e/agents/llm-agent.mjs b/tests/live-e2e/agents/llm-agent.mjs new file mode 100644 index 000000000..97cd93873 --- /dev/null +++ b/tests/live-e2e/agents/llm-agent.mjs @@ -0,0 +1,186 @@ +/** + * LLM-backed VariantAgent for the live-mode E2E suite. + * + * Implements the same one-method interface as createFakeAgent() in + * tests/live-e2e/agent.mjs: generateVariants(event, context) returns + * { scopedCss, variants[] }. The orchestrator handles wrap, write, accept, + * and carbonize cleanup deterministically, so this module's only job is + * producing variant content for the wrapper. + * + * Default model: Claude Haiku 4.5 — fast, cheap, smart enough for variant + * generation in test fixtures. Override via { model } when constructing, + * or via the IMPECCABLE_E2E_LLM_MODEL env var at the call site (test runner). + * + * Prompt caching: live.md (the live-mode skill spec) is the bulk of the + * system prompt and is stable across calls. We mark a cache_control breakpoint + * on the last system block so both the JSON-contract instructions and the + * spec are cached as one prefix. Subsequent calls in the same run pay only + * the cache-read rate (~0.1× input). + * + * Returns null from createLlmAgent() when ANTHROPIC_API_KEY is unset; the + * test runner reads that and skips the case rather than failing. + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Anthropic from '@anthropic-ai/sdk'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.join(__dirname, '..', '..', '..'); +const LIVE_MD_PATH = path.join( + REPO_ROOT, + 'source', + 'skills', + 'impeccable', + 'reference', + 'live.md', +); + +const DEFAULT_MODEL = 'claude-haiku-4-5'; + +const SYSTEM_INSTRUCTIONS = [ + 'You are an automated subagent inside Impeccable\'s live-mode test harness.', + 'Given an element the user picked, an action, and a count, you produce variant DOM content in a strict JSON shape.', + '', + 'OUTPUT CONTRACT — return ONLY a JSON object with this exact shape. No prose, no code fences, no commentary:', + '', + '{', + ' "scopedCss": "string — contents of a