From 84135db0e6bdd58d22828f7bc8331cae7bde3e7f Mon Sep 17 00:00:00 2001 From: Abdul Wahab <32850166+abdulwahabone@users.noreply.github.com> Date: Sat, 23 May 2026 01:28:36 +0900 Subject: [PATCH] Add DeepSeek live E2E adapter (#163) * Add DeepSeek live E2E adapter * Fix DeepSeek live E2E review issues * Harden live-e2e helpers against silent failures - htmlToJsx: match multi-line inline style attributes ([\s\S]*?) - readCliOption: throw when --flag value is missing or another --flag - llm-agent: echo parsed payload (first 500 chars) in schema-error throws * Bind hoisted inline styles to their owning tag normalizeVariantOutput previously hoisted every stripped style attribute onto a selector derived from the variant's first tag, so a style on a nested landed on

. Now walks each opening tag and emits one rule per styled element with a descendant combinator so nested-element styles target the correct node. Also fixes the duplicated multi-line style regex bug (.*?) -> ([\s\S]*?) that survived the previous round. Extracts parseVariantResponse from llm-agent for direct schema-throw testing, and lifts readCliOption into its own module so its new missing-value throws can be unit-tested. Adds tests for: - multi-line style hoisting - nested-element tag binding and per-tag rule emission - astro-global-prefixed selector shape - no-op identity-return path - opts.config short-circuit in createLlmAgent - all four parseVariantResponse schema previews + JSON-parse failure - readCliOption value/throw matrix * Hoist inline styles via data attribute, not tag name Two bugs in normalizeVariantOutput that Bugbot flagged: 1. Hoisted rules like `:scope span` matched every same-tag descendant of the variant wrap, so a style on one of several s leaked onto its siblings. 2. The opening-tag scan used `[^>]*` for attributes, so a literal `>` inside a quoted attribute value (e.g. `aria-label="x > y"`) terminated the match early and the trailing `style="..."` was never seen. stripInlineStylesPerElement now walks each opening tag character by character respecting quoted attribute values, and tags every styled element with `data-impeccable-hoist-id="N"`. Rules select on the attribute so they bind to exactly the one element they came from. The attribute is stripped during carbonize cleanup so it does not survive into the final source. * Harden live E2E variant CSS normalization * Fix Radix tests * Harden live E2E pick clicks --- AGENTS.md | 2 +- package.json | 2 +- tests/live-e2e-agent-output.test.mjs | 226 ++++++++++++++++++++++ tests/live-e2e-cli-options.test.mjs | 44 +++++ tests/live-e2e-llm-agent.test.mjs | 188 +++++++++++++++++++ tests/live-e2e.test.mjs | 23 ++- tests/live-e2e/agent.mjs | 270 ++++++++++++++++++++++++++- tests/live-e2e/agents/llm-agent.mjs | 139 ++++++++++---- tests/live-e2e/cli-options.mjs | 23 +++ tests/live-e2e/ui.mjs | 59 +++++- 10 files changed, 912 insertions(+), 64 deletions(-) create mode 100644 tests/live-e2e-agent-output.test.mjs create mode 100644 tests/live-e2e-cli-options.test.mjs create mode 100644 tests/live-e2e-llm-agent.test.mjs create mode 100644 tests/live-e2e/cli-options.mjs diff --git a/AGENTS.md b/AGENTS.md index dabda041f..7babd8c76 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ Tests use Bun’s test runner plus Node’s built-in `--test`. Name tests `*.tes For changes to `skill/scripts/live-*.{mjs,js}`, also run `bun run test:live-e2e` (kept out of the default suite because it does real `npm install` per fixture and boots framework dev servers). Scope to one fixture with `IMPECCABLE_E2E_ONLY=` while iterating; pass `IMPECCABLE_E2E_DEBUG=1` for page-DOM and dev-server-log dumps on failure. Schema and authoring guide for new fixtures live in `tests/framework-fixtures/README.md`. -Set `IMPECCABLE_E2E_AGENT=llm` to swap the deterministic fake agent for a Claude-backed one (`tests/live-e2e/agents/llm-agent.mjs`, default Haiku 4.5, override via `IMPECCABLE_E2E_LLM_MODEL`). Requires `ANTHROPIC_API_KEY`; tests skip cleanly when it's unset. This path hits the API — use it for verification, not CI. +Set `IMPECCABLE_E2E_AGENT=llm` to swap the deterministic fake agent for an API-backed one (`tests/live-e2e/agents/llm-agent.mjs`). Claude Haiku 4.5 is the primary path whenever `ANTHROPIC_API_KEY` is set. DeepSeek V4 Flash is the secondary cheap fallback when only `DEEPSEEK_API_KEY` is set, and can be forced with `IMPECCABLE_E2E_LLM_PROVIDER=deepseek` or `bun run test:live-e2e -- --llm-provider=deepseek`; override either model via `IMPECCABLE_E2E_LLM_MODEL` or `--llm-model=`. Tests skip cleanly when the selected provider key is unset. This path hits the API — use it for verification, not CI. ## Anti-pattern detection rules diff --git a/package.json b/package.json index 9067ed3c3..834ab952a 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "dev": "npx astro dev", "preview": "bun run build && npx astro preview", "deploy": "bun run build && wrangler pages deploy build/", - "test": "bun test tests/build.test.js tests/detect-antipatterns.test.js tests/windows-path-fix.test.js tests/lib/detector-bundle.test.js tests/lib/provider-blocks.test.js tests/lib/transformers/provider-blocks.test.js && node --test tests/critique-storage.test.mjs && node --test tests/detect-antipatterns-fixtures.test.mjs && node --test tests/detect-antipatterns-browser.test.mjs && node --test tests/cleanup-deprecated.test.mjs && node --test tests/impeccable-paths.test.mjs && node --test tests/live-wrap.test.mjs && node --test tests/live-reference.test.mjs && node --test tests/live-accept.test.mjs && node --test tests/live-inject.test.mjs && node --test tests/live-poll.test.mjs && node --test tests/live-server.test.mjs && node --test tests/live-browser-regression.test.mjs && node --test tests/live-session-store.test.mjs && node --test tests/live-browser-session.test.mjs && node --test tests/live-browser-source.test.mjs && node --test tests/live-completion.test.mjs && node --test tests/live-recovery-commands.test.mjs && node --test tests/framework-fixtures.test.mjs", + "test": "bun test tests/build.test.js tests/detect-antipatterns.test.js tests/windows-path-fix.test.js tests/lib/detector-bundle.test.js tests/lib/provider-blocks.test.js tests/lib/transformers/provider-blocks.test.js && node --test tests/critique-storage.test.mjs && node --test tests/detect-antipatterns-fixtures.test.mjs && node --test tests/detect-antipatterns-browser.test.mjs && node --test tests/cleanup-deprecated.test.mjs && node --test tests/impeccable-paths.test.mjs && node --test tests/live-wrap.test.mjs && node --test tests/live-reference.test.mjs && node --test tests/live-e2e-agent-output.test.mjs && node --test tests/live-e2e-llm-agent.test.mjs && node --test tests/live-e2e-cli-options.test.mjs && node --test tests/live-accept.test.mjs && node --test tests/live-inject.test.mjs && node --test tests/live-poll.test.mjs && node --test tests/live-server.test.mjs && node --test tests/live-browser-regression.test.mjs && node --test tests/live-session-store.test.mjs && node --test tests/live-browser-session.test.mjs && node --test tests/live-browser-source.test.mjs && node --test tests/live-completion.test.mjs && node --test tests/live-recovery-commands.test.mjs && node --test tests/framework-fixtures.test.mjs", "test:live-e2e": "node --test --test-timeout=600000 tests/live-e2e.test.mjs", "bench:detector": "node scripts/benchmark-detector.mjs", "bench:detector:browser": "node scripts/benchmark-detector.mjs --browser", diff --git a/tests/live-e2e-agent-output.test.mjs b/tests/live-e2e-agent-output.test.mjs new file mode 100644 index 000000000..24c7f52b8 --- /dev/null +++ b/tests/live-e2e-agent-output.test.mjs @@ -0,0 +1,226 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { htmlToJsx, normalizeVariantOutput } from './live-e2e/agent.mjs'; + +describe('live-e2e agent output translation', () => { + it('converts HTML class and inline style attributes to JSX syntax', () => { + const jsx = htmlToJsx( + '

Title

', + ); + + assert.equal( + jsx, + '

Title

', + ); + }); + + it('camel-cases vendor-prefixed style properties', () => { + const jsx = htmlToJsx( + '

Title

', + ); + + assert.equal( + jsx, + '

Title

', + ); + }); + + it('keeps semicolons inside quoted and parenthesized style values', () => { + const jsx = htmlToJsx( + `

Title

`, + ); + + assert.equal( + jsx, + `

Title

`, + ); + }); + + it('does not rewrite class inside data-class attributes', () => { + const jsx = htmlToJsx('

Title

'); + + assert.equal(jsx, '

Title

'); + }); + + it('hoists model inline styles into variant-scoped CSS', () => { + const output = normalizeVariantOutput( + { + scopedCss: '', + variants: [ + { + innerHtml: '

Title

', + }, + { + innerHtml: `

Title

`, + }, + ], + }, + { styleMode: 'scoped' }, + ); + + assert.equal( + output.variants[0].innerHtml, + '

Title

', + ); + assert.equal( + output.variants[1].innerHtml, + '

Title

', + ); + assert.match(output.scopedCss, /@scope \(\[data-impeccable-variant="1"\]\)/); + assert.match(output.scopedCss, /:scope \[data-impeccable-hoist-id="1"\]\s*\{/); + assert.match(output.scopedCss, /color: red;/); + assert.match(output.scopedCss, /font-weight: 700;/); + assert.match(output.scopedCss, /content: "a;b";/); + assert.match(output.scopedCss, /background-image: url\("foo;bar"\);/); + }); + + it('hoists styles split across multiple lines', () => { + const output = normalizeVariantOutput( + { + scopedCss: '', + variants: [ + { + innerHtml: '

Title

', + }, + ], + }, + { styleMode: 'scoped' }, + ); + + assert.equal( + output.variants[0].innerHtml, + '

Title

', + ); + assert.match(output.scopedCss, /color: red;/); + assert.match(output.scopedCss, /font-size: 2rem;/); + }); + + it('binds hoisted rules to the element the style was on, not the variant root', () => { + const output = normalizeVariantOutput( + { + scopedCss: '', + variants: [ + { + innerHtml: '

Title

', + }, + ], + }, + { styleMode: 'scoped' }, + ); + + assert.equal( + output.variants[0].innerHtml, + '

Title

', + ); + assert.match(output.scopedCss, /:scope \[data-impeccable-hoist-id="1"\]\s*\{/); + assert.match(output.scopedCss, /color: red;/); + assert.match(output.scopedCss, /transform: scale\(1\.1\);/); + }); + + it('emits a separate rule per styled element inside one variant', () => { + const output = normalizeVariantOutput( + { + scopedCss: '', + variants: [ + { + innerHtml: '

Title

', + }, + ], + }, + { styleMode: 'scoped' }, + ); + + assert.match(output.scopedCss, /:scope \[data-impeccable-hoist-id="1"\]\s*\{[^}]*color: red;/); + assert.match(output.scopedCss, /:scope \[data-impeccable-hoist-id="2"\]\s*\{[^}]*font-weight: 700;/); + }); + + it('targets only the styled element when same-tag siblings are present', () => { + const output = normalizeVariantOutput( + { + scopedCss: '', + variants: [ + { + innerHtml: '
plainstyled
', + }, + ], + }, + { styleMode: 'scoped' }, + ); + + const hoistMatches = output.variants[0].innerHtml.match(/data-impeccable-hoist-id=/g) || []; + assert.equal(hoistMatches.length, 1, 'only the styled span should carry the hoist attribute'); + assert.match( + output.variants[0].innerHtml, + /styled<\/span>/, + ); + assert.match(output.variants[0].innerHtml, /plain<\/span>/); + assert.match(output.scopedCss, /:scope \[data-impeccable-hoist-id="1"\]\s*\{[^}]*color: red;/); + }); + + it('handles > inside a quoted attribute value without losing the style', () => { + const output = normalizeVariantOutput( + { + scopedCss: '', + variants: [ + { + innerHtml: '

Title

', + }, + ], + }, + { styleMode: 'scoped' }, + ); + + assert.match(output.variants[0].innerHtml, /aria-label="x > y"/); + assert.match(output.variants[0].innerHtml, /data-impeccable-hoist-id="1"/); + assert.doesNotMatch(output.variants[0].innerHtml, /style=/); + assert.match(output.scopedCss, /color: red;/); + }); + + it('emits the astro-global-prefixed selector shape when styleMode requests it', () => { + const output = normalizeVariantOutput( + { + scopedCss: '', + variants: [ + { innerHtml: '

Title

' }, + ], + }, + { styleMode: 'astro-global-prefixed' }, + ); + + assert.match( + output.scopedCss, + /\[data-impeccable-variant="1"\] \[data-impeccable-hoist-id="1"\] \{/, + ); + assert.doesNotMatch(output.scopedCss, /@scope/); + }); + + it('fills missing base variant rules when a model emits only param-conditioned CSS', () => { + const output = normalizeVariantOutput( + { + scopedCss: [ + '@scope ([data-impeccable-variant="1"]) { :scope > h1 { color: blue; } }', + '@scope ([data-impeccable-variant="2"][data-p-uppercase]) { :scope > h1 { text-transform: uppercase; } }', + ].join('\n'), + variants: [ + { innerHtml: '

One

' }, + { innerHtml: '

Two

' }, + ], + }, + { styleMode: 'scoped' }, + ); + + assert.match(output.scopedCss, /@scope \(\[data-impeccable-variant="2"\]\)/); + assert.match(output.scopedCss, /--impeccable-variant-ready: 1;/); + assert.match(output.scopedCss, /@scope \(\[data-impeccable-variant="2"\]\[data-p-uppercase\]\)/); + }); + + it('returns the original output untouched when no inline styles are present', () => { + const original = { + scopedCss: '@scope ([data-impeccable-variant="1"]) { :scope > h1 { color: blue; } }', + variants: [{ innerHtml: '

Title

' }], + }; + const result = normalizeVariantOutput(original, { styleMode: 'scoped' }); + assert.equal(result, original); + }); + +}); diff --git a/tests/live-e2e-cli-options.test.mjs b/tests/live-e2e-cli-options.test.mjs new file mode 100644 index 000000000..31136d6da --- /dev/null +++ b/tests/live-e2e-cli-options.test.mjs @@ -0,0 +1,44 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { readCliOption } from './live-e2e/cli-options.mjs'; + +describe('live-e2e readCliOption', () => { + const baseArgv = ['node', 'runner.mjs']; + + it('reads --name=value form', () => { + assert.equal(readCliOption([...baseArgv, '--llm-model=foo'], 'llm-model'), 'foo'); + }); + + it('reads --name value form', () => { + assert.equal(readCliOption([...baseArgv, '--llm-model', 'foo'], 'llm-model'), 'foo'); + }); + + it('returns undefined when the flag is absent', () => { + assert.equal(readCliOption(baseArgv, 'llm-model'), undefined); + }); + + it('returns the first match when the flag appears more than once', () => { + assert.equal( + readCliOption([...baseArgv, '--llm-model=first', '--llm-model=second'], 'llm-model'), + 'first', + ); + }); + + it('throws when --name appears as the last argument with no value', () => { + assert.throws( + () => readCliOption([...baseArgv, '--llm-model'], 'llm-model'), + /--llm-model requires a value \(received no value\)/, + ); + }); + + it('throws when the next argv would consume another flag as the value', () => { + assert.throws( + () => readCliOption([...baseArgv, '--llm-model', '--llm-provider=deepseek'], 'llm-model'), + /--llm-model requires a value \(received "--llm-provider=deepseek"\)/, + ); + }); + + it('treats --name= (empty `=`) as an empty-string value, not a throw', () => { + assert.equal(readCliOption([...baseArgv, '--llm-model='], 'llm-model'), ''); + }); +}); diff --git a/tests/live-e2e-llm-agent.test.mjs b/tests/live-e2e-llm-agent.test.mjs new file mode 100644 index 000000000..082ec0fe6 --- /dev/null +++ b/tests/live-e2e-llm-agent.test.mjs @@ -0,0 +1,188 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { + createLlmAgent, + parseVariantResponse, + resolveLlmAgentConfig, +} from './live-e2e/agents/llm-agent.mjs'; + +describe('live-e2e LLM agent provider config', () => { + it('defaults to Anthropic and Claude Haiku when no keys are present', () => { + const config = resolveLlmAgentConfig({}, {}); + + assert.equal(config.provider, 'anthropic'); + assert.equal(config.model, 'claude-haiku-4-5'); + assert.equal(config.requiredEnv, 'ANTHROPIC_API_KEY'); + assert.equal(config.apiKey, undefined); + assert.equal(config.baseURL, undefined); + }); + + it('prefers Anthropic when both provider keys are present', () => { + const config = resolveLlmAgentConfig({}, { + ANTHROPIC_API_KEY: 'claude-key', + DEEPSEEK_API_KEY: 'deepseek-key', + }); + + assert.equal(config.provider, 'anthropic'); + assert.equal(config.model, 'claude-haiku-4-5'); + assert.equal(config.requiredEnv, 'ANTHROPIC_API_KEY'); + assert.equal(config.apiKey, 'claude-key'); + assert.equal(config.baseURL, undefined); + }); + + it('falls back to DeepSeek V4 Flash when only DEEPSEEK_API_KEY is present', () => { + const config = resolveLlmAgentConfig({}, { + DEEPSEEK_API_KEY: 'test-key', + }); + + assert.equal(config.provider, 'deepseek'); + assert.equal(config.model, 'deepseek-v4-flash'); + assert.equal(config.requiredEnv, 'DEEPSEEK_API_KEY'); + assert.equal(config.apiKey, 'test-key'); + assert.equal(config.baseURL, 'https://api.deepseek.com/anthropic'); + }); + + it('explicitly selects DeepSeek over Anthropic', () => { + const config = resolveLlmAgentConfig({}, { + IMPECCABLE_E2E_LLM_PROVIDER: 'deepseek', + ANTHROPIC_API_KEY: 'claude-key', + DEEPSEEK_API_KEY: 'deepseek-key', + }); + + assert.equal(config.provider, 'deepseek'); + assert.equal(config.model, 'deepseek-v4-flash'); + assert.equal(config.requiredEnv, 'DEEPSEEK_API_KEY'); + assert.equal(config.apiKey, 'deepseek-key'); + assert.equal(config.baseURL, 'https://api.deepseek.com/anthropic'); + }); + + it('allows explicit model and base URL overrides', () => { + const config = resolveLlmAgentConfig( + { model: 'custom-model', baseURL: 'https://example.test/anthropic' }, + { + IMPECCABLE_E2E_LLM_PROVIDER: 'deepseek', + IMPECCABLE_E2E_LLM_MODEL: 'ignored-model', + DEEPSEEK_API_KEY: 'test-key', + }, + ); + + assert.equal(config.model, 'custom-model'); + assert.equal(config.baseURL, 'https://example.test/anthropic'); + }); + + it('allows the DeepSeek API base URL to come from env', () => { + const config = resolveLlmAgentConfig({}, { + IMPECCABLE_E2E_LLM_PROVIDER: 'deepseek', + DEEPSEEK_API_KEY: 'test-key', + DEEPSEEK_API_BASE_URL: 'https://proxy.example.test/anthropic', + }); + + assert.equal(config.baseURL, 'https://proxy.example.test/anthropic'); + }); + + it('rejects unsupported providers', () => { + assert.throws( + () => resolveLlmAgentConfig({}, { IMPECCABLE_E2E_LLM_PROVIDER: 'other' }), + /Unsupported IMPECCABLE_E2E_LLM_PROVIDER: other/, + ); + }); +}); + +describe('live-e2e LLM agent createLlmAgent', () => { + it('uses an explicit opts.config without re-reading env', async () => { + const agent = await createLlmAgent({ + config: { + provider: 'anthropic', + model: 'test-model', + apiKey: 'test-key', + baseURL: undefined, + requiredEnv: 'ANTHROPIC_API_KEY', + }, + }); + assert.ok(agent, 'agent should be returned when config.apiKey is set'); + assert.equal(typeof agent.generateVariants, 'function'); + }); + + it('returns null when the resolved config has no apiKey', async () => { + const agent = await createLlmAgent({ + config: { + provider: 'deepseek', + model: 'deepseek-v4-flash', + apiKey: undefined, + baseURL: 'https://api.deepseek.com/anthropic', + requiredEnv: 'DEEPSEEK_API_KEY', + }, + }); + assert.equal(agent, null); + }); +}); + +describe('live-e2e LLM agent parseVariantResponse', () => { + const validParsed = { + scopedCss: '@scope ([data-impeccable-variant="1"]) {}', + variants: [{ innerHtml: '

Title

' }], + }; + + it('parses a well-formed response', () => { + const parsed = parseVariantResponse(JSON.stringify(validParsed)); + assert.deepEqual(parsed, validParsed); + }); + + it('strips a single surrounding ```json fence', () => { + const parsed = parseVariantResponse( + '```json\n' + JSON.stringify(validParsed) + '\n```', + ); + assert.deepEqual(parsed, validParsed); + }); + + it('echoes the raw payload (first 500 chars) on JSON-parse failure', () => { + assert.throws( + () => parseVariantResponse('not valid json {'), + (err) => err.message.includes('First 500 chars:') && err.message.includes('not valid json {'), + ); + }); + + it('echoes the parsed payload on missing scopedCss', () => { + const body = JSON.stringify({ variants: [{ innerHtml: '

x

' }] }); + assert.throws( + () => parseVariantResponse(body), + (err) => + /missing or non-string scopedCss/.test(err.message) + && /Parsed \(first 500 chars\):/.test(err.message) + && err.message.includes('"variants"'), + ); + }); + + it('echoes the parsed payload on empty variants array', () => { + const body = JSON.stringify({ scopedCss: '', variants: [] }); + assert.throws( + () => parseVariantResponse(body), + (err) => + /variants must be a non-empty array/.test(err.message) + && /Parsed \(first 500 chars\):/.test(err.message), + ); + }); + + it('echoes the parsed payload on empty innerHtml', () => { + const body = JSON.stringify({ scopedCss: '', variants: [{ innerHtml: '' }] }); + assert.throws( + () => parseVariantResponse(body), + (err) => + /variants\[0\]\.innerHtml missing or empty/.test(err.message) + && /Parsed \(first 500 chars\):/.test(err.message), + ); + }); + + it('echoes the parsed payload on non-array params', () => { + const body = JSON.stringify({ + scopedCss: '', + variants: [{ innerHtml: '

x

', params: 'not-an-array' }], + }); + assert.throws( + () => parseVariantResponse(body), + (err) => + /variants\[0\]\.params must be an array/.test(err.message) + && /Parsed \(first 500 chars\):/.test(err.message), + ); + }); +}); diff --git a/tests/live-e2e.test.mjs b/tests/live-e2e.test.mjs index 8596df514..bfbe7bd59 100644 --- a/tests/live-e2e.test.mjs +++ b/tests/live-e2e.test.mjs @@ -25,7 +25,8 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createFakeAgent } from './live-e2e/agent.mjs'; -import { createLlmAgent } from './live-e2e/agents/llm-agent.mjs'; +import { createLlmAgent, resolveLlmAgentConfig } from './live-e2e/agents/llm-agent.mjs'; +import { readCliOption } from './live-e2e/cli-options.mjs'; import { bootFixtureSession, FIXTURES_DIR } from './live-e2e/session.mjs'; import { clickAccept, @@ -64,6 +65,9 @@ const fixtures = onlyName ? allFixtures.filter((f) => f.name === onlyName) : allFixtures; +const cliLlmProvider = readCliOption(process.argv, 'llm-provider'); +const cliLlmModel = readCliOption(process.argv, 'llm-model'); + if (fixtures.length === 0) { describe('live-e2e (no runtime fixtures registered)', () => { it('is a no-op', () => assert.ok(true)); @@ -103,22 +107,27 @@ for (const { name, fixture } of fixtures) { // the limitation is visible in the run output. const knownLimitation = fixture.runtime.knownLimitation; - // Pick the agent. `IMPECCABLE_E2E_AGENT=llm` opts into the real Claude - // API; everything else uses the deterministic fake. Skip rather than - // fail when LLM is requested but no API key is set so default suite + // Pick the agent. `IMPECCABLE_E2E_AGENT=llm` opts into Claude first, + // with DeepSeek as the secondary fallback/override; everything else + // uses the deterministic fake. Skip rather than fail when LLM is + // requested but the selected provider key is missing so default suite // runs in unauthenticated environments still pass. const agentMode = process.env.IMPECCABLE_E2E_AGENT || 'fake'; let agent; if (agentMode === 'llm') { + const llmConfig = resolveLlmAgentConfig({ + provider: cliLlmProvider, + model: cliLlmModel || process.env.IMPECCABLE_E2E_LLM_MODEL, + }); agent = await createLlmAgent({ - model: process.env.IMPECCABLE_E2E_LLM_MODEL, + config: llmConfig, log: (m) => t.diagnostic('[llm] ' + m), }); if (!agent) { - t.skip('IMPECCABLE_E2E_AGENT=llm requires ANTHROPIC_API_KEY'); + t.skip(`IMPECCABLE_E2E_AGENT=llm with provider=${llmConfig.provider} requires ${llmConfig.requiredEnv}`); return; } - t.diagnostic(`Using LLM agent (model=${process.env.IMPECCABLE_E2E_LLM_MODEL || 'claude-haiku-4-5'})`); + t.diagnostic(`Using LLM agent (provider=${llmConfig.provider} model=${llmConfig.model})`); } else { agent = createFakeAgent(); } diff --git a/tests/live-e2e/agent.mjs b/tests/live-e2e/agent.mjs index 6522c2d69..8cbfed8eb 100644 --- a/tests/live-e2e/agent.mjs +++ b/tests/live-e2e/agent.mjs @@ -187,12 +187,260 @@ function attrEscape(str, { svelte = false } = {}) { } /** - * Translate an HTML snippet to JSX. Currently: class= → className=, optionally - * preserves whitespace + tags. The fake agent writes innerHtml in HTML form; - * the orchestrator translates per the target file's syntax. + * Translate an HTML snippet to JSX. The fake and LLM agents write innerHtml + * in HTML form; the orchestrator translates per the target file's syntax. */ -function htmlToJsx(html) { - return html.replace(/\bclass=/g, 'className='); +export function htmlToJsx(html) { + return html + .replace(/(^|[\s<])class=/g, '$1className=') + .replace(/\sstyle=(["'])([\s\S]*?)\1/g, (_match, _quote, value) => { + const entries = parseInlineStyle(value); + if (entries.length === 0) return ''; + return ' style={{ ' + entries.map(({ prop, value }) => `${formatJsxStyleKey(prop)}: ${JSON.stringify(value)}`).join(', ') + ' }}'; + }); +} + +function parseInlineStyle(style) { + return splitInlineStyleDeclarations(String(style)) + .map((decl) => decl.trim()) + .filter(Boolean) + .map(parseInlineStyleDeclaration) + .filter(Boolean); +} + +function splitInlineStyleDeclarations(style) { + const declarations = []; + let quote = null; + let escaped = false; + let parenDepth = 0; + let start = 0; + + for (let i = 0; i < style.length; i++) { + const ch = style[i]; + if (escaped) { + escaped = false; + continue; + } + if (ch === '\\') { + escaped = true; + continue; + } + if (quote) { + if (ch === quote) quote = null; + continue; + } + if (ch === '"' || ch === "'") { + quote = ch; + continue; + } + if (ch === '(') { + parenDepth++; + continue; + } + if (ch === ')' && parenDepth > 0) { + parenDepth--; + continue; + } + if (ch === ';' && parenDepth === 0) { + declarations.push(style.slice(start, i)); + start = i + 1; + } + } + + declarations.push(style.slice(start)); + return declarations; +} + +function parseInlineStyleDeclaration(decl) { + const colon = decl.indexOf(':'); + if (colon <= 0) return null; + const prop = decl.slice(0, colon).trim(); + const value = decl.slice(colon + 1).trim(); + if (!prop || !value) return null; + return { prop, value }; +} + +function formatJsxStyleKey(prop) { + if (prop.startsWith('--')) return JSON.stringify(prop); + const reactKey = cssPropertyToReactKey(prop); + return /^[A-Za-z_$][\w$]*$/.test(reactKey) ? reactKey : JSON.stringify(prop); +} + +function cssPropertyToReactKey(prop) { + const lower = prop.toLowerCase(); + if (lower.startsWith('-webkit-')) return 'Webkit' + capitalize(camelCaseCssProperty(lower.slice(8))); + if (lower.startsWith('-moz-')) return 'Moz' + capitalize(camelCaseCssProperty(lower.slice(5))); + if (lower.startsWith('-o-')) return 'O' + capitalize(camelCaseCssProperty(lower.slice(3))); + if (lower.startsWith('-ms-')) return 'ms' + camelCaseCssProperty(lower.slice(4)); + if (lower === 'float') return 'cssFloat'; + return camelCaseCssProperty(prop); +} + +function camelCaseCssProperty(prop) { + return prop.replace(/-([a-z])/gi, (_match, ch) => ch.toUpperCase()); +} + +function capitalize(str) { + return str ? str[0].toUpperCase() + str.slice(1) : str; +} + +export const HOIST_ATTR = 'data-impeccable-hoist-id'; + +export function normalizeVariantOutput(output, wrapInfo = {}) { + const extraCss = []; + const variants = output.variants.map((variant, i) => { + const { innerHtml, groups } = stripInlineStylesPerElement(String(variant.innerHtml)); + + for (const { hoistId, declarations } of groups) { + extraCss.push(renderHoistedInlineStyleRule({ + variantId: i + 1, + hoistId, + declarations, + styleMode: wrapInfo.styleMode, + })); + } + + return { ...variant, innerHtml }; + }); + + const baseCss = renderMissingBaseVariantRules({ + scopedCss: output.scopedCss || '', + count: output.variants.length, + styleMode: wrapInfo.styleMode, + }); + if (extraCss.length === 0 && baseCss.length === 0) return output; + const scopedCss = [output.scopedCss || '', ...extraCss, ...baseCss] + .map((chunk) => String(chunk).trim()) + .filter(Boolean) + .join('\n'); + + return { ...output, scopedCss, variants }; +} + +function renderMissingBaseVariantRules({ scopedCss, count, styleMode }) { + const rules = []; + for (let i = 1; i <= count; i++) { + if (!hasBaseVariantRule(scopedCss, i, styleMode)) { + rules.push(renderBaseVariantRule(i, styleMode)); + } + } + return rules; +} + +function hasBaseVariantRule(scopedCss, variantId, styleMode) { + const q = String.raw`["']${variantId}["']`; + if (styleMode === 'astro-global-prefixed') { + return new RegExp(String.raw`\[data-impeccable-variant=${q}\](?:\s|>|\.|#|\[${HOIST_ATTR}=)`).test(scopedCss); + } + return new RegExp(String.raw`@scope\s*\(\s*\[data-impeccable-variant=${q}\]\s*\)`).test(scopedCss); +} + +function renderBaseVariantRule(variantId, styleMode) { + if (styleMode === 'astro-global-prefixed') { + return [ + `[data-impeccable-variant="${variantId}"] > * {`, + ' --impeccable-variant-ready: 1;', + '}', + ].join('\n'); + } + return [ + `@scope ([data-impeccable-variant="${variantId}"]) {`, + ' :scope > * { --impeccable-variant-ready: 1; }', + '}', + ].join('\n'); +} + +// Walk each opening tag char-by-char (respecting quotes so a literal `>` +// inside an attribute value doesn't terminate the tag early), strip any +// `style="..."`, and tag the element with `data-impeccable-hoist-id="N"`. +// The downstream rule selects on that attribute so it targets the exact +// element that was styled — never sibling tags of the same name. +function stripInlineStylesPerElement(innerHtml) { + const groups = []; + const styleRe = /\sstyle=(["'])([\s\S]*?)\1/; + let counter = 0; + let result = ''; + let i = 0; + + while (i < innerHtml.length) { + const lt = innerHtml.indexOf('<', i); + if (lt === -1) { + result += innerHtml.slice(i); + break; + } + result += innerHtml.slice(i, lt); + + const tagMatch = innerHtml.slice(lt + 1).match(/^([A-Za-z][\w:-]*)/); + if (!tagMatch) { + // , comments, text content — copy `<` and continue. + result += '<'; + i = lt + 1; + continue; + } + const tagName = tagMatch[1]; + + let j = lt + 1 + tagName.length; + let quote = null; + while (j < innerHtml.length) { + const ch = innerHtml[j]; + if (quote) { + if (ch === quote) quote = null; + } else if (ch === '"' || ch === "'") { + quote = ch; + } else if (ch === '>') { + break; + } + j++; + } + if (j >= innerHtml.length) { + // Unterminated tag (malformed input): copy verbatim and stop. + result += innerHtml.slice(lt); + break; + } + + const attrs = innerHtml.slice(lt + 1 + tagName.length, j); + const styleMatch = attrs.match(styleRe); + if (!styleMatch) { + result += innerHtml.slice(lt, j + 1); + i = j + 1; + continue; + } + const entries = parseInlineStyle(styleMatch[2]); + const strippedAttrs = attrs.replace(styleRe, ''); + if (entries.length === 0) { + result += `<${tagName}${strippedAttrs}>`; + i = j + 1; + continue; + } + counter++; + const hoistId = String(counter); + groups.push({ hoistId, declarations: entries }); + result += `<${tagName} ${HOIST_ATTR}="${hoistId}"${strippedAttrs}>`; + i = j + 1; + } + return { innerHtml: result, groups }; +} + +function renderHoistedInlineStyleRule({ variantId, hoistId, declarations, styleMode }) { + // Select on the per-element hoist attribute, not the tag name, so two + // s in the same variant where only one had an inline style cannot + // both pick up the hoisted declarations. + const lines = declarations.map(({ prop, value }) => ` ${prop}: ${value};`); + const target = `[${HOIST_ATTR}="${hoistId}"]`; + if (styleMode === 'astro-global-prefixed') { + return [ + `[data-impeccable-variant="${variantId}"] ${target} {`, + ...lines.map((line) => line.slice(2)), + '}', + ].join('\n'); + } + return [ + `@scope ([data-impeccable-variant="${variantId}"]) {`, + ` :scope ${target} {`, + ...lines, + ' }', + '}', + ].join('\n'); } /** @@ -202,7 +450,7 @@ function htmlToJsx(html) { * - wraps CSS in a template literal so JSX * doesn't choke on the {} in CSS * - non-default visible variants use style={{display: 'none'}} - * - inner element class= becomes className= + * - inner element class= becomes className=, style="..." becomes JSX style={{ ... }} * - data-impeccable-params stays a single-quoted JSON string (JSX-legal) */ function renderVariantsBlock({ sessionId, indent, output, commentSyntax, file, styleMode }) { @@ -353,7 +601,8 @@ export async function runAgentLoop({ log(`wrapped: ${wrapInfo.file} insertLine=${wrapInfo.insertLine}`); // 2. Agent generates variant content (LLM-pluggable seam) - const output = await agent.generateVariants(event, { wrapTarget, wrapInfo }); + let output = await agent.generateVariants(event, { wrapTarget, wrapInfo }); + output = normalizeVariantOutput(output, wrapInfo); if (output.variants.length !== event.count) { log(`warning: agent returned ${output.variants.length} variants, expected ${event.count}`); } @@ -508,6 +757,13 @@ async function runCarbonizeCleanup({ tmp, file, sessionId /* , variant */ }) { }, ); + // 3. Strip any `data-impeccable-hoist-id` attributes the normalize step + // may have injected when the model emitted inline styles. The hoisted + // CSS already migrated into the project stylesheet (real agent) or was + // dropped with the carbonize block (fake agent); the attribute on the + // element is now dead weight. + body = body.replace(/\s+data-impeccable-hoist-id="[^"]*"/g, ''); + await fs.writeFile(filePath, body, 'utf-8'); } diff --git a/tests/live-e2e/agents/llm-agent.mjs b/tests/live-e2e/agents/llm-agent.mjs index 5bf186247..34f75d5bc 100644 --- a/tests/live-e2e/agents/llm-agent.mjs +++ b/tests/live-e2e/agents/llm-agent.mjs @@ -7,18 +7,20 @@ * and carbonize cleanup deterministically, so this module's only job is * producing variant content for the wrapper. * - * Default model: Claude Haiku 4.5 — fast, cheap, smart enough for variant - * generation in test fixtures. Override via { model } when constructing, - * or via the IMPECCABLE_E2E_LLM_MODEL env var at the call site (test runner). + * Primary provider/model: Anthropic + Claude Haiku 4.5. DeepSeek V4 Flash is + * a secondary cheap fallback used only when ANTHROPIC_API_KEY is absent and + * DEEPSEEK_API_KEY is present, or when explicitly forced with + * IMPECCABLE_E2E_LLM_PROVIDER=deepseek. Override the model via { model } when + * constructing, or via IMPECCABLE_E2E_LLM_MODEL at the call site. * * Prompt caching: live.md (the live-mode skill spec) is the bulk of the * system prompt and is stable across calls. We mark a cache_control breakpoint * on the last system block so both the JSON-contract instructions and the * spec are cached as one prefix. Subsequent calls in the same run pay only - * the cache-read rate (~0.1× input). + * the cache-read rate (~0.1× input) when the selected provider honors it. * - * Returns null from createLlmAgent() when ANTHROPIC_API_KEY is unset; the - * test runner reads that and skips the case rather than failing. + * Returns null from createLlmAgent() when the selected provider's API key is + * unset; the test runner reads that and skips the case rather than failing. */ import fs from 'node:fs/promises'; @@ -30,7 +32,10 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.join(__dirname, '..', '..', '..'); const LIVE_MD_PATH = path.join(REPO_ROOT, 'skill', 'reference', 'live.md'); -const DEFAULT_MODEL = 'claude-haiku-4-5'; +const DEFAULT_ANTHROPIC_MODEL = 'claude-haiku-4-5'; +// DeepSeek model list: https://api-docs.deepseek.com/api/list-models +const DEFAULT_DEEPSEEK_MODEL = 'deepseek-v4-flash'; +const DEFAULT_DEEPSEEK_API_BASE_URL = 'https://api.deepseek.com/anthropic'; const SYSTEM_INSTRUCTIONS = [ 'You are an automated subagent inside Impeccable\'s live-mode test harness.', @@ -60,6 +65,7 @@ const SYSTEM_INSTRUCTIONS = [ '- Mix the param kinds across the variant set: include at least one range, one steps, and one toggle when count >= 3.', '- The scopedCss must follow wrapInfo.cssAuthoring exactly: use its selector strategy, rulePattern, requirements, and forbidden patterns.', '- Wire scopedCss rules against the params you emit (CSS vars for range/toggle, attribute selectors for steps/toggle).', + '- Put visual styling in scopedCss, not style= attributes inside variant.innerHtml.', '- Use HTML attribute syntax in innerHtml (class=, not className=). The orchestrator translates per file syntax.', '- Do NOT emit the wrapping
. The orchestrator wraps your content.', '- Do NOT emit the outer