mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-12 14:16:28 +03:00
Add DeepSeek live E2E adapter (#163)
* Add DeepSeek live E2E adapter * Fix DeepSeek live E2E review issues * Harden live-e2e helpers against silent failures - htmlToJsx: match multi-line inline style attributes ([\s\S]*?) - readCliOption: throw when --flag value is missing or another --flag - llm-agent: echo parsed payload (first 500 chars) in schema-error throws * Bind hoisted inline styles to their owning tag normalizeVariantOutput previously hoisted every stripped style attribute onto a selector derived from the variant's first tag, so a style on a nested <span> landed on <h1>. Now walks each opening tag and emits one rule per styled element with a descendant combinator so nested-element styles target the correct node. Also fixes the duplicated multi-line style regex bug (.*?) -> ([\s\S]*?) that survived the previous round. Extracts parseVariantResponse from llm-agent for direct schema-throw testing, and lifts readCliOption into its own module so its new missing-value throws can be unit-tested. Adds tests for: - multi-line style hoisting - nested-element tag binding and per-tag rule emission - astro-global-prefixed selector shape - no-op identity-return path - opts.config short-circuit in createLlmAgent - all four parseVariantResponse schema previews + JSON-parse failure - readCliOption value/throw matrix * Hoist inline styles via data attribute, not tag name Two bugs in normalizeVariantOutput that Bugbot flagged: 1. Hoisted rules like `:scope span` matched every same-tag descendant of the variant wrap, so a style on one of several <span>s leaked onto its siblings. 2. The opening-tag scan used `[^>]*` for attributes, so a literal `>` inside a quoted attribute value (e.g. `aria-label="x > y"`) terminated the match early and the trailing `style="..."` was never seen. stripInlineStylesPerElement now walks each opening tag character by character respecting quoted attribute values, and tags every styled element with `data-impeccable-hoist-id="N"`. Rules select on the attribute so they bind to exactly the one element they came from. The attribute is stripped during carbonize cleanup so it does not survive into the final source. * Harden live E2E variant CSS normalization * Fix Radix tests * Harden live E2E pick clicks
This commit is contained in:
Abdul Wahab
@@ -39,7 +39,7 @@ Tests use Bun’s test runner plus Node’s built-in `--test`. Name tests `*.tes
|
||||
|
||||
For changes to `skill/scripts/live-*.{mjs,js}`, also run `bun run test:live-e2e` (kept out of the default suite because it does real `npm install` per fixture and boots framework dev servers). Scope to one fixture with `IMPECCABLE_E2E_ONLY=<fixture-name>` while iterating; pass `IMPECCABLE_E2E_DEBUG=1` for page-DOM and dev-server-log dumps on failure. Schema and authoring guide for new fixtures live in `tests/framework-fixtures/README.md`.
|
||||
|
||||
Set `IMPECCABLE_E2E_AGENT=llm` to swap the deterministic fake agent for a Claude-backed one (`tests/live-e2e/agents/llm-agent.mjs`, default Haiku 4.5, override via `IMPECCABLE_E2E_LLM_MODEL`). Requires `ANTHROPIC_API_KEY`; tests skip cleanly when it's unset. This path hits the API — use it for verification, not CI.
|
||||
Set `IMPECCABLE_E2E_AGENT=llm` to swap the deterministic fake agent for an API-backed one (`tests/live-e2e/agents/llm-agent.mjs`). Claude Haiku 4.5 is the primary path whenever `ANTHROPIC_API_KEY` is set. DeepSeek V4 Flash is the secondary cheap fallback when only `DEEPSEEK_API_KEY` is set, and can be forced with `IMPECCABLE_E2E_LLM_PROVIDER=deepseek` or `bun run test:live-e2e -- --llm-provider=deepseek`; override either model via `IMPECCABLE_E2E_LLM_MODEL` or `--llm-model=<model>`. Tests skip cleanly when the selected provider key is unset. This path hits the API — use it for verification, not CI.
|
||||
|
||||
## Anti-pattern detection rules
|
||||
|
||||
|
||||
+1
-1
@@ -50,7 +50,7 @@
|
||||
"dev": "npx astro dev",
|
||||
"preview": "bun run build && npx astro preview",
|
||||
"deploy": "bun run build && wrangler pages deploy build/",
|
||||
"test": "bun test tests/build.test.js tests/detect-antipatterns.test.js tests/windows-path-fix.test.js tests/lib/detector-bundle.test.js tests/lib/provider-blocks.test.js tests/lib/transformers/provider-blocks.test.js && node --test tests/critique-storage.test.mjs && node --test tests/detect-antipatterns-fixtures.test.mjs && node --test tests/detect-antipatterns-browser.test.mjs && node --test tests/cleanup-deprecated.test.mjs && node --test tests/impeccable-paths.test.mjs && node --test tests/live-wrap.test.mjs && node --test tests/live-reference.test.mjs && node --test tests/live-accept.test.mjs && node --test tests/live-inject.test.mjs && node --test tests/live-poll.test.mjs && node --test tests/live-server.test.mjs && node --test tests/live-browser-regression.test.mjs && node --test tests/live-session-store.test.mjs && node --test tests/live-browser-session.test.mjs && node --test tests/live-browser-source.test.mjs && node --test tests/live-completion.test.mjs && node --test tests/live-recovery-commands.test.mjs && node --test tests/framework-fixtures.test.mjs",
|
||||
"test": "bun test tests/build.test.js tests/detect-antipatterns.test.js tests/windows-path-fix.test.js tests/lib/detector-bundle.test.js tests/lib/provider-blocks.test.js tests/lib/transformers/provider-blocks.test.js && node --test tests/critique-storage.test.mjs && node --test tests/detect-antipatterns-fixtures.test.mjs && node --test tests/detect-antipatterns-browser.test.mjs && node --test tests/cleanup-deprecated.test.mjs && node --test tests/impeccable-paths.test.mjs && node --test tests/live-wrap.test.mjs && node --test tests/live-reference.test.mjs && node --test tests/live-e2e-agent-output.test.mjs && node --test tests/live-e2e-llm-agent.test.mjs && node --test tests/live-e2e-cli-options.test.mjs && node --test tests/live-accept.test.mjs && node --test tests/live-inject.test.mjs && node --test tests/live-poll.test.mjs && node --test tests/live-server.test.mjs && node --test tests/live-browser-regression.test.mjs && node --test tests/live-session-store.test.mjs && node --test tests/live-browser-session.test.mjs && node --test tests/live-browser-source.test.mjs && node --test tests/live-completion.test.mjs && node --test tests/live-recovery-commands.test.mjs && node --test tests/framework-fixtures.test.mjs",
|
||||
"test:live-e2e": "node --test --test-timeout=600000 tests/live-e2e.test.mjs",
|
||||
"bench:detector": "node scripts/benchmark-detector.mjs",
|
||||
"bench:detector:browser": "node scripts/benchmark-detector.mjs --browser",
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import { htmlToJsx, normalizeVariantOutput } from './live-e2e/agent.mjs';
|
||||
|
||||
describe('live-e2e agent output translation', () => {
|
||||
it('converts HTML class and inline style attributes to JSX syntax', () => {
|
||||
const jsx = htmlToJsx(
|
||||
'<h1 class="hero-title" style="--p-scale:1; font-size:2.25rem; font-weight:700">Title</h1>',
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
jsx,
|
||||
'<h1 className="hero-title" style={{ "--p-scale": "1", fontSize: "2.25rem", fontWeight: "700" }}>Title</h1>',
|
||||
);
|
||||
});
|
||||
|
||||
it('camel-cases vendor-prefixed style properties', () => {
|
||||
const jsx = htmlToJsx(
|
||||
'<h1 class="hero-title" style="-webkit-background-clip:text; background-clip:text; color:transparent">Title</h1>',
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
jsx,
|
||||
'<h1 className="hero-title" style={{ WebkitBackgroundClip: "text", backgroundClip: "text", color: "transparent" }}>Title</h1>',
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps semicolons inside quoted and parenthesized style values', () => {
|
||||
const jsx = htmlToJsx(
|
||||
`<h1 class="hero-title" style='content:"a;b"; background-image:url("foo;bar"); font-size:1rem'>Title</h1>`,
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
jsx,
|
||||
`<h1 className="hero-title" style={{ content: "\\"a;b\\"", backgroundImage: "url(\\"foo;bar\\")", fontSize: "1rem" }}>Title</h1>`,
|
||||
);
|
||||
});
|
||||
|
||||
it('does not rewrite class inside data-class attributes', () => {
|
||||
const jsx = htmlToJsx('<h1 data-class="hero" class="hero-title">Title</h1>');
|
||||
|
||||
assert.equal(jsx, '<h1 data-class="hero" className="hero-title">Title</h1>');
|
||||
});
|
||||
|
||||
it('hoists model inline styles into variant-scoped CSS', () => {
|
||||
const output = normalizeVariantOutput(
|
||||
{
|
||||
scopedCss: '',
|
||||
variants: [
|
||||
{
|
||||
innerHtml: '<h1 class="hero-title" style="color:red; font-weight:700">Title</h1>',
|
||||
},
|
||||
{
|
||||
innerHtml: `<h1 class="hero-title" style='content:"a;b"; background-image:url("foo;bar")'>Title</h1>`,
|
||||
},
|
||||
],
|
||||
},
|
||||
{ styleMode: 'scoped' },
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
output.variants[0].innerHtml,
|
||||
'<h1 data-impeccable-hoist-id="1" class="hero-title">Title</h1>',
|
||||
);
|
||||
assert.equal(
|
||||
output.variants[1].innerHtml,
|
||||
'<h1 data-impeccable-hoist-id="1" class="hero-title">Title</h1>',
|
||||
);
|
||||
assert.match(output.scopedCss, /@scope \(\[data-impeccable-variant="1"\]\)/);
|
||||
assert.match(output.scopedCss, /:scope \[data-impeccable-hoist-id="1"\]\s*\{/);
|
||||
assert.match(output.scopedCss, /color: red;/);
|
||||
assert.match(output.scopedCss, /font-weight: 700;/);
|
||||
assert.match(output.scopedCss, /content: "a;b";/);
|
||||
assert.match(output.scopedCss, /background-image: url\("foo;bar"\);/);
|
||||
});
|
||||
|
||||
it('hoists styles split across multiple lines', () => {
|
||||
const output = normalizeVariantOutput(
|
||||
{
|
||||
scopedCss: '',
|
||||
variants: [
|
||||
{
|
||||
innerHtml: '<h1 class="hero-title" style="\n color: red;\n font-size: 2rem;\n">Title</h1>',
|
||||
},
|
||||
],
|
||||
},
|
||||
{ styleMode: 'scoped' },
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
output.variants[0].innerHtml,
|
||||
'<h1 data-impeccable-hoist-id="1" class="hero-title">Title</h1>',
|
||||
);
|
||||
assert.match(output.scopedCss, /color: red;/);
|
||||
assert.match(output.scopedCss, /font-size: 2rem;/);
|
||||
});
|
||||
|
||||
it('binds hoisted rules to the element the style was on, not the variant root', () => {
|
||||
const output = normalizeVariantOutput(
|
||||
{
|
||||
scopedCss: '',
|
||||
variants: [
|
||||
{
|
||||
innerHtml: '<h1 class="hero-title"><span style="color:red; transform:scale(1.1)">Title</span></h1>',
|
||||
},
|
||||
],
|
||||
},
|
||||
{ styleMode: 'scoped' },
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
output.variants[0].innerHtml,
|
||||
'<h1 class="hero-title"><span data-impeccable-hoist-id="1">Title</span></h1>',
|
||||
);
|
||||
assert.match(output.scopedCss, /:scope \[data-impeccable-hoist-id="1"\]\s*\{/);
|
||||
assert.match(output.scopedCss, /color: red;/);
|
||||
assert.match(output.scopedCss, /transform: scale\(1\.1\);/);
|
||||
});
|
||||
|
||||
it('emits a separate rule per styled element inside one variant', () => {
|
||||
const output = normalizeVariantOutput(
|
||||
{
|
||||
scopedCss: '',
|
||||
variants: [
|
||||
{
|
||||
innerHtml: '<h1 style="color:red"><span style="font-weight:700">Title</span></h1>',
|
||||
},
|
||||
],
|
||||
},
|
||||
{ styleMode: 'scoped' },
|
||||
);
|
||||
|
||||
assert.match(output.scopedCss, /:scope \[data-impeccable-hoist-id="1"\]\s*\{[^}]*color: red;/);
|
||||
assert.match(output.scopedCss, /:scope \[data-impeccable-hoist-id="2"\]\s*\{[^}]*font-weight: 700;/);
|
||||
});
|
||||
|
||||
it('targets only the styled element when same-tag siblings are present', () => {
|
||||
const output = normalizeVariantOutput(
|
||||
{
|
||||
scopedCss: '',
|
||||
variants: [
|
||||
{
|
||||
innerHtml: '<div><span>plain</span><span style="color:red">styled</span></div>',
|
||||
},
|
||||
],
|
||||
},
|
||||
{ styleMode: 'scoped' },
|
||||
);
|
||||
|
||||
const hoistMatches = output.variants[0].innerHtml.match(/data-impeccable-hoist-id=/g) || [];
|
||||
assert.equal(hoistMatches.length, 1, 'only the styled span should carry the hoist attribute');
|
||||
assert.match(
|
||||
output.variants[0].innerHtml,
|
||||
/<span data-impeccable-hoist-id="1">styled<\/span>/,
|
||||
);
|
||||
assert.match(output.variants[0].innerHtml, /<span>plain<\/span>/);
|
||||
assert.match(output.scopedCss, /:scope \[data-impeccable-hoist-id="1"\]\s*\{[^}]*color: red;/);
|
||||
});
|
||||
|
||||
it('handles > inside a quoted attribute value without losing the style', () => {
|
||||
const output = normalizeVariantOutput(
|
||||
{
|
||||
scopedCss: '',
|
||||
variants: [
|
||||
{
|
||||
innerHtml: '<h1 aria-label="x > y" style="color:red">Title</h1>',
|
||||
},
|
||||
],
|
||||
},
|
||||
{ styleMode: 'scoped' },
|
||||
);
|
||||
|
||||
assert.match(output.variants[0].innerHtml, /aria-label="x > y"/);
|
||||
assert.match(output.variants[0].innerHtml, /data-impeccable-hoist-id="1"/);
|
||||
assert.doesNotMatch(output.variants[0].innerHtml, /style=/);
|
||||
assert.match(output.scopedCss, /color: red;/);
|
||||
});
|
||||
|
||||
it('emits the astro-global-prefixed selector shape when styleMode requests it', () => {
|
||||
const output = normalizeVariantOutput(
|
||||
{
|
||||
scopedCss: '',
|
||||
variants: [
|
||||
{ innerHtml: '<h1 style="color:red">Title</h1>' },
|
||||
],
|
||||
},
|
||||
{ styleMode: 'astro-global-prefixed' },
|
||||
);
|
||||
|
||||
assert.match(
|
||||
output.scopedCss,
|
||||
/\[data-impeccable-variant="1"\] \[data-impeccable-hoist-id="1"\] \{/,
|
||||
);
|
||||
assert.doesNotMatch(output.scopedCss, /@scope/);
|
||||
});
|
||||
|
||||
it('fills missing base variant rules when a model emits only param-conditioned CSS', () => {
|
||||
const output = normalizeVariantOutput(
|
||||
{
|
||||
scopedCss: [
|
||||
'@scope ([data-impeccable-variant="1"]) { :scope > h1 { color: blue; } }',
|
||||
'@scope ([data-impeccable-variant="2"][data-p-uppercase]) { :scope > h1 { text-transform: uppercase; } }',
|
||||
].join('\n'),
|
||||
variants: [
|
||||
{ innerHtml: '<h1 class="hero-title">One</h1>' },
|
||||
{ innerHtml: '<h1 class="hero-title">Two</h1>' },
|
||||
],
|
||||
},
|
||||
{ styleMode: 'scoped' },
|
||||
);
|
||||
|
||||
assert.match(output.scopedCss, /@scope \(\[data-impeccable-variant="2"\]\)/);
|
||||
assert.match(output.scopedCss, /--impeccable-variant-ready: 1;/);
|
||||
assert.match(output.scopedCss, /@scope \(\[data-impeccable-variant="2"\]\[data-p-uppercase\]\)/);
|
||||
});
|
||||
|
||||
it('returns the original output untouched when no inline styles are present', () => {
|
||||
const original = {
|
||||
scopedCss: '@scope ([data-impeccable-variant="1"]) { :scope > h1 { color: blue; } }',
|
||||
variants: [{ innerHtml: '<h1 class="hero-title">Title</h1>' }],
|
||||
};
|
||||
const result = normalizeVariantOutput(original, { styleMode: 'scoped' });
|
||||
assert.equal(result, original);
|
||||
});
|
||||
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import { readCliOption } from './live-e2e/cli-options.mjs';
|
||||
|
||||
describe('live-e2e readCliOption', () => {
|
||||
const baseArgv = ['node', 'runner.mjs'];
|
||||
|
||||
it('reads --name=value form', () => {
|
||||
assert.equal(readCliOption([...baseArgv, '--llm-model=foo'], 'llm-model'), 'foo');
|
||||
});
|
||||
|
||||
it('reads --name value form', () => {
|
||||
assert.equal(readCliOption([...baseArgv, '--llm-model', 'foo'], 'llm-model'), 'foo');
|
||||
});
|
||||
|
||||
it('returns undefined when the flag is absent', () => {
|
||||
assert.equal(readCliOption(baseArgv, 'llm-model'), undefined);
|
||||
});
|
||||
|
||||
it('returns the first match when the flag appears more than once', () => {
|
||||
assert.equal(
|
||||
readCliOption([...baseArgv, '--llm-model=first', '--llm-model=second'], 'llm-model'),
|
||||
'first',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when --name appears as the last argument with no value', () => {
|
||||
assert.throws(
|
||||
() => readCliOption([...baseArgv, '--llm-model'], 'llm-model'),
|
||||
/--llm-model requires a value \(received no value\)/,
|
||||
);
|
||||
});
|
||||
|
||||
it('throws when the next argv would consume another flag as the value', () => {
|
||||
assert.throws(
|
||||
() => readCliOption([...baseArgv, '--llm-model', '--llm-provider=deepseek'], 'llm-model'),
|
||||
/--llm-model requires a value \(received "--llm-provider=deepseek"\)/,
|
||||
);
|
||||
});
|
||||
|
||||
it('treats --name= (empty `=`) as an empty-string value, not a throw', () => {
|
||||
assert.equal(readCliOption([...baseArgv, '--llm-model='], 'llm-model'), '');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,188 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { describe, it } from 'node:test';
|
||||
import {
|
||||
createLlmAgent,
|
||||
parseVariantResponse,
|
||||
resolveLlmAgentConfig,
|
||||
} from './live-e2e/agents/llm-agent.mjs';
|
||||
|
||||
describe('live-e2e LLM agent provider config', () => {
|
||||
it('defaults to Anthropic and Claude Haiku when no keys are present', () => {
|
||||
const config = resolveLlmAgentConfig({}, {});
|
||||
|
||||
assert.equal(config.provider, 'anthropic');
|
||||
assert.equal(config.model, 'claude-haiku-4-5');
|
||||
assert.equal(config.requiredEnv, 'ANTHROPIC_API_KEY');
|
||||
assert.equal(config.apiKey, undefined);
|
||||
assert.equal(config.baseURL, undefined);
|
||||
});
|
||||
|
||||
it('prefers Anthropic when both provider keys are present', () => {
|
||||
const config = resolveLlmAgentConfig({}, {
|
||||
ANTHROPIC_API_KEY: 'claude-key',
|
||||
DEEPSEEK_API_KEY: 'deepseek-key',
|
||||
});
|
||||
|
||||
assert.equal(config.provider, 'anthropic');
|
||||
assert.equal(config.model, 'claude-haiku-4-5');
|
||||
assert.equal(config.requiredEnv, 'ANTHROPIC_API_KEY');
|
||||
assert.equal(config.apiKey, 'claude-key');
|
||||
assert.equal(config.baseURL, undefined);
|
||||
});
|
||||
|
||||
it('falls back to DeepSeek V4 Flash when only DEEPSEEK_API_KEY is present', () => {
|
||||
const config = resolveLlmAgentConfig({}, {
|
||||
DEEPSEEK_API_KEY: 'test-key',
|
||||
});
|
||||
|
||||
assert.equal(config.provider, 'deepseek');
|
||||
assert.equal(config.model, 'deepseek-v4-flash');
|
||||
assert.equal(config.requiredEnv, 'DEEPSEEK_API_KEY');
|
||||
assert.equal(config.apiKey, 'test-key');
|
||||
assert.equal(config.baseURL, 'https://api.deepseek.com/anthropic');
|
||||
});
|
||||
|
||||
it('explicitly selects DeepSeek over Anthropic', () => {
|
||||
const config = resolveLlmAgentConfig({}, {
|
||||
IMPECCABLE_E2E_LLM_PROVIDER: 'deepseek',
|
||||
ANTHROPIC_API_KEY: 'claude-key',
|
||||
DEEPSEEK_API_KEY: 'deepseek-key',
|
||||
});
|
||||
|
||||
assert.equal(config.provider, 'deepseek');
|
||||
assert.equal(config.model, 'deepseek-v4-flash');
|
||||
assert.equal(config.requiredEnv, 'DEEPSEEK_API_KEY');
|
||||
assert.equal(config.apiKey, 'deepseek-key');
|
||||
assert.equal(config.baseURL, 'https://api.deepseek.com/anthropic');
|
||||
});
|
||||
|
||||
it('allows explicit model and base URL overrides', () => {
|
||||
const config = resolveLlmAgentConfig(
|
||||
{ model: 'custom-model', baseURL: 'https://example.test/anthropic' },
|
||||
{
|
||||
IMPECCABLE_E2E_LLM_PROVIDER: 'deepseek',
|
||||
IMPECCABLE_E2E_LLM_MODEL: 'ignored-model',
|
||||
DEEPSEEK_API_KEY: 'test-key',
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(config.model, 'custom-model');
|
||||
assert.equal(config.baseURL, 'https://example.test/anthropic');
|
||||
});
|
||||
|
||||
it('allows the DeepSeek API base URL to come from env', () => {
|
||||
const config = resolveLlmAgentConfig({}, {
|
||||
IMPECCABLE_E2E_LLM_PROVIDER: 'deepseek',
|
||||
DEEPSEEK_API_KEY: 'test-key',
|
||||
DEEPSEEK_API_BASE_URL: 'https://proxy.example.test/anthropic',
|
||||
});
|
||||
|
||||
assert.equal(config.baseURL, 'https://proxy.example.test/anthropic');
|
||||
});
|
||||
|
||||
it('rejects unsupported providers', () => {
|
||||
assert.throws(
|
||||
() => resolveLlmAgentConfig({}, { IMPECCABLE_E2E_LLM_PROVIDER: 'other' }),
|
||||
/Unsupported IMPECCABLE_E2E_LLM_PROVIDER: other/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('live-e2e LLM agent createLlmAgent', () => {
|
||||
it('uses an explicit opts.config without re-reading env', async () => {
|
||||
const agent = await createLlmAgent({
|
||||
config: {
|
||||
provider: 'anthropic',
|
||||
model: 'test-model',
|
||||
apiKey: 'test-key',
|
||||
baseURL: undefined,
|
||||
requiredEnv: 'ANTHROPIC_API_KEY',
|
||||
},
|
||||
});
|
||||
assert.ok(agent, 'agent should be returned when config.apiKey is set');
|
||||
assert.equal(typeof agent.generateVariants, 'function');
|
||||
});
|
||||
|
||||
it('returns null when the resolved config has no apiKey', async () => {
|
||||
const agent = await createLlmAgent({
|
||||
config: {
|
||||
provider: 'deepseek',
|
||||
model: 'deepseek-v4-flash',
|
||||
apiKey: undefined,
|
||||
baseURL: 'https://api.deepseek.com/anthropic',
|
||||
requiredEnv: 'DEEPSEEK_API_KEY',
|
||||
},
|
||||
});
|
||||
assert.equal(agent, null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('live-e2e LLM agent parseVariantResponse', () => {
|
||||
const validParsed = {
|
||||
scopedCss: '@scope ([data-impeccable-variant="1"]) {}',
|
||||
variants: [{ innerHtml: '<h1 class="hero-title">Title</h1>' }],
|
||||
};
|
||||
|
||||
it('parses a well-formed response', () => {
|
||||
const parsed = parseVariantResponse(JSON.stringify(validParsed));
|
||||
assert.deepEqual(parsed, validParsed);
|
||||
});
|
||||
|
||||
it('strips a single surrounding ```json fence', () => {
|
||||
const parsed = parseVariantResponse(
|
||||
'```json\n' + JSON.stringify(validParsed) + '\n```',
|
||||
);
|
||||
assert.deepEqual(parsed, validParsed);
|
||||
});
|
||||
|
||||
it('echoes the raw payload (first 500 chars) on JSON-parse failure', () => {
|
||||
assert.throws(
|
||||
() => parseVariantResponse('not valid json {'),
|
||||
(err) => err.message.includes('First 500 chars:') && err.message.includes('not valid json {'),
|
||||
);
|
||||
});
|
||||
|
||||
it('echoes the parsed payload on missing scopedCss', () => {
|
||||
const body = JSON.stringify({ variants: [{ innerHtml: '<h1>x</h1>' }] });
|
||||
assert.throws(
|
||||
() => parseVariantResponse(body),
|
||||
(err) =>
|
||||
/missing or non-string scopedCss/.test(err.message)
|
||||
&& /Parsed \(first 500 chars\):/.test(err.message)
|
||||
&& err.message.includes('"variants"'),
|
||||
);
|
||||
});
|
||||
|
||||
it('echoes the parsed payload on empty variants array', () => {
|
||||
const body = JSON.stringify({ scopedCss: '', variants: [] });
|
||||
assert.throws(
|
||||
() => parseVariantResponse(body),
|
||||
(err) =>
|
||||
/variants must be a non-empty array/.test(err.message)
|
||||
&& /Parsed \(first 500 chars\):/.test(err.message),
|
||||
);
|
||||
});
|
||||
|
||||
it('echoes the parsed payload on empty innerHtml', () => {
|
||||
const body = JSON.stringify({ scopedCss: '', variants: [{ innerHtml: '' }] });
|
||||
assert.throws(
|
||||
() => parseVariantResponse(body),
|
||||
(err) =>
|
||||
/variants\[0\]\.innerHtml missing or empty/.test(err.message)
|
||||
&& /Parsed \(first 500 chars\):/.test(err.message),
|
||||
);
|
||||
});
|
||||
|
||||
it('echoes the parsed payload on non-array params', () => {
|
||||
const body = JSON.stringify({
|
||||
scopedCss: '',
|
||||
variants: [{ innerHtml: '<h1>x</h1>', params: 'not-an-array' }],
|
||||
});
|
||||
assert.throws(
|
||||
() => parseVariantResponse(body),
|
||||
(err) =>
|
||||
/variants\[0\]\.params must be an array/.test(err.message)
|
||||
&& /Parsed \(first 500 chars\):/.test(err.message),
|
||||
);
|
||||
});
|
||||
});
|
||||
+16
-7
@@ -25,7 +25,8 @@ import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { createFakeAgent } from './live-e2e/agent.mjs';
|
||||
import { createLlmAgent } from './live-e2e/agents/llm-agent.mjs';
|
||||
import { createLlmAgent, resolveLlmAgentConfig } from './live-e2e/agents/llm-agent.mjs';
|
||||
import { readCliOption } from './live-e2e/cli-options.mjs';
|
||||
import { bootFixtureSession, FIXTURES_DIR } from './live-e2e/session.mjs';
|
||||
import {
|
||||
clickAccept,
|
||||
@@ -64,6 +65,9 @@ const fixtures = onlyName
|
||||
? allFixtures.filter((f) => f.name === onlyName)
|
||||
: allFixtures;
|
||||
|
||||
const cliLlmProvider = readCliOption(process.argv, 'llm-provider');
|
||||
const cliLlmModel = readCliOption(process.argv, 'llm-model');
|
||||
|
||||
if (fixtures.length === 0) {
|
||||
describe('live-e2e (no runtime fixtures registered)', () => {
|
||||
it('is a no-op', () => assert.ok(true));
|
||||
@@ -103,22 +107,27 @@ for (const { name, fixture } of fixtures) {
|
||||
// the limitation is visible in the run output.
|
||||
const knownLimitation = fixture.runtime.knownLimitation;
|
||||
|
||||
// Pick the agent. `IMPECCABLE_E2E_AGENT=llm` opts into the real Claude
|
||||
// API; everything else uses the deterministic fake. Skip rather than
|
||||
// fail when LLM is requested but no API key is set so default suite
|
||||
// Pick the agent. `IMPECCABLE_E2E_AGENT=llm` opts into Claude first,
|
||||
// with DeepSeek as the secondary fallback/override; everything else
|
||||
// uses the deterministic fake. Skip rather than fail when LLM is
|
||||
// requested but the selected provider key is missing so default suite
|
||||
// runs in unauthenticated environments still pass.
|
||||
const agentMode = process.env.IMPECCABLE_E2E_AGENT || 'fake';
|
||||
let agent;
|
||||
if (agentMode === 'llm') {
|
||||
const llmConfig = resolveLlmAgentConfig({
|
||||
provider: cliLlmProvider,
|
||||
model: cliLlmModel || process.env.IMPECCABLE_E2E_LLM_MODEL,
|
||||
});
|
||||
agent = await createLlmAgent({
|
||||
model: process.env.IMPECCABLE_E2E_LLM_MODEL,
|
||||
config: llmConfig,
|
||||
log: (m) => t.diagnostic('[llm] ' + m),
|
||||
});
|
||||
if (!agent) {
|
||||
t.skip('IMPECCABLE_E2E_AGENT=llm requires ANTHROPIC_API_KEY');
|
||||
t.skip(`IMPECCABLE_E2E_AGENT=llm with provider=${llmConfig.provider} requires ${llmConfig.requiredEnv}`);
|
||||
return;
|
||||
}
|
||||
t.diagnostic(`Using LLM agent (model=${process.env.IMPECCABLE_E2E_LLM_MODEL || 'claude-haiku-4-5'})`);
|
||||
t.diagnostic(`Using LLM agent (provider=${llmConfig.provider} model=${llmConfig.model})`);
|
||||
} else {
|
||||
agent = createFakeAgent();
|
||||
}
|
||||
|
||||
+263
-7
@@ -187,12 +187,260 @@ function attrEscape(str, { svelte = false } = {}) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate an HTML snippet to JSX. Currently: class= → className=, optionally
|
||||
* preserves whitespace + tags. The fake agent writes innerHtml in HTML form;
|
||||
* the orchestrator translates per the target file's syntax.
|
||||
* Translate an HTML snippet to JSX. The fake and LLM agents write innerHtml
|
||||
* in HTML form; the orchestrator translates per the target file's syntax.
|
||||
*/
|
||||
function htmlToJsx(html) {
|
||||
return html.replace(/\bclass=/g, 'className=');
|
||||
export function htmlToJsx(html) {
|
||||
return html
|
||||
.replace(/(^|[\s<])class=/g, '$1className=')
|
||||
.replace(/\sstyle=(["'])([\s\S]*?)\1/g, (_match, _quote, value) => {
|
||||
const entries = parseInlineStyle(value);
|
||||
if (entries.length === 0) return '';
|
||||
return ' style={{ ' + entries.map(({ prop, value }) => `${formatJsxStyleKey(prop)}: ${JSON.stringify(value)}`).join(', ') + ' }}';
|
||||
});
|
||||
}
|
||||
|
||||
function parseInlineStyle(style) {
|
||||
return splitInlineStyleDeclarations(String(style))
|
||||
.map((decl) => decl.trim())
|
||||
.filter(Boolean)
|
||||
.map(parseInlineStyleDeclaration)
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function splitInlineStyleDeclarations(style) {
|
||||
const declarations = [];
|
||||
let quote = null;
|
||||
let escaped = false;
|
||||
let parenDepth = 0;
|
||||
let start = 0;
|
||||
|
||||
for (let i = 0; i < style.length; i++) {
|
||||
const ch = style[i];
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
continue;
|
||||
}
|
||||
if (ch === '\\') {
|
||||
escaped = true;
|
||||
continue;
|
||||
}
|
||||
if (quote) {
|
||||
if (ch === quote) quote = null;
|
||||
continue;
|
||||
}
|
||||
if (ch === '"' || ch === "'") {
|
||||
quote = ch;
|
||||
continue;
|
||||
}
|
||||
if (ch === '(') {
|
||||
parenDepth++;
|
||||
continue;
|
||||
}
|
||||
if (ch === ')' && parenDepth > 0) {
|
||||
parenDepth--;
|
||||
continue;
|
||||
}
|
||||
if (ch === ';' && parenDepth === 0) {
|
||||
declarations.push(style.slice(start, i));
|
||||
start = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
declarations.push(style.slice(start));
|
||||
return declarations;
|
||||
}
|
||||
|
||||
function parseInlineStyleDeclaration(decl) {
|
||||
const colon = decl.indexOf(':');
|
||||
if (colon <= 0) return null;
|
||||
const prop = decl.slice(0, colon).trim();
|
||||
const value = decl.slice(colon + 1).trim();
|
||||
if (!prop || !value) return null;
|
||||
return { prop, value };
|
||||
}
|
||||
|
||||
function formatJsxStyleKey(prop) {
|
||||
if (prop.startsWith('--')) return JSON.stringify(prop);
|
||||
const reactKey = cssPropertyToReactKey(prop);
|
||||
return /^[A-Za-z_$][\w$]*$/.test(reactKey) ? reactKey : JSON.stringify(prop);
|
||||
}
|
||||
|
||||
function cssPropertyToReactKey(prop) {
|
||||
const lower = prop.toLowerCase();
|
||||
if (lower.startsWith('-webkit-')) return 'Webkit' + capitalize(camelCaseCssProperty(lower.slice(8)));
|
||||
if (lower.startsWith('-moz-')) return 'Moz' + capitalize(camelCaseCssProperty(lower.slice(5)));
|
||||
if (lower.startsWith('-o-')) return 'O' + capitalize(camelCaseCssProperty(lower.slice(3)));
|
||||
if (lower.startsWith('-ms-')) return 'ms' + camelCaseCssProperty(lower.slice(4));
|
||||
if (lower === 'float') return 'cssFloat';
|
||||
return camelCaseCssProperty(prop);
|
||||
}
|
||||
|
||||
function camelCaseCssProperty(prop) {
|
||||
return prop.replace(/-([a-z])/gi, (_match, ch) => ch.toUpperCase());
|
||||
}
|
||||
|
||||
function capitalize(str) {
|
||||
return str ? str[0].toUpperCase() + str.slice(1) : str;
|
||||
}
|
||||
|
||||
export const HOIST_ATTR = 'data-impeccable-hoist-id';
|
||||
|
||||
export function normalizeVariantOutput(output, wrapInfo = {}) {
|
||||
const extraCss = [];
|
||||
const variants = output.variants.map((variant, i) => {
|
||||
const { innerHtml, groups } = stripInlineStylesPerElement(String(variant.innerHtml));
|
||||
|
||||
for (const { hoistId, declarations } of groups) {
|
||||
extraCss.push(renderHoistedInlineStyleRule({
|
||||
variantId: i + 1,
|
||||
hoistId,
|
||||
declarations,
|
||||
styleMode: wrapInfo.styleMode,
|
||||
}));
|
||||
}
|
||||
|
||||
return { ...variant, innerHtml };
|
||||
});
|
||||
|
||||
const baseCss = renderMissingBaseVariantRules({
|
||||
scopedCss: output.scopedCss || '',
|
||||
count: output.variants.length,
|
||||
styleMode: wrapInfo.styleMode,
|
||||
});
|
||||
if (extraCss.length === 0 && baseCss.length === 0) return output;
|
||||
const scopedCss = [output.scopedCss || '', ...extraCss, ...baseCss]
|
||||
.map((chunk) => String(chunk).trim())
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
|
||||
return { ...output, scopedCss, variants };
|
||||
}
|
||||
|
||||
function renderMissingBaseVariantRules({ scopedCss, count, styleMode }) {
|
||||
const rules = [];
|
||||
for (let i = 1; i <= count; i++) {
|
||||
if (!hasBaseVariantRule(scopedCss, i, styleMode)) {
|
||||
rules.push(renderBaseVariantRule(i, styleMode));
|
||||
}
|
||||
}
|
||||
return rules;
|
||||
}
|
||||
|
||||
function hasBaseVariantRule(scopedCss, variantId, styleMode) {
|
||||
const q = String.raw`["']${variantId}["']`;
|
||||
if (styleMode === 'astro-global-prefixed') {
|
||||
return new RegExp(String.raw`\[data-impeccable-variant=${q}\](?:\s|>|\.|#|\[${HOIST_ATTR}=)`).test(scopedCss);
|
||||
}
|
||||
return new RegExp(String.raw`@scope\s*\(\s*\[data-impeccable-variant=${q}\]\s*\)`).test(scopedCss);
|
||||
}
|
||||
|
||||
function renderBaseVariantRule(variantId, styleMode) {
|
||||
if (styleMode === 'astro-global-prefixed') {
|
||||
return [
|
||||
`[data-impeccable-variant="${variantId}"] > * {`,
|
||||
' --impeccable-variant-ready: 1;',
|
||||
'}',
|
||||
].join('\n');
|
||||
}
|
||||
return [
|
||||
`@scope ([data-impeccable-variant="${variantId}"]) {`,
|
||||
' :scope > * { --impeccable-variant-ready: 1; }',
|
||||
'}',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
// Walk each opening tag char-by-char (respecting quotes so a literal `>`
|
||||
// inside an attribute value doesn't terminate the tag early), strip any
|
||||
// `style="..."`, and tag the element with `data-impeccable-hoist-id="N"`.
|
||||
// The downstream rule selects on that attribute so it targets the exact
|
||||
// element that was styled — never sibling tags of the same name.
|
||||
function stripInlineStylesPerElement(innerHtml) {
|
||||
const groups = [];
|
||||
const styleRe = /\sstyle=(["'])([\s\S]*?)\1/;
|
||||
let counter = 0;
|
||||
let result = '';
|
||||
let i = 0;
|
||||
|
||||
while (i < innerHtml.length) {
|
||||
const lt = innerHtml.indexOf('<', i);
|
||||
if (lt === -1) {
|
||||
result += innerHtml.slice(i);
|
||||
break;
|
||||
}
|
||||
result += innerHtml.slice(i, lt);
|
||||
|
||||
const tagMatch = innerHtml.slice(lt + 1).match(/^([A-Za-z][\w:-]*)/);
|
||||
if (!tagMatch) {
|
||||
// </tag>, comments, text content — copy `<` and continue.
|
||||
result += '<';
|
||||
i = lt + 1;
|
||||
continue;
|
||||
}
|
||||
const tagName = tagMatch[1];
|
||||
|
||||
let j = lt + 1 + tagName.length;
|
||||
let quote = null;
|
||||
while (j < innerHtml.length) {
|
||||
const ch = innerHtml[j];
|
||||
if (quote) {
|
||||
if (ch === quote) quote = null;
|
||||
} else if (ch === '"' || ch === "'") {
|
||||
quote = ch;
|
||||
} else if (ch === '>') {
|
||||
break;
|
||||
}
|
||||
j++;
|
||||
}
|
||||
if (j >= innerHtml.length) {
|
||||
// Unterminated tag (malformed input): copy verbatim and stop.
|
||||
result += innerHtml.slice(lt);
|
||||
break;
|
||||
}
|
||||
|
||||
const attrs = innerHtml.slice(lt + 1 + tagName.length, j);
|
||||
const styleMatch = attrs.match(styleRe);
|
||||
if (!styleMatch) {
|
||||
result += innerHtml.slice(lt, j + 1);
|
||||
i = j + 1;
|
||||
continue;
|
||||
}
|
||||
const entries = parseInlineStyle(styleMatch[2]);
|
||||
const strippedAttrs = attrs.replace(styleRe, '');
|
||||
if (entries.length === 0) {
|
||||
result += `<${tagName}${strippedAttrs}>`;
|
||||
i = j + 1;
|
||||
continue;
|
||||
}
|
||||
counter++;
|
||||
const hoistId = String(counter);
|
||||
groups.push({ hoistId, declarations: entries });
|
||||
result += `<${tagName} ${HOIST_ATTR}="${hoistId}"${strippedAttrs}>`;
|
||||
i = j + 1;
|
||||
}
|
||||
return { innerHtml: result, groups };
|
||||
}
|
||||
|
||||
function renderHoistedInlineStyleRule({ variantId, hoistId, declarations, styleMode }) {
|
||||
// Select on the per-element hoist attribute, not the tag name, so two
|
||||
// <span>s in the same variant where only one had an inline style cannot
|
||||
// both pick up the hoisted declarations.
|
||||
const lines = declarations.map(({ prop, value }) => ` ${prop}: ${value};`);
|
||||
const target = `[${HOIST_ATTR}="${hoistId}"]`;
|
||||
if (styleMode === 'astro-global-prefixed') {
|
||||
return [
|
||||
`[data-impeccable-variant="${variantId}"] ${target} {`,
|
||||
...lines.map((line) => line.slice(2)),
|
||||
'}',
|
||||
].join('\n');
|
||||
}
|
||||
return [
|
||||
`@scope ([data-impeccable-variant="${variantId}"]) {`,
|
||||
` :scope ${target} {`,
|
||||
...lines,
|
||||
' }',
|
||||
'}',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -202,7 +450,7 @@ function htmlToJsx(html) {
|
||||
* - <style>{`@scope ... { ... }`}</style> wraps CSS in a template literal so JSX
|
||||
* doesn't choke on the {} in CSS
|
||||
* - non-default visible variants use style={{display: 'none'}}
|
||||
* - inner element class= becomes className=
|
||||
* - inner element class= becomes className=, style="..." becomes JSX style={{ ... }}
|
||||
* - data-impeccable-params stays a single-quoted JSON string (JSX-legal)
|
||||
*/
|
||||
function renderVariantsBlock({ sessionId, indent, output, commentSyntax, file, styleMode }) {
|
||||
@@ -353,7 +601,8 @@ export async function runAgentLoop({
|
||||
log(`wrapped: ${wrapInfo.file} insertLine=${wrapInfo.insertLine}`);
|
||||
|
||||
// 2. Agent generates variant content (LLM-pluggable seam)
|
||||
const output = await agent.generateVariants(event, { wrapTarget, wrapInfo });
|
||||
let output = await agent.generateVariants(event, { wrapTarget, wrapInfo });
|
||||
output = normalizeVariantOutput(output, wrapInfo);
|
||||
if (output.variants.length !== event.count) {
|
||||
log(`warning: agent returned ${output.variants.length} variants, expected ${event.count}`);
|
||||
}
|
||||
@@ -508,6 +757,13 @@ async function runCarbonizeCleanup({ tmp, file, sessionId /* , variant */ }) {
|
||||
},
|
||||
);
|
||||
|
||||
// 3. Strip any `data-impeccable-hoist-id` attributes the normalize step
|
||||
// may have injected when the model emitted inline styles. The hoisted
|
||||
// CSS already migrated into the project stylesheet (real agent) or was
|
||||
// dropped with the carbonize block (fake agent); the attribute on the
|
||||
// element is now dead weight.
|
||||
body = body.replace(/\s+data-impeccable-hoist-id="[^"]*"/g, '');
|
||||
|
||||
await fs.writeFile(filePath, body, 'utf-8');
|
||||
}
|
||||
|
||||
|
||||
@@ -7,18 +7,20 @@
|
||||
* and carbonize cleanup deterministically, so this module's only job is
|
||||
* producing variant content for the wrapper.
|
||||
*
|
||||
* Default model: Claude Haiku 4.5 — fast, cheap, smart enough for variant
|
||||
* generation in test fixtures. Override via { model } when constructing,
|
||||
* or via the IMPECCABLE_E2E_LLM_MODEL env var at the call site (test runner).
|
||||
* Primary provider/model: Anthropic + Claude Haiku 4.5. DeepSeek V4 Flash is
|
||||
* a secondary cheap fallback used only when ANTHROPIC_API_KEY is absent and
|
||||
* DEEPSEEK_API_KEY is present, or when explicitly forced with
|
||||
* IMPECCABLE_E2E_LLM_PROVIDER=deepseek. Override the model via { model } when
|
||||
* constructing, or via IMPECCABLE_E2E_LLM_MODEL at the call site.
|
||||
*
|
||||
* Prompt caching: live.md (the live-mode skill spec) is the bulk of the
|
||||
* system prompt and is stable across calls. We mark a cache_control breakpoint
|
||||
* on the last system block so both the JSON-contract instructions and the
|
||||
* spec are cached as one prefix. Subsequent calls in the same run pay only
|
||||
* the cache-read rate (~0.1× input).
|
||||
* the cache-read rate (~0.1× input) when the selected provider honors it.
|
||||
*
|
||||
* Returns null from createLlmAgent() when ANTHROPIC_API_KEY is unset; the
|
||||
* test runner reads that and skips the case rather than failing.
|
||||
* Returns null from createLlmAgent() when the selected provider's API key is
|
||||
* unset; the test runner reads that and skips the case rather than failing.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
@@ -30,7 +32,10 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = path.join(__dirname, '..', '..', '..');
|
||||
const LIVE_MD_PATH = path.join(REPO_ROOT, 'skill', 'reference', 'live.md');
|
||||
|
||||
const DEFAULT_MODEL = 'claude-haiku-4-5';
|
||||
const DEFAULT_ANTHROPIC_MODEL = 'claude-haiku-4-5';
|
||||
// DeepSeek model list: https://api-docs.deepseek.com/api/list-models
|
||||
const DEFAULT_DEEPSEEK_MODEL = 'deepseek-v4-flash';
|
||||
const DEFAULT_DEEPSEEK_API_BASE_URL = 'https://api.deepseek.com/anthropic';
|
||||
|
||||
const SYSTEM_INSTRUCTIONS = [
|
||||
'You are an automated subagent inside Impeccable\'s live-mode test harness.',
|
||||
@@ -60,6 +65,7 @@ const SYSTEM_INSTRUCTIONS = [
|
||||
'- Mix the param kinds across the variant set: include at least one range, one steps, and one toggle when count >= 3.',
|
||||
'- The scopedCss must follow wrapInfo.cssAuthoring exactly: use its selector strategy, rulePattern, requirements, and forbidden patterns.',
|
||||
'- Wire scopedCss rules against the params you emit (CSS vars for range/toggle, attribute selectors for steps/toggle).',
|
||||
'- Put visual styling in scopedCss, not style= attributes inside variant.innerHtml.',
|
||||
'- Use HTML attribute syntax in innerHtml (class=, not className=). The orchestrator translates per file syntax.',
|
||||
'- Do NOT emit the wrapping <div data-impeccable-variant="N">. The orchestrator wraps your content.',
|
||||
'- Do NOT emit the outer <style data-impeccable-css> tag. Only its contents go in scopedCss.',
|
||||
@@ -70,24 +76,61 @@ const SYSTEM_INSTRUCTIONS = [
|
||||
|
||||
/**
|
||||
* @typedef {object} LlmAgentOptions
|
||||
* @property {string=} apiKey Override ANTHROPIC_API_KEY env var.
|
||||
* @property {string=} model Default 'claude-haiku-4-5'. Override to 'claude-sonnet-4-6' if Haiku produces unreliable JSON.
|
||||
* @property {'anthropic' | 'deepseek'=} provider Override IMPECCABLE_E2E_LLM_PROVIDER.
|
||||
* @property {string=} apiKey Override the selected provider's API key env var.
|
||||
* @property {string=} model Override the selected provider's default model.
|
||||
* @property {string=} baseURL Override the provider API base URL.
|
||||
* @property {object=} config Pre-resolved provider config from resolveLlmAgentConfig().
|
||||
* @property {(msg: string) => void=} log Optional logger for debug output.
|
||||
*/
|
||||
|
||||
export function resolveLlmAgentConfig(opts = {}, env = process.env) {
|
||||
const provider = resolveProvider(opts, env);
|
||||
|
||||
if (provider === 'anthropic') {
|
||||
return {
|
||||
provider,
|
||||
model: opts.model || env.IMPECCABLE_E2E_LLM_MODEL || DEFAULT_ANTHROPIC_MODEL,
|
||||
apiKey: opts.apiKey || env.ANTHROPIC_API_KEY,
|
||||
requiredEnv: 'ANTHROPIC_API_KEY',
|
||||
baseURL: opts.baseURL || env.ANTHROPIC_BASE_URL,
|
||||
};
|
||||
}
|
||||
|
||||
if (provider === 'deepseek') {
|
||||
return {
|
||||
provider,
|
||||
model: opts.model || env.IMPECCABLE_E2E_LLM_MODEL || DEFAULT_DEEPSEEK_MODEL,
|
||||
apiKey: opts.apiKey || env.DEEPSEEK_API_KEY,
|
||||
requiredEnv: 'DEEPSEEK_API_KEY',
|
||||
baseURL: opts.baseURL || env.DEEPSEEK_API_BASE_URL || DEFAULT_DEEPSEEK_API_BASE_URL,
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported IMPECCABLE_E2E_LLM_PROVIDER: ${provider}`);
|
||||
}
|
||||
|
||||
function resolveProvider(opts, env) {
|
||||
const explicit = opts.provider || env.IMPECCABLE_E2E_LLM_PROVIDER;
|
||||
if (explicit) return String(explicit).trim().toLowerCase();
|
||||
if (env.ANTHROPIC_API_KEY) return 'anthropic';
|
||||
if (env.DEEPSEEK_API_KEY) return 'deepseek';
|
||||
return 'anthropic';
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {LlmAgentOptions} [opts]
|
||||
* @returns {Promise<{generateVariants: (event: object, context: object) => Promise<{scopedCss: string, variants: object[]}>} | null>}
|
||||
*/
|
||||
export async function createLlmAgent(opts = {}) {
|
||||
const apiKey = opts.apiKey || process.env.ANTHROPIC_API_KEY;
|
||||
if (!apiKey) return null;
|
||||
const config = opts.config || resolveLlmAgentConfig(opts);
|
||||
if (!config.apiKey) return null;
|
||||
|
||||
const model = opts.model || DEFAULT_MODEL;
|
||||
const { apiKey, baseURL, model, provider } = config;
|
||||
const log = opts.log || (() => {});
|
||||
|
||||
const liveMd = await fs.readFile(LIVE_MD_PATH, 'utf-8');
|
||||
const client = new Anthropic({ apiKey });
|
||||
const client = new Anthropic({ apiKey, ...(baseURL ? { baseURL } : {}) });
|
||||
|
||||
return {
|
||||
async generateVariants(event, context = {}) {
|
||||
@@ -125,7 +168,9 @@ export async function createLlmAgent(opts = {}) {
|
||||
{ type: 'text', text: SYSTEM_INSTRUCTIONS },
|
||||
// Cacheable: the entire stable prefix (instructions + spec) is
|
||||
// cached up to this breakpoint. The user message holds all the
|
||||
// per-call volatile content.
|
||||
// per-call volatile content. DeepSeek compatibility support is
|
||||
// provider-reported and best-effort; the usage log below tells us
|
||||
// whether cache reads/writes actually happened.
|
||||
{ type: 'text', text: liveMd, cache_control: { type: 'ephemeral' } },
|
||||
],
|
||||
messages: [{ role: 'user', content: userMessage }],
|
||||
@@ -136,7 +181,7 @@ export async function createLlmAgent(opts = {}) {
|
||||
const inputTokens = response.usage?.input_tokens ?? 0;
|
||||
const outputTokens = response.usage?.output_tokens ?? 0;
|
||||
log(
|
||||
`model=${model} input=${inputTokens} output=${outputTokens} cache_read=${cacheRead} cache_write=${cacheWrite}`,
|
||||
`provider=${provider} model=${model} input=${inputTokens} output=${outputTokens} cache_read=${cacheRead} cache_write=${cacheWrite}`,
|
||||
);
|
||||
|
||||
const text = response.content
|
||||
@@ -144,36 +189,48 @@ export async function createLlmAgent(opts = {}) {
|
||||
.map((b) => b.text)
|
||||
.join('');
|
||||
|
||||
const cleaned = stripCodeFence(text.trim());
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(cleaned);
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`LLM agent: response was not valid JSON (${err.message}). First 500 chars:\n${cleaned.slice(0, 500)}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof parsed.scopedCss !== 'string') {
|
||||
throw new Error(`LLM agent: missing or non-string scopedCss in response`);
|
||||
}
|
||||
if (!Array.isArray(parsed.variants) || parsed.variants.length === 0) {
|
||||
throw new Error(`LLM agent: variants must be a non-empty array`);
|
||||
}
|
||||
for (const [i, v] of parsed.variants.entries()) {
|
||||
if (typeof v.innerHtml !== 'string' || !v.innerHtml.trim()) {
|
||||
throw new Error(`LLM agent: variants[${i}].innerHtml missing or empty`);
|
||||
}
|
||||
if (v.params !== undefined && !Array.isArray(v.params)) {
|
||||
throw new Error(`LLM agent: variants[${i}].params must be an array if present`);
|
||||
}
|
||||
}
|
||||
|
||||
return parsed;
|
||||
return parseVariantResponse(text);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse and validate a model response into the variant-output schema. Throws
|
||||
* with a `Parsed (first 500 chars): ...` echo on every schema failure so the
|
||||
* caller can see what the model actually emitted.
|
||||
*/
|
||||
export function parseVariantResponse(text) {
|
||||
const cleaned = stripCodeFence(String(text).trim());
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(cleaned);
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`LLM agent: response was not valid JSON (${err.message}). First 500 chars:\n${cleaned.slice(0, 500)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const previewParsed = () => {
|
||||
try { return JSON.stringify(parsed).slice(0, 500); }
|
||||
catch { return '[unstringifiable]'; }
|
||||
};
|
||||
if (typeof parsed.scopedCss !== 'string') {
|
||||
throw new Error(`LLM agent: missing or non-string scopedCss in response. Parsed (first 500 chars):\n${previewParsed()}`);
|
||||
}
|
||||
if (!Array.isArray(parsed.variants) || parsed.variants.length === 0) {
|
||||
throw new Error(`LLM agent: variants must be a non-empty array. Parsed (first 500 chars):\n${previewParsed()}`);
|
||||
}
|
||||
for (const [i, v] of parsed.variants.entries()) {
|
||||
if (typeof v.innerHtml !== 'string' || !v.innerHtml.trim()) {
|
||||
throw new Error(`LLM agent: variants[${i}].innerHtml missing or empty. Parsed (first 500 chars):\n${previewParsed()}`);
|
||||
}
|
||||
if (v.params !== undefined && !Array.isArray(v.params)) {
|
||||
throw new Error(`LLM agent: variants[${i}].params must be an array if present. Parsed (first 500 chars):\n${previewParsed()}`);
|
||||
}
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Some models wrap JSON in ```json … ``` fences despite the instruction not to.
|
||||
* Strip a single optional fence, leave anything else alone.
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* Minimal CLI option reader for live-e2e tests. Supports `--name=value` and
|
||||
* `--name value` forms. Throws when `--name` appears without a value or
|
||||
* another flag is about to be consumed as the value, since the wrong-value
|
||||
* failure mode was the main reason this was extracted from the runner.
|
||||
*/
|
||||
export function readCliOption(argv, name) {
|
||||
const prefix = '--' + name + '=';
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i];
|
||||
if (arg.startsWith(prefix)) return arg.slice(prefix.length);
|
||||
if (arg === '--' + name) {
|
||||
const next = argv[i + 1];
|
||||
if (next === undefined || next.startsWith('--')) {
|
||||
throw new Error(
|
||||
`--${name} requires a value (received ${next === undefined ? 'no value' : JSON.stringify(next)}). Use --${name}=<value> or --${name} <value>.`,
|
||||
);
|
||||
}
|
||||
return next;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
+52
-7
@@ -13,6 +13,7 @@
|
||||
const BAR_ID = '#impeccable-live-bar';
|
||||
const GLOBAL_BAR_ID = '#impeccable-live-global-bar';
|
||||
const PICKER_ID = '#impeccable-live-picker';
|
||||
const PICK_TOGGLE_ID = '#impeccable-live-pick-toggle';
|
||||
|
||||
/**
|
||||
* Wait for the live handshake to complete:
|
||||
@@ -43,13 +44,24 @@ export async function waitForHandshake(page, { timeout = 20_000 } = {}) {
|
||||
*/
|
||||
export async function pickElement(page, selector) {
|
||||
const el = await page.waitForSelector(selector, { timeout: 5_000 });
|
||||
await el.hover();
|
||||
// Tiny settle: live-browser updates `hoveredElement` on mousemove, and the
|
||||
// click handler reads from it.
|
||||
await page.waitForTimeout(50);
|
||||
await el.click();
|
||||
// Per-element bar mounts on click → wait for it.
|
||||
await page.waitForSelector(BAR_ID, { state: 'visible', timeout: 5_000 });
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
await ensurePickerActive(page);
|
||||
await el.hover();
|
||||
// Tiny settle: live-browser updates `hoveredElement` on mousemove, and the
|
||||
// click handler reads from it.
|
||||
await page.waitForTimeout(50);
|
||||
await clickPickTarget(page, el);
|
||||
// Per-element bar mounts on click → wait for it. Dialog fixtures can
|
||||
// briefly hide the global live chrome while preActions open a portal, so
|
||||
// retry once after explicitly re-arming picker mode.
|
||||
const visible = await page
|
||||
.waitForSelector(BAR_ID, { state: 'visible', timeout: 5_000 })
|
||||
.then(() => true, () => false);
|
||||
if (visible) break;
|
||||
if (attempt === 1) {
|
||||
await page.waitForSelector(BAR_ID, { state: 'visible', timeout: 1 });
|
||||
}
|
||||
}
|
||||
// Wait specifically for the Configure-row Go button to be in the bar.
|
||||
// pickElement returning before that race-conditions with clickGo on
|
||||
// fixtures whose framework re-renders right after pick (modal open, tab
|
||||
@@ -68,6 +80,39 @@ export async function pickElement(page, selector) {
|
||||
);
|
||||
}
|
||||
|
||||
async function clickPickTarget(page, el) {
|
||||
const box = await el.boundingBox();
|
||||
if (box) {
|
||||
await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
|
||||
return;
|
||||
}
|
||||
await el.evaluate((node) => node.click());
|
||||
}
|
||||
|
||||
async function ensurePickerActive(page) {
|
||||
await page.waitForSelector(GLOBAL_BAR_ID, { timeout: 5_000 });
|
||||
const active = await page
|
||||
.locator(PICK_TOGGLE_ID)
|
||||
.evaluate((el) => el.dataset.active === 'true')
|
||||
.catch(() => false);
|
||||
if (active) return;
|
||||
|
||||
const clicked = await page.evaluate((sel) => {
|
||||
const btn = document.querySelector(sel);
|
||||
if (!btn) return false;
|
||||
btn.click();
|
||||
return true;
|
||||
}, PICK_TOGGLE_ID);
|
||||
if (!clicked) {
|
||||
await page.locator(PICK_TOGGLE_ID).click({ timeout: 5_000 });
|
||||
}
|
||||
await page.waitForFunction(
|
||||
(sel) => document.querySelector(sel)?.dataset.active === 'true',
|
||||
PICK_TOGGLE_ID,
|
||||
{ timeout: 5_000 },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the variant count by clicking the count button (cycles 2 → 3 → 4 → 2).
|
||||
* Default is 3. If the desired count is already showing, this is a no-op.
|
||||
|
||||
Reference in New Issue
Block a user