Files
pbakaus_impeccable/tests/live-e2e/agent.mjs
T
Paul BakausandClaude Opus 4.7 c8de59d81e test(live): add full-cycle E2E framework-fixture suite with pluggable agent
19 fixtures (11 styling/build variants + 4 conditional-render scenarios + 4
meta-frameworks) drive the entire user flow end-to-end: handshake, pick,
configure, Go, cycle, accept, carbonize cleanup. Each fixture installs real
deps, boots the framework dev server, and runs Playwright Chromium against a
deterministic fake agent that produces realistic variants (colocated style
with @scope rules, full data-impeccable-params manifests covering range +
steps + toggle, JSX/HTML/Svelte syntax-aware rendering).

The agent is pluggable via a one-method interface — generateVariants(event) —
so a future LLM-backed agent slots in by implementing the same shape. The
orchestrator handles wrap, file write, accept, and carbonize cleanup
deterministically regardless of which agent is plugged in.

Schema extensions (tests/framework-fixtures/README.md): runtime block adds
preActions / reloadProbe / pickSelector / scheme / ignoreHTTPSErrors so
fixtures can drive conditional UI (modal, tab, route) before pick and verify
the carbonized variant survives a reload.

Static fixture suite filtered to skip dirs without fixture.json so empty
scaffold dirs no longer break discovery. Total: 178 static checks, 19 E2E
full cycles, ~107s wall clock for the E2E suite.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-24 23:37:04 -07:00

497 lines
19 KiB
JavaScript
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Agent module for the live-mode E2E test suite.
*
* Two layers:
*
* 1. `runAgentLoop(opts)` — the deterministic wrapper around the live-mode
* poll/wrap/write/accept protocol. This is identical for fake and real
* agents; only the variant-content production step differs.
*
* 2. `createFakeAgent()` — produces canned variants in the EXACT format
* `source/skills/impeccable/reference/live.md` describes: a colocated
* `<style data-impeccable-css="ID">` block with `@scope ([data-impeccable-variant="N"])`
* rules, a `data-impeccable-params` JSON manifest covering range + steps + toggle
* kinds across the variant set, single top-level element per variant matching
* the original tag.
*
* A future LLM-backed agent slots in by implementing the same VariantAgent
* interface (one method, `generateVariants(event, context)`), so the loop and
* harness stay unchanged.
*/
import fs from 'node:fs/promises';
import path from 'node:path';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
const execFileP = promisify(execFile);
// ---------------------------------------------------------------------------
// Variant-output schema
// ---------------------------------------------------------------------------
/**
* @typedef {Object} ParamSpec
* @property {string} id
* @property {'range' | 'steps' | 'toggle'} kind
* @property {string} label
* @property {*} default
* @property {number=} min
* @property {number=} max
* @property {number=} step
* @property {Array<{value: string, label: string}>=} options
*
* @typedef {Object} VariantSpec
* @property {string} innerHtml Single top-level element matching the
* original's tag (e.g. '<h1 ...>...</h1>').
* @property {ParamSpec[]=} params Optional 0-4 param manifest.
*
* @typedef {Object} GenerateOutput
* @property {string} scopedCss Contents of the <style data-impeccable-css>
* block — `@scope` rules per variant.
* @property {VariantSpec[]} variants
*
* @typedef {Object} VariantAgent
* @property {(event: object, context: object) => Promise<GenerateOutput>} generateVariants
*/
// ---------------------------------------------------------------------------
// Fake agent — canned, format-faithful variants
// ---------------------------------------------------------------------------
/**
* Build a fake agent that produces deterministic variants for an `<h1 class="hero-title">`
* target. The exact CSS values are chosen so the test can later assert them
* via `getComputedStyle` — variant 1 → red, variant 2 → bold, variant 3 → uppercase.
*
* The output mirrors a real agent's write-back faithfully:
* - <style data-impeccable-css="ID"> with @scope rules per variant
* - data-impeccable-params manifest with range + steps + toggle kinds
* - first variant visible (no display:none), rest hidden by the agent caller
* - inner content = single <h1> per variant
*/
export function createFakeAgent() {
return {
/** @type {VariantAgent['generateVariants']} */
async generateVariants(event /*, context */) {
const text = extractText(event.element?.outerHTML) || 'Title';
const cls = 'hero-title';
// Variant 1 — red color, with a `range` param tuning hue lightness.
const variant1 = {
innerHtml: `<h1 class="${cls}">${text}</h1>`,
params: [
{
id: 'lightness',
kind: 'range',
min: 0.3,
max: 0.7,
step: 0.05,
default: 0.5,
label: 'Lightness',
},
],
};
// Variant 2 — bold weight, with a `steps` param for serif/sans/mono.
const variant2 = {
innerHtml: `<h1 class="${cls}">${text}</h1>`,
params: [
{
id: 'face',
kind: 'steps',
default: 'sans',
label: 'Face',
options: [
{ value: 'sans', label: 'Sans' },
{ value: 'serif', label: 'Serif' },
{ value: 'mono', label: 'Mono' },
],
},
],
};
// Variant 3 — uppercase, with a `toggle` param for italic.
const variant3 = {
innerHtml: `<h1 class="${cls}">${text}</h1>`,
params: [
{
id: 'italic',
kind: 'toggle',
default: false,
label: 'Italic',
},
],
};
// Scoped CSS — `@scope ([data-impeccable-variant="N"])` per variant,
// wired against the params declared above.
const scopedCss = [
'@scope ([data-impeccable-variant="1"]) {',
' :scope > h1 {',
' color: oklch(var(--p-lightness, 0.5) 0.25 25);',
' }',
'}',
'@scope ([data-impeccable-variant="2"]) {',
' :scope > h1 { font-weight: 900; }',
' :scope[data-p-face="serif"] > h1 { font-family: ui-serif, serif; }',
' :scope[data-p-face="mono"] > h1 { font-family: ui-monospace, monospace; }',
'}',
'@scope ([data-impeccable-variant="3"]) {',
' :scope > h1 { text-transform: uppercase; letter-spacing: 0.04em; }',
' :scope[data-p-italic] > h1 { font-style: italic; }',
'}',
].join('\n');
return {
scopedCss,
variants: [variant1, variant2, variant3],
};
},
};
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function extractText(outerHTML) {
if (!outerHTML) return null;
const m = outerHTML.match(/>([^<]+)</);
return m ? m[1].trim() : null;
}
function attrEscape(str, { svelte = false } = {}) {
let s = String(str).replace(/&/g, '&amp;').replace(/'/g, '&apos;');
if (svelte) {
// Svelte parses `{` in attribute values as expression starters even
// inside quoted strings — see https://svelte.dev/e/expected_token .
// Escape with HTML numeric entities so the literal characters land in
// the rendered DOM attribute.
s = s.replace(/\{/g, '&#123;').replace(/\}/g, '&#125;');
}
return s;
}
/**
* Translate an HTML snippet to JSX. Currently: class= → className=, optionally
* preserves whitespace + tags. The fake agent writes innerHtml in HTML form;
* the orchestrator translates per the target file's syntax.
*/
function htmlToJsx(html) {
return html.replace(/\bclass=/g, 'className=');
}
/**
* Render the variants block in either HTML or JSX, depending on commentSyntax.
* In JSX:
* - comments use {/* ... */} (already what commentSyntax.open is)
* - <style>{`@scope ... { ... }`}</style> wraps CSS in a template literal so JSX
* doesn't choke on the {} in CSS
* - non-default visible variants use style={{display: 'none'}}
* - inner element class= becomes className=
* - data-impeccable-params stays a single-quoted JSON string (JSX-legal)
*/
function renderVariantsBlock({ sessionId, indent, output, commentSyntax, file }) {
const isJsx = commentSyntax.open === '{/*';
const isSvelte = !!file && file.endsWith('.svelte');
const styleLines = isJsx
? [
indent + ' <style data-impeccable-css="' + sessionId + '">{`',
...output.scopedCss.split('\n').map((l) => indent + ' ' + l),
indent + ' `}</style>',
]
: [
indent + ' <style data-impeccable-css="' + sessionId + '">',
...output.scopedCss.split('\n').map((l) => indent + ' ' + l),
indent + ' </style>',
];
const variantBlocks = output.variants.map((v, i) => {
const idx = i + 1;
const paramsAttr = v.params && v.params.length
? " data-impeccable-params='" + attrEscape(JSON.stringify(v.params), { svelte: isSvelte }) + "'"
: '';
let styleAttr = '';
if (i !== 0) styleAttr = isJsx ? " style={{display: 'none'}}" : ' style="display: none"';
const inner = isJsx ? htmlToJsx(v.innerHtml) : v.innerHtml;
return [
indent + ' ' + commentSyntax.open + ' Variant ' + idx + ' ' + commentSyntax.close,
indent + ' <div data-impeccable-variant="' + idx + '"' + styleAttr + paramsAttr + '>',
indent + ' ' + inner,
indent + ' </div>',
].join('\n');
});
return [...styleLines, ...variantBlocks].join('\n');
}
/**
* Read the wrapped file, find the "insert below this line" marker, splice in
* the rendered variants block, write back.
*/
async function spliceVariantsIntoWrapper({ tmp, wrapInfo, sessionId, output }) {
const filePath = path.join(tmp, wrapInfo.file);
const src = await fs.readFile(filePath, 'utf-8');
const lines = src.split('\n');
// Find the "Variants: insert below this line" comment line — definitive
// marker, robust to any indentation off-by-one. Matches in any comment
// style (HTML / JSX / Astro).
const markerIdx = lines.findIndex((l) =>
l.includes('Variants: insert below this line'),
);
if (markerIdx === -1) {
throw new Error('insert marker not found in ' + wrapInfo.file);
}
const indent = (lines[markerIdx].match(/^\s*/) || [''])[0];
// Indent INSIDE the wrapper is one level shallower (the marker is indented
// 2 spaces relative to the wrapper opening). Remove the 2-space comment
// indent to get the wrapper indent.
const wrapperIndent = indent.replace(/ $/, '');
const block = renderVariantsBlock({
sessionId,
indent: wrapperIndent,
output,
commentSyntax: wrapInfo.commentSyntax,
file: wrapInfo.file,
});
const next = [
...lines.slice(0, markerIdx + 1),
block,
...lines.slice(markerIdx + 1),
];
await fs.writeFile(filePath, next.join('\n'), 'utf-8');
}
// ---------------------------------------------------------------------------
// Poll loop — the "agent" runs this until aborted
// ---------------------------------------------------------------------------
/**
* @param {object} opts
* @param {string} opts.tmp Project tmp dir (cwd for live-* scripts).
* @param {string} opts.scriptsDir Path to the impeccable scripts dir.
* @param {number} opts.port live-server port.
* @param {string} opts.token live-server token.
* @param {VariantAgent} opts.agent
* @param {AbortSignal} opts.signal
* @param {(msg: string) => void} [opts.log]
* @param {object} [opts.wrapTarget] Default target for live-wrap when an
* element comes from the picker without an
* id we can resolve. e.g. {classes:'hero-title', tag:'h1'}.
*/
export async function runAgentLoop({
tmp,
scriptsDir,
port,
token,
agent,
signal,
log = () => {},
wrapTarget = { classes: 'hero-title', tag: 'h1' },
}) {
const base = `http://127.0.0.1:${port}`;
while (!signal.aborted) {
let event;
try {
const res = await fetch(`${base}/poll?token=${token}&timeout=5000`, { signal });
event = await res.json();
} catch (err) {
if (signal.aborted) return;
log('poll error: ' + err.message);
await new Promise((r) => setTimeout(r, 200));
continue;
}
if (event.type === 'timeout') continue;
if (event.type === 'exit') return;
if (event.type === 'prefetch') continue;
if (event.type === 'connected') continue;
if (event.type === 'generate') {
log(`generate id=${event.id} action=${event.action} count=${event.count}`);
try {
// 1. Wrap the original element in the variant scaffold (deterministic CLI)
const wrapInfo = await runWrap({
tmp,
scriptsDir,
id: event.id,
count: event.count,
...wrapTarget,
});
log(`wrapped: ${wrapInfo.file} insertLine=${wrapInfo.insertLine}`);
// 2. Agent generates variant content (LLM-pluggable seam)
const output = await agent.generateVariants(event, { wrapTarget });
if (output.variants.length !== event.count) {
log(`warning: agent returned ${output.variants.length} variants, expected ${event.count}`);
}
// 3. Splice variants block into the wrapper (deterministic fs)
await spliceVariantsIntoWrapper({ tmp, wrapInfo, sessionId: event.id, output });
if (process.env.IMPECCABLE_E2E_DEBUG) {
const post = await fs.readFile(path.join(tmp, wrapInfo.file), 'utf-8');
log(`--- post-splice (variants written) ---\n${post}`);
}
// 4. Tell the server we're done (broadcasts SSE done → browser settles to CYCLING)
await fetch(`${base}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token, type: 'done', id: event.id }),
signal,
});
} catch (err) {
if (signal.aborted) return;
log('generate failed: ' + err.message);
await fetch(`${base}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token, type: 'error', id: event.id, message: err.message }),
signal,
}).catch(() => {});
}
continue;
}
if (event.type === 'accept') {
log(`accept id=${event.id} variantId=${event.variantId}`);
try {
const acceptResult = await runAccept({
tmp,
scriptsDir,
id: event.id,
variant: event.variantId,
paramValues: event.paramValues,
});
// Carbonize cleanup — required after accept per the live skill spec.
// For the fake agent, we perform a faithful but minimal cleanup:
// delete the carbonize block (markers + dead variants + inline <style>
// + param-values comment) and unwrap the temporary variant div around
// the accepted content. A real LLM agent would additionally migrate
// the @scope rules into the project's stylesheet — out of scope for
// a deterministic test.
if (acceptResult.handled === true && acceptResult.carbonize === true && acceptResult.file) {
if (process.env.IMPECCABLE_E2E_DEBUG) {
const post = await fs.readFile(path.join(tmp, acceptResult.file), 'utf-8');
log(`--- post-accept (pre-carbonize) ---\n${post}`);
}
await runCarbonizeCleanup({ tmp, file: acceptResult.file, sessionId: event.id, variant: event.variantId });
log(`carbonize cleanup done on ${acceptResult.file}`);
}
await fetch(`${base}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token, type: 'accept', id: event.id, data: { _acceptResult: acceptResult } }),
signal,
});
} catch (err) {
if (signal.aborted) return;
log('accept failed: ' + err.message);
}
continue;
}
if (event.type === 'discard') {
log(`discard id=${event.id}`);
try {
const discardResult = await runAccept({ tmp, scriptsDir, id: event.id, discard: true });
await fetch(`${base}/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token, type: 'discard', id: event.id, data: { _acceptResult: discardResult } }),
signal,
});
} catch (err) {
if (signal.aborted) return;
log('discard failed: ' + err.message);
}
continue;
}
log(`unhandled event: ${event.type}`);
}
}
async function runWrap({ tmp, scriptsDir, id, count, classes, tag, elementId }) {
const args = [path.join(scriptsDir, 'live-wrap.mjs'), '--id', id, '--count', String(count)];
if (elementId) args.push('--element-id', elementId);
if (classes) args.push('--classes', classes);
if (tag) args.push('--tag', tag);
const { stdout } = await execFileP(process.execPath, args, { cwd: tmp });
const last = stdout.trim().split('\n').filter(Boolean).pop();
return JSON.parse(last);
}
/**
* Apply the post-accept carbonize cleanup to the given file. Mirrors the
* five-step rewrite the live skill expects of the agent:
*
* 1. Locate the carbonize block (bracketed by `impeccable-carbonize-start`
* and `impeccable-carbonize-end`).
* 2. Step 2 ("move CSS into the project stylesheet") is skipped — that
* requires per-project judgment about which file owns these styles.
* The fake agent leaves CSS migration to the LLM-backed agent.
* 3-5. Strip the carbonize block entirely AND unwrap the temporary
* `<div data-impeccable-variant="N" style="display: contents"|...>` wrapper
* that holds the accepted content. The accepted inner element survives.
*/
async function runCarbonizeCleanup({ tmp, file, sessionId /* , variant */ }) {
const filePath = path.join(tmp, file);
let body = await fs.readFile(filePath, 'utf-8');
// 1. Strip the carbonize block. We match either comment style so this
// works for both HTML and JSX targets.
const startRe = new RegExp('[ \\t]*(?:<!--|\\{/\\*)\\s*impeccable-carbonize-start\\s+' + sessionId + '\\s*(?:-->|\\*/\\})\\n');
const endRe = new RegExp('[ \\t]*(?:<!--|\\{/\\*)\\s*impeccable-carbonize-end\\s+' + sessionId + '\\s*(?:-->|\\*/\\})\\n?');
const startMatch = body.match(startRe);
const endMatch = body.match(endRe);
if (startMatch && endMatch && startMatch.index < endMatch.index) {
const startIdx = startMatch.index;
const endIdx = endMatch.index + endMatch[0].length;
body = body.slice(0, startIdx) + body.slice(endIdx);
}
// 2. Unwrap the temporary `<div data-impeccable-variant="N" ...>` placed
// around the accepted content. live-accept emits this wrapper with
// `style="display: contents"` so it doesn't affect layout. We strip the
// wrapper open/close lines and keep what's between.
// Match the opening div (any single line) followed by inner content
// followed by `</div>`, where the open carries data-impeccable-variant
// and is NOT inside a data-impeccable-variants wrapper (the variants
// wrapper has the trailing `s`).
body = body.replace(
/^([ \t]*)<div\b[^>]*\bdata-impeccable-variant="[^"]+"[^>]*>\n([\s\S]*?)\n[ \t]*<\/div>\n/m,
(match, indent, inner) => {
// Re-indent inner content to the wrapper's indent level.
const innerLines = inner.split('\n');
const innerIndent = (innerLines[0].match(/^\s*/) || [''])[0];
const dedented = innerLines.map((l) => {
if (l.startsWith(innerIndent)) return indent + l.slice(innerIndent.length);
return l;
}).join('\n');
return dedented + '\n';
},
);
await fs.writeFile(filePath, body, 'utf-8');
}
async function runAccept({ tmp, scriptsDir, id, variant, discard, paramValues }) {
const args = [path.join(scriptsDir, 'live-accept.mjs'), '--id', id];
if (discard) args.push('--discard');
else args.push('--variant', String(variant));
if (paramValues) args.push('--param-values', JSON.stringify(paramValues));
const { stdout } = await execFileP(process.execPath, args, { cwd: tmp });
const last = stdout.trim().split('\n').filter(Boolean).pop();
return JSON.parse(last);
}