mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-19 01:26:29 +03:00
test(live): add full-cycle E2E framework-fixture suite with pluggable agent
19 fixtures (11 styling/build variants + 4 conditional-render scenarios + 4 meta-frameworks) drive the entire user flow end-to-end: handshake, pick, configure, Go, cycle, accept, carbonize cleanup. Each fixture installs real deps, boots the framework dev server, and runs Playwright Chromium against a deterministic fake agent that produces realistic variants (colocated style with @scope rules, full data-impeccable-params manifests covering range + steps + toggle, JSX/HTML/Svelte syntax-aware rendering). The agent is pluggable via a one-method interface — generateVariants(event) — so a future LLM-backed agent slots in by implementing the same shape. The orchestrator handles wrap, file write, accept, and carbonize cleanup deterministically regardless of which agent is plugged in. Schema extensions (tests/framework-fixtures/README.md): runtime block adds preActions / reloadProbe / pickSelector / scheme / ignoreHTTPSErrors so fixtures can drive conditional UI (modal, tab, route) before pick and verify the carbonized variant survives a reload. Static fixture suite filtered to skip dirs without fixture.json so empty scaffold dirs no longer break discovery. Total: 178 static checks, 19 E2E full cycles, ~107s wall clock for the E2E suite. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Bakaus
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
d29a690797
commit
c8de59d81e
@@ -0,0 +1,496 @@
|
||||
/**
|
||||
* Agent module for the live-mode E2E test suite.
|
||||
*
|
||||
* Two layers:
|
||||
*
|
||||
* 1. `runAgentLoop(opts)` — the deterministic wrapper around the live-mode
|
||||
* poll/wrap/write/accept protocol. This is identical for fake and real
|
||||
* agents; only the variant-content production step differs.
|
||||
*
|
||||
* 2. `createFakeAgent()` — produces canned variants in the EXACT format
|
||||
* `source/skills/impeccable/reference/live.md` describes: a colocated
|
||||
* `<style data-impeccable-css="ID">` block with `@scope ([data-impeccable-variant="N"])`
|
||||
* rules, a `data-impeccable-params` JSON manifest covering range + steps + toggle
|
||||
* kinds across the variant set, single top-level element per variant matching
|
||||
* the original tag.
|
||||
*
|
||||
* A future LLM-backed agent slots in by implementing the same VariantAgent
|
||||
* interface (one method, `generateVariants(event, context)`), so the loop and
|
||||
* harness stay unchanged.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileP = promisify(execFile);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Variant-output schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @typedef {Object} ParamSpec
|
||||
* @property {string} id
|
||||
* @property {'range' | 'steps' | 'toggle'} kind
|
||||
* @property {string} label
|
||||
* @property {*} default
|
||||
* @property {number=} min
|
||||
* @property {number=} max
|
||||
* @property {number=} step
|
||||
* @property {Array<{value: string, label: string}>=} options
|
||||
*
|
||||
* @typedef {Object} VariantSpec
|
||||
* @property {string} innerHtml Single top-level element matching the
|
||||
* original's tag (e.g. '<h1 ...>...</h1>').
|
||||
* @property {ParamSpec[]=} params Optional 0-4 param manifest.
|
||||
*
|
||||
* @typedef {Object} GenerateOutput
|
||||
* @property {string} scopedCss Contents of the <style data-impeccable-css>
|
||||
* block — `@scope` rules per variant.
|
||||
* @property {VariantSpec[]} variants
|
||||
*
|
||||
* @typedef {Object} VariantAgent
|
||||
* @property {(event: object, context: object) => Promise<GenerateOutput>} generateVariants
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fake agent — canned, format-faithful variants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Build a fake agent that produces deterministic variants for an `<h1 class="hero-title">`
|
||||
* target. The exact CSS values are chosen so the test can later assert them
|
||||
* via `getComputedStyle` — variant 1 → red, variant 2 → bold, variant 3 → uppercase.
|
||||
*
|
||||
* The output mirrors a real agent's write-back faithfully:
|
||||
* - <style data-impeccable-css="ID"> with @scope rules per variant
|
||||
* - data-impeccable-params manifest with range + steps + toggle kinds
|
||||
* - first variant visible (no display:none), rest hidden by the agent caller
|
||||
* - inner content = single <h1> per variant
|
||||
*/
|
||||
export function createFakeAgent() {
|
||||
return {
|
||||
/** @type {VariantAgent['generateVariants']} */
|
||||
async generateVariants(event /*, context */) {
|
||||
const text = extractText(event.element?.outerHTML) || 'Title';
|
||||
const cls = 'hero-title';
|
||||
|
||||
// Variant 1 — red color, with a `range` param tuning hue lightness.
|
||||
const variant1 = {
|
||||
innerHtml: `<h1 class="${cls}">${text}</h1>`,
|
||||
params: [
|
||||
{
|
||||
id: 'lightness',
|
||||
kind: 'range',
|
||||
min: 0.3,
|
||||
max: 0.7,
|
||||
step: 0.05,
|
||||
default: 0.5,
|
||||
label: 'Lightness',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Variant 2 — bold weight, with a `steps` param for serif/sans/mono.
|
||||
const variant2 = {
|
||||
innerHtml: `<h1 class="${cls}">${text}</h1>`,
|
||||
params: [
|
||||
{
|
||||
id: 'face',
|
||||
kind: 'steps',
|
||||
default: 'sans',
|
||||
label: 'Face',
|
||||
options: [
|
||||
{ value: 'sans', label: 'Sans' },
|
||||
{ value: 'serif', label: 'Serif' },
|
||||
{ value: 'mono', label: 'Mono' },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Variant 3 — uppercase, with a `toggle` param for italic.
|
||||
const variant3 = {
|
||||
innerHtml: `<h1 class="${cls}">${text}</h1>`,
|
||||
params: [
|
||||
{
|
||||
id: 'italic',
|
||||
kind: 'toggle',
|
||||
default: false,
|
||||
label: 'Italic',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Scoped CSS — `@scope ([data-impeccable-variant="N"])` per variant,
|
||||
// wired against the params declared above.
|
||||
const scopedCss = [
|
||||
'@scope ([data-impeccable-variant="1"]) {',
|
||||
' :scope > h1 {',
|
||||
' color: oklch(var(--p-lightness, 0.5) 0.25 25);',
|
||||
' }',
|
||||
'}',
|
||||
'@scope ([data-impeccable-variant="2"]) {',
|
||||
' :scope > h1 { font-weight: 900; }',
|
||||
' :scope[data-p-face="serif"] > h1 { font-family: ui-serif, serif; }',
|
||||
' :scope[data-p-face="mono"] > h1 { font-family: ui-monospace, monospace; }',
|
||||
'}',
|
||||
'@scope ([data-impeccable-variant="3"]) {',
|
||||
' :scope > h1 { text-transform: uppercase; letter-spacing: 0.04em; }',
|
||||
' :scope[data-p-italic] > h1 { font-style: italic; }',
|
||||
'}',
|
||||
].join('\n');
|
||||
|
||||
return {
|
||||
scopedCss,
|
||||
variants: [variant1, variant2, variant3],
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function extractText(outerHTML) {
|
||||
if (!outerHTML) return null;
|
||||
const m = outerHTML.match(/>([^<]+)</);
|
||||
return m ? m[1].trim() : null;
|
||||
}
|
||||
|
||||
function attrEscape(str, { svelte = false } = {}) {
|
||||
let s = String(str).replace(/&/g, '&').replace(/'/g, ''');
|
||||
if (svelte) {
|
||||
// Svelte parses `{` in attribute values as expression starters even
|
||||
// inside quoted strings — see https://svelte.dev/e/expected_token .
|
||||
// Escape with HTML numeric entities so the literal characters land in
|
||||
// the rendered DOM attribute.
|
||||
s = s.replace(/\{/g, '{').replace(/\}/g, '}');
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate an HTML snippet to JSX. Currently: class= → className=, optionally
|
||||
* preserves whitespace + tags. The fake agent writes innerHtml in HTML form;
|
||||
* the orchestrator translates per the target file's syntax.
|
||||
*/
|
||||
function htmlToJsx(html) {
|
||||
return html.replace(/\bclass=/g, 'className=');
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the variants block in either HTML or JSX, depending on commentSyntax.
|
||||
* In JSX:
|
||||
* - comments use {/* ... */} (already what commentSyntax.open is)
|
||||
* - <style>{`@scope ... { ... }`}</style> wraps CSS in a template literal so JSX
|
||||
* doesn't choke on the {} in CSS
|
||||
* - non-default visible variants use style={{display: 'none'}}
|
||||
* - inner element class= becomes className=
|
||||
* - data-impeccable-params stays a single-quoted JSON string (JSX-legal)
|
||||
*/
|
||||
function renderVariantsBlock({ sessionId, indent, output, commentSyntax, file }) {
|
||||
const isJsx = commentSyntax.open === '{/*';
|
||||
const isSvelte = !!file && file.endsWith('.svelte');
|
||||
|
||||
const styleLines = isJsx
|
||||
? [
|
||||
indent + ' <style data-impeccable-css="' + sessionId + '">{`',
|
||||
...output.scopedCss.split('\n').map((l) => indent + ' ' + l),
|
||||
indent + ' `}</style>',
|
||||
]
|
||||
: [
|
||||
indent + ' <style data-impeccable-css="' + sessionId + '">',
|
||||
...output.scopedCss.split('\n').map((l) => indent + ' ' + l),
|
||||
indent + ' </style>',
|
||||
];
|
||||
|
||||
const variantBlocks = output.variants.map((v, i) => {
|
||||
const idx = i + 1;
|
||||
const paramsAttr = v.params && v.params.length
|
||||
? " data-impeccable-params='" + attrEscape(JSON.stringify(v.params), { svelte: isSvelte }) + "'"
|
||||
: '';
|
||||
let styleAttr = '';
|
||||
if (i !== 0) styleAttr = isJsx ? " style={{display: 'none'}}" : ' style="display: none"';
|
||||
const inner = isJsx ? htmlToJsx(v.innerHtml) : v.innerHtml;
|
||||
return [
|
||||
indent + ' ' + commentSyntax.open + ' Variant ' + idx + ' ' + commentSyntax.close,
|
||||
indent + ' <div data-impeccable-variant="' + idx + '"' + styleAttr + paramsAttr + '>',
|
||||
indent + ' ' + inner,
|
||||
indent + ' </div>',
|
||||
].join('\n');
|
||||
});
|
||||
|
||||
return [...styleLines, ...variantBlocks].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the wrapped file, find the "insert below this line" marker, splice in
|
||||
* the rendered variants block, write back.
|
||||
*/
|
||||
async function spliceVariantsIntoWrapper({ tmp, wrapInfo, sessionId, output }) {
|
||||
const filePath = path.join(tmp, wrapInfo.file);
|
||||
const src = await fs.readFile(filePath, 'utf-8');
|
||||
const lines = src.split('\n');
|
||||
|
||||
// Find the "Variants: insert below this line" comment line — definitive
|
||||
// marker, robust to any indentation off-by-one. Matches in any comment
|
||||
// style (HTML / JSX / Astro).
|
||||
const markerIdx = lines.findIndex((l) =>
|
||||
l.includes('Variants: insert below this line'),
|
||||
);
|
||||
if (markerIdx === -1) {
|
||||
throw new Error('insert marker not found in ' + wrapInfo.file);
|
||||
}
|
||||
|
||||
const indent = (lines[markerIdx].match(/^\s*/) || [''])[0];
|
||||
// Indent INSIDE the wrapper is one level shallower (the marker is indented
|
||||
// 2 spaces relative to the wrapper opening). Remove the 2-space comment
|
||||
// indent to get the wrapper indent.
|
||||
const wrapperIndent = indent.replace(/ $/, '');
|
||||
|
||||
const block = renderVariantsBlock({
|
||||
sessionId,
|
||||
indent: wrapperIndent,
|
||||
output,
|
||||
commentSyntax: wrapInfo.commentSyntax,
|
||||
file: wrapInfo.file,
|
||||
});
|
||||
|
||||
const next = [
|
||||
...lines.slice(0, markerIdx + 1),
|
||||
block,
|
||||
...lines.slice(markerIdx + 1),
|
||||
];
|
||||
await fs.writeFile(filePath, next.join('\n'), 'utf-8');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Poll loop — the "agent" runs this until aborted
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @param {object} opts
|
||||
* @param {string} opts.tmp Project tmp dir (cwd for live-* scripts).
|
||||
* @param {string} opts.scriptsDir Path to the impeccable scripts dir.
|
||||
* @param {number} opts.port live-server port.
|
||||
* @param {string} opts.token live-server token.
|
||||
* @param {VariantAgent} opts.agent
|
||||
* @param {AbortSignal} opts.signal
|
||||
* @param {(msg: string) => void} [opts.log]
|
||||
* @param {object} [opts.wrapTarget] Default target for live-wrap when an
|
||||
* element comes from the picker without an
|
||||
* id we can resolve. e.g. {classes:'hero-title', tag:'h1'}.
|
||||
*/
|
||||
export async function runAgentLoop({
|
||||
tmp,
|
||||
scriptsDir,
|
||||
port,
|
||||
token,
|
||||
agent,
|
||||
signal,
|
||||
log = () => {},
|
||||
wrapTarget = { classes: 'hero-title', tag: 'h1' },
|
||||
}) {
|
||||
const base = `http://127.0.0.1:${port}`;
|
||||
|
||||
while (!signal.aborted) {
|
||||
let event;
|
||||
try {
|
||||
const res = await fetch(`${base}/poll?token=${token}&timeout=5000`, { signal });
|
||||
event = await res.json();
|
||||
} catch (err) {
|
||||
if (signal.aborted) return;
|
||||
log('poll error: ' + err.message);
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (event.type === 'timeout') continue;
|
||||
if (event.type === 'exit') return;
|
||||
if (event.type === 'prefetch') continue;
|
||||
if (event.type === 'connected') continue;
|
||||
|
||||
if (event.type === 'generate') {
|
||||
log(`generate id=${event.id} action=${event.action} count=${event.count}`);
|
||||
try {
|
||||
// 1. Wrap the original element in the variant scaffold (deterministic CLI)
|
||||
const wrapInfo = await runWrap({
|
||||
tmp,
|
||||
scriptsDir,
|
||||
id: event.id,
|
||||
count: event.count,
|
||||
...wrapTarget,
|
||||
});
|
||||
log(`wrapped: ${wrapInfo.file} insertLine=${wrapInfo.insertLine}`);
|
||||
|
||||
// 2. Agent generates variant content (LLM-pluggable seam)
|
||||
const output = await agent.generateVariants(event, { wrapTarget });
|
||||
if (output.variants.length !== event.count) {
|
||||
log(`warning: agent returned ${output.variants.length} variants, expected ${event.count}`);
|
||||
}
|
||||
|
||||
// 3. Splice variants block into the wrapper (deterministic fs)
|
||||
await spliceVariantsIntoWrapper({ tmp, wrapInfo, sessionId: event.id, output });
|
||||
if (process.env.IMPECCABLE_E2E_DEBUG) {
|
||||
const post = await fs.readFile(path.join(tmp, wrapInfo.file), 'utf-8');
|
||||
log(`--- post-splice (variants written) ---\n${post}`);
|
||||
}
|
||||
|
||||
// 4. Tell the server we're done (broadcasts SSE done → browser settles to CYCLING)
|
||||
await fetch(`${base}/poll`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ token, type: 'done', id: event.id }),
|
||||
signal,
|
||||
});
|
||||
} catch (err) {
|
||||
if (signal.aborted) return;
|
||||
log('generate failed: ' + err.message);
|
||||
await fetch(`${base}/poll`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ token, type: 'error', id: event.id, message: err.message }),
|
||||
signal,
|
||||
}).catch(() => {});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (event.type === 'accept') {
|
||||
log(`accept id=${event.id} variantId=${event.variantId}`);
|
||||
try {
|
||||
const acceptResult = await runAccept({
|
||||
tmp,
|
||||
scriptsDir,
|
||||
id: event.id,
|
||||
variant: event.variantId,
|
||||
paramValues: event.paramValues,
|
||||
});
|
||||
|
||||
// Carbonize cleanup — required after accept per the live skill spec.
|
||||
// For the fake agent, we perform a faithful but minimal cleanup:
|
||||
// delete the carbonize block (markers + dead variants + inline <style>
|
||||
// + param-values comment) and unwrap the temporary variant div around
|
||||
// the accepted content. A real LLM agent would additionally migrate
|
||||
// the @scope rules into the project's stylesheet — out of scope for
|
||||
// a deterministic test.
|
||||
if (acceptResult.handled === true && acceptResult.carbonize === true && acceptResult.file) {
|
||||
if (process.env.IMPECCABLE_E2E_DEBUG) {
|
||||
const post = await fs.readFile(path.join(tmp, acceptResult.file), 'utf-8');
|
||||
log(`--- post-accept (pre-carbonize) ---\n${post}`);
|
||||
}
|
||||
await runCarbonizeCleanup({ tmp, file: acceptResult.file, sessionId: event.id, variant: event.variantId });
|
||||
log(`carbonize cleanup done on ${acceptResult.file}`);
|
||||
}
|
||||
|
||||
await fetch(`${base}/poll`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ token, type: 'accept', id: event.id, data: { _acceptResult: acceptResult } }),
|
||||
signal,
|
||||
});
|
||||
} catch (err) {
|
||||
if (signal.aborted) return;
|
||||
log('accept failed: ' + err.message);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (event.type === 'discard') {
|
||||
log(`discard id=${event.id}`);
|
||||
try {
|
||||
const discardResult = await runAccept({ tmp, scriptsDir, id: event.id, discard: true });
|
||||
await fetch(`${base}/poll`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ token, type: 'discard', id: event.id, data: { _acceptResult: discardResult } }),
|
||||
signal,
|
||||
});
|
||||
} catch (err) {
|
||||
if (signal.aborted) return;
|
||||
log('discard failed: ' + err.message);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
log(`unhandled event: ${event.type}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function runWrap({ tmp, scriptsDir, id, count, classes, tag, elementId }) {
|
||||
const args = [path.join(scriptsDir, 'live-wrap.mjs'), '--id', id, '--count', String(count)];
|
||||
if (elementId) args.push('--element-id', elementId);
|
||||
if (classes) args.push('--classes', classes);
|
||||
if (tag) args.push('--tag', tag);
|
||||
const { stdout } = await execFileP(process.execPath, args, { cwd: tmp });
|
||||
const last = stdout.trim().split('\n').filter(Boolean).pop();
|
||||
return JSON.parse(last);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the post-accept carbonize cleanup to the given file. Mirrors the
|
||||
* five-step rewrite the live skill expects of the agent:
|
||||
*
|
||||
* 1. Locate the carbonize block (bracketed by `impeccable-carbonize-start`
|
||||
* and `impeccable-carbonize-end`).
|
||||
* 2. Step 2 ("move CSS into the project stylesheet") is skipped — that
|
||||
* requires per-project judgment about which file owns these styles.
|
||||
* The fake agent leaves CSS migration to the LLM-backed agent.
|
||||
* 3-5. Strip the carbonize block entirely AND unwrap the temporary
|
||||
* `<div data-impeccable-variant="N" style="display: contents"|...>` wrapper
|
||||
* that holds the accepted content. The accepted inner element survives.
|
||||
*/
|
||||
async function runCarbonizeCleanup({ tmp, file, sessionId /* , variant */ }) {
|
||||
const filePath = path.join(tmp, file);
|
||||
let body = await fs.readFile(filePath, 'utf-8');
|
||||
|
||||
// 1. Strip the carbonize block. We match either comment style so this
|
||||
// works for both HTML and JSX targets.
|
||||
const startRe = new RegExp('[ \\t]*(?:<!--|\\{/\\*)\\s*impeccable-carbonize-start\\s+' + sessionId + '\\s*(?:-->|\\*/\\})\\n');
|
||||
const endRe = new RegExp('[ \\t]*(?:<!--|\\{/\\*)\\s*impeccable-carbonize-end\\s+' + sessionId + '\\s*(?:-->|\\*/\\})\\n?');
|
||||
const startMatch = body.match(startRe);
|
||||
const endMatch = body.match(endRe);
|
||||
if (startMatch && endMatch && startMatch.index < endMatch.index) {
|
||||
const startIdx = startMatch.index;
|
||||
const endIdx = endMatch.index + endMatch[0].length;
|
||||
body = body.slice(0, startIdx) + body.slice(endIdx);
|
||||
}
|
||||
|
||||
// 2. Unwrap the temporary `<div data-impeccable-variant="N" ...>` placed
|
||||
// around the accepted content. live-accept emits this wrapper with
|
||||
// `style="display: contents"` so it doesn't affect layout. We strip the
|
||||
// wrapper open/close lines and keep what's between.
|
||||
// Match the opening div (any single line) followed by inner content
|
||||
// followed by `</div>`, where the open carries data-impeccable-variant
|
||||
// and is NOT inside a data-impeccable-variants wrapper (the variants
|
||||
// wrapper has the trailing `s`).
|
||||
body = body.replace(
|
||||
/^([ \t]*)<div\b[^>]*\bdata-impeccable-variant="[^"]+"[^>]*>\n([\s\S]*?)\n[ \t]*<\/div>\n/m,
|
||||
(match, indent, inner) => {
|
||||
// Re-indent inner content to the wrapper's indent level.
|
||||
const innerLines = inner.split('\n');
|
||||
const innerIndent = (innerLines[0].match(/^\s*/) || [''])[0];
|
||||
const dedented = innerLines.map((l) => {
|
||||
if (l.startsWith(innerIndent)) return indent + l.slice(innerIndent.length);
|
||||
return l;
|
||||
}).join('\n');
|
||||
return dedented + '\n';
|
||||
},
|
||||
);
|
||||
|
||||
await fs.writeFile(filePath, body, 'utf-8');
|
||||
}
|
||||
|
||||
async function runAccept({ tmp, scriptsDir, id, variant, discard, paramValues }) {
|
||||
const args = [path.join(scriptsDir, 'live-accept.mjs'), '--id', id];
|
||||
if (discard) args.push('--discard');
|
||||
else args.push('--variant', String(variant));
|
||||
if (paramValues) args.push('--param-values', JSON.stringify(paramValues));
|
||||
const { stdout } = await execFileP(process.execPath, args, { cwd: tmp });
|
||||
const last = stdout.trim().split('\n').filter(Boolean).pop();
|
||||
return JSON.parse(last);
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
/**
|
||||
* Per-fixture session lifecycle for live-mode E2E tests.
|
||||
*
|
||||
* Composes:
|
||||
* - tmp staging (clones the fixture, git init, writes the inject config)
|
||||
* - npm install (the fixture's runtime.install command)
|
||||
* - live-server.mjs --background (returns {pid, port, token})
|
||||
* - live-inject.mjs --port (patches the framework HTML entry)
|
||||
* - the fixture's framework dev server (vite, vite dev, npx vite, ...)
|
||||
* - Playwright Chromium page
|
||||
* - the fake-agent poll loop (in this same node process)
|
||||
*
|
||||
* Returns handles + a single `teardown()` that cleans them all up in order.
|
||||
*/
|
||||
|
||||
import { execFileSync, spawn } from 'node:child_process';
|
||||
import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { runAgentLoop } from './agent.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = join(__dirname, '..', '..');
|
||||
const SCRIPTS_DIR = join(REPO_ROOT, 'source', 'skills', 'impeccable', 'scripts');
|
||||
const FIXTURES_DIR = join(REPO_ROOT, 'tests', 'framework-fixtures');
|
||||
|
||||
export { SCRIPTS_DIR, FIXTURES_DIR, REPO_ROOT };
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function stageFixture(name, fixture) {
|
||||
const fixtureRoot = join(FIXTURES_DIR, name);
|
||||
const gitignore = readFileSync(join(fixtureRoot, 'gitignore.txt'), 'utf-8');
|
||||
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'impeccable-e2e-'));
|
||||
cpSync(join(fixtureRoot, 'files'), tmp, { recursive: true });
|
||||
writeFileSync(join(tmp, '.gitignore'), gitignore);
|
||||
writeFileSync(join(tmp, 'impeccable-live.config.json'), JSON.stringify(fixture.config));
|
||||
|
||||
execFileSync('git', ['init', '-q'], { cwd: tmp });
|
||||
execFileSync('git', ['config', 'user.email', 'test@example.com'], { cwd: tmp });
|
||||
execFileSync('git', ['config', 'user.name', 'Fixture'], { cwd: tmp });
|
||||
execFileSync('git', ['add', '-A'], { cwd: tmp });
|
||||
execFileSync('git', ['commit', '-qm', 'fixture'], { cwd: tmp });
|
||||
|
||||
return tmp;
|
||||
}
|
||||
|
||||
export function runInstall(tmp, command) {
|
||||
const [cmd, ...args] = command;
|
||||
execFileSync(cmd, args, { cwd: tmp, stdio: 'inherit' });
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// live-server (background mode prints {pid, port, token})
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function startLiveServer(tmp) {
|
||||
const out = execFileSync(
|
||||
process.execPath,
|
||||
[join(SCRIPTS_DIR, 'live-server.mjs'), '--background'],
|
||||
{ cwd: tmp, encoding: 'utf-8' },
|
||||
);
|
||||
const jsonLine = out.trim().split('\n').filter(Boolean).pop();
|
||||
const info = JSON.parse(jsonLine);
|
||||
if (!info.port || !info.pid) {
|
||||
throw new Error('live-server --background returned unexpected payload: ' + jsonLine);
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
export function stopLiveServer(tmp) {
|
||||
try {
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[join(SCRIPTS_DIR, 'live-server.mjs'), 'stop', '--keep-inject'],
|
||||
{ cwd: tmp, stdio: 'ignore' },
|
||||
);
|
||||
} catch { /* already gone */ }
|
||||
}
|
||||
|
||||
export function runInject(tmp, port) {
|
||||
const out = execFileSync(
|
||||
process.execPath,
|
||||
[join(SCRIPTS_DIR, 'live-inject.mjs'), '--port', String(port)],
|
||||
{
|
||||
cwd: tmp,
|
||||
encoding: 'utf-8',
|
||||
env: { ...process.env, IMPECCABLE_LIVE_CONFIG: join(tmp, 'impeccable-live.config.json') },
|
||||
},
|
||||
);
|
||||
const last = out.trim().split('\n').filter(Boolean).pop();
|
||||
return JSON.parse(last);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Framework dev server
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function startDevServer(tmp, runtime) {
|
||||
const [cmd, ...args] = runtime.devCommand;
|
||||
const child = spawn(cmd, args, {
|
||||
cwd: tmp,
|
||||
env: { ...process.env, FORCE_COLOR: '0', NO_COLOR: '1' },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
|
||||
const readyRe = new RegExp(runtime.readyPattern);
|
||||
const bufLog = [];
|
||||
const capture = (chunk) => {
|
||||
const s = chunk.toString();
|
||||
bufLog.push(s);
|
||||
if (bufLog.length > 200) bufLog.shift();
|
||||
};
|
||||
child.stdout.on('data', capture);
|
||||
child.stderr.on('data', capture);
|
||||
|
||||
const ready = new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
reject(new Error(
|
||||
`dev server ready timeout (${runtime.readyTimeoutMs}ms). Tail:\n${bufLog.join('')}`,
|
||||
));
|
||||
}, runtime.readyTimeoutMs ?? 120_000);
|
||||
|
||||
const checkMatch = (buf) => {
|
||||
const m = buf.toString().match(readyRe);
|
||||
if (m && m[1]) {
|
||||
clearTimeout(timeout);
|
||||
resolve({ port: Number(m[1]) });
|
||||
}
|
||||
};
|
||||
child.stdout.on('data', checkMatch);
|
||||
child.stderr.on('data', checkMatch);
|
||||
child.on('exit', (code) => {
|
||||
clearTimeout(timeout);
|
||||
reject(new Error(`dev server exited before ready (code=${code}). Tail:\n${bufLog.join('')}`));
|
||||
});
|
||||
});
|
||||
|
||||
return { child, ready, log: () => bufLog.join('') };
|
||||
}
|
||||
|
||||
export async function stopDevServer(child) {
|
||||
if (!child || child.killed) return;
|
||||
const exited = new Promise((resolve) => child.once('exit', resolve));
|
||||
child.kill('SIGTERM');
|
||||
const timeoutPromise = new Promise((resolve) => setTimeout(resolve, 5_000));
|
||||
await Promise.race([exited, timeoutPromise]);
|
||||
if (!child.killed) child.kill('SIGKILL');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Composite: full stage → ready
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Boots everything and returns the connected page + handles + teardown.
|
||||
*
|
||||
* @param {object} opts
|
||||
* @param {string} opts.name fixture name
|
||||
* @param {object} opts.fixture fixture.json contents
|
||||
* @param {import('playwright').Browser} opts.browser shared browser instance
|
||||
* @param {object} opts.agent VariantAgent (defaults to fake)
|
||||
* @param {(msg: string) => void} [opts.log]
|
||||
*/
|
||||
export async function bootFixtureSession({ name, fixture, browser, agent, log = () => {} }) {
|
||||
const runtime = fixture.runtime;
|
||||
if (!runtime) throw new Error(`fixture ${name} has no runtime block`);
|
||||
|
||||
const tmp = stageFixture(name, fixture);
|
||||
let live;
|
||||
let dev;
|
||||
let agentAbort;
|
||||
let agentDone;
|
||||
let ctx;
|
||||
|
||||
const teardown = async () => {
|
||||
try { if (ctx) await ctx.close(); } catch {}
|
||||
try { if (agentAbort) agentAbort.abort(); } catch {}
|
||||
try { if (agentDone) await agentDone.catch(() => {}); } catch {}
|
||||
try { if (dev?.child) await stopDevServer(dev.child); } catch {}
|
||||
try { if (live) stopLiveServer(tmp); } catch {}
|
||||
try { rmSync(tmp, { recursive: true, force: true }); } catch {}
|
||||
};
|
||||
|
||||
try {
|
||||
log(`installing deps`);
|
||||
runInstall(tmp, runtime.install);
|
||||
|
||||
log(`starting live-server`);
|
||||
live = startLiveServer(tmp);
|
||||
|
||||
log(`live-inject --port ${live.port}`);
|
||||
const injectResult = runInject(tmp, live.port);
|
||||
if (!injectResult.ok) throw new Error('live-inject failed: ' + JSON.stringify(injectResult));
|
||||
|
||||
log(`spawning dev server: ${runtime.devCommand.join(' ')}`);
|
||||
dev = startDevServer(tmp, runtime);
|
||||
const { port: devPort } = await dev.ready;
|
||||
log(`dev server ready on ${devPort}`);
|
||||
|
||||
// Agent loop runs concurrently — abort on teardown.
|
||||
agentAbort = new AbortController();
|
||||
agentDone = runAgentLoop({
|
||||
tmp,
|
||||
scriptsDir: SCRIPTS_DIR,
|
||||
port: live.port,
|
||||
token: live.token,
|
||||
agent,
|
||||
signal: agentAbort.signal,
|
||||
log: (m) => log('[agent] ' + m),
|
||||
});
|
||||
|
||||
const scheme = runtime.scheme || 'http';
|
||||
ctx = await browser.newContext({
|
||||
ignoreHTTPSErrors: runtime.ignoreHTTPSErrors === true,
|
||||
});
|
||||
const page = await ctx.newPage();
|
||||
const consoleErrors = [];
|
||||
page.on('pageerror', (err) => {
|
||||
consoleErrors.push(`pageerror: ${err.message}\n${err.stack || ''}`);
|
||||
});
|
||||
page.on('console', (msg) => {
|
||||
if (msg.type() === 'error') consoleErrors.push(`console.error: ${msg.text()}`);
|
||||
});
|
||||
|
||||
await page.goto(`${scheme}://127.0.0.1:${devPort}`, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 30_000,
|
||||
});
|
||||
|
||||
return {
|
||||
tmp,
|
||||
page,
|
||||
ctx,
|
||||
dev,
|
||||
live,
|
||||
consoleErrors,
|
||||
teardown,
|
||||
};
|
||||
} catch (err) {
|
||||
if (dev?.log) err.message += `\n\n--- dev server tail ---\n${dev.log()}`;
|
||||
await teardown();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* Playwright helpers that drive the live-mode bar UI exactly the way a user
|
||||
* would: pick an element, configure, Go, cycle, accept.
|
||||
*
|
||||
* Selector strategy: live-browser.js uses deterministic ids (`impeccable-live-*`)
|
||||
* for the global bar, per-element bar, action picker, and params panel. Buttons
|
||||
* inside the per-element bar are matched by visible text or unicode glyph
|
||||
* (`Go →`, `← / →`, `✓ Accept`, `✕`). All selectors below come from
|
||||
* source/skills/impeccable/scripts/live-browser.js — keep this file in sync if
|
||||
* the bar's text content changes.
|
||||
*/
|
||||
|
||||
const BAR_ID = '#impeccable-live-bar';
|
||||
const GLOBAL_BAR_ID = '#impeccable-live-global-bar';
|
||||
const PICKER_ID = '#impeccable-live-picker';
|
||||
|
||||
/**
|
||||
* Wait for the live handshake to complete:
|
||||
* - window.__IMPECCABLE_LIVE_INIT__ set
|
||||
* - global bar mounted
|
||||
* - SSE connection established (state transitioned to PICKING)
|
||||
*
|
||||
* Times out generously since some frameworks delay first render.
|
||||
*/
|
||||
export async function waitForHandshake(page, { timeout = 20_000 } = {}) {
|
||||
await page.waitForFunction(
|
||||
() => window.__IMPECCABLE_LIVE_INIT__ === true,
|
||||
{ timeout },
|
||||
);
|
||||
await page.waitForSelector(GLOBAL_BAR_ID, { timeout });
|
||||
// Wait for the picker mode to be active (live.js flips state PICKING after
|
||||
// SSE 'connected' arrives). We can detect it via the global bar's pick
|
||||
// toggle being in its ready state. Soft wait — fall through after a beat
|
||||
// even if the toggle hasn't visibly shifted.
|
||||
await page.waitForTimeout(250);
|
||||
}
|
||||
|
||||
/**
|
||||
* Click an in-page element to select it. live-browser.js's picker only acts
|
||||
* when state === 'PICKING' AND pickActive is true; pickActive starts true on
|
||||
* connect. The handler reads the hovered element from `mousemove`, so we
|
||||
* dispatch a hover before the click.
|
||||
*/
|
||||
export async function pickElement(page, selector) {
|
||||
const el = await page.waitForSelector(selector, { timeout: 5_000 });
|
||||
await el.hover();
|
||||
// Tiny settle: live-browser updates `hoveredElement` on mousemove, and the
|
||||
// click handler reads from it.
|
||||
await page.waitForTimeout(50);
|
||||
await el.click();
|
||||
// Per-element bar mounts on click → wait for it.
|
||||
await page.waitForSelector(BAR_ID, { state: 'visible', timeout: 5_000 });
|
||||
// Wait specifically for the Configure-row Go button to be in the bar.
|
||||
// pickElement returning before that race-conditions with clickGo on
|
||||
// fixtures whose framework re-renders right after pick (modal open, tab
|
||||
// switch). Anchoring the wait on the Go button's text is robust: the bar
|
||||
// can be visible-but-empty (state=PICKING) before showBar('configure')
|
||||
// populates the row.
|
||||
await page.waitForFunction(
|
||||
(barSel) => {
|
||||
const bar = document.querySelector(barSel);
|
||||
if (!bar) return false;
|
||||
const btns = [...bar.querySelectorAll('button')];
|
||||
return btns.some((b) => /Go\b/.test(b.textContent || ''));
|
||||
},
|
||||
BAR_ID,
|
||||
{ timeout: 5_000 },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the variant count by clicking the count button (cycles 2 → 3 → 4 → 2).
|
||||
* Default is 3. If the desired count is already showing, this is a no-op.
|
||||
*/
|
||||
export async function setCount(page, count) {
|
||||
if (count < 2 || count > 4) throw new Error('count must be 2..4');
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const current = await page.evaluate((barSel) => {
|
||||
const bar = document.querySelector(barSel);
|
||||
if (!bar) return null;
|
||||
const btns = [...bar.querySelectorAll('button')];
|
||||
const btn = btns.find((b) => /^×\d+$/.test((b.textContent || '').trim()));
|
||||
if (!btn) return null;
|
||||
return parseInt((btn.textContent || '').trim().slice(1), 10);
|
||||
}, BAR_ID);
|
||||
if (current === count) return;
|
||||
await page.locator(`${BAR_ID} button`, { hasText: /^×\d+$/ }).click();
|
||||
}
|
||||
throw new Error(`could not cycle count to ${count}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Click Go. Browser POSTs the generate event; the agent picks it up.
|
||||
*/
|
||||
export async function clickGo(page) {
|
||||
await page.locator(`${BAR_ID} button`, { hasText: /Go\b/ }).click();
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the bar to enter CYCLING state — happens after the agent's
|
||||
* variants land in the DOM via HMR and the MutationObserver counts them.
|
||||
*
|
||||
* The cycling row has the visible counter `N/M` in monospaced font; we
|
||||
* detect it by content. The bar can also auto-reload if HMR was slow, so
|
||||
* we give it a generous window.
|
||||
*/
|
||||
export async function waitForCycling(page, expectedCount, { timeout = 30_000 } = {}) {
|
||||
await page.waitForFunction(
|
||||
({ barSel, expected }) => {
|
||||
const bar = document.querySelector(barSel);
|
||||
if (!bar) return false;
|
||||
const text = bar.textContent || '';
|
||||
// Counter format: "1/3", "2/3" etc. Look for any "i/N" with N matching.
|
||||
const m = text.match(/(\d+)\s*\/\s*(\d+)/);
|
||||
if (!m) return false;
|
||||
return parseInt(m[2], 10) === expected;
|
||||
},
|
||||
{ barSel: BAR_ID, expected: expectedCount },
|
||||
{ timeout },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Click the next variant button (right arrow).
|
||||
*/
|
||||
export async function clickNext(page) {
|
||||
await page.locator(`${BAR_ID} button`, { hasText: '→' }).click();
|
||||
}
|
||||
|
||||
export async function clickPrev(page) {
|
||||
await page.locator(`${BAR_ID} button`, { hasText: '←' }).click();
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the currently visible variant index (the "i" in "i/N").
|
||||
*/
|
||||
export async function getVisibleVariant(page) {
|
||||
return page.evaluate((barSel) => {
|
||||
const bar = document.querySelector(barSel);
|
||||
if (!bar) return null;
|
||||
const m = (bar.textContent || '').match(/(\d+)\s*\/\s*(\d+)/);
|
||||
return m ? parseInt(m[1], 10) : null;
|
||||
}, BAR_ID);
|
||||
}
|
||||
|
||||
/**
|
||||
* Click Accept — sends accept event with current variantId + paramValues.
|
||||
* The bar transitions to a "Saving..." spinner, then a green confirmed row.
|
||||
*/
|
||||
export async function clickAccept(page) {
|
||||
await page.locator(`${BAR_ID} button`, { hasText: /Accept/ }).click();
|
||||
}
|
||||
|
||||
/**
|
||||
* Click Discard — sends discard event. live-accept.mjs unwinds the wrapper
|
||||
* and restores the original.
|
||||
*/
|
||||
export async function clickDiscard(page) {
|
||||
// The discard button has just a "✕" glyph as text content.
|
||||
await page.locator(`${BAR_ID} button`, { hasText: '✕' }).click();
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the bar to go away (after accept/discard the bar hides on confirm).
|
||||
*/
|
||||
export async function waitForBarHidden(page, { timeout = 10_000 } = {}) {
|
||||
await page.waitForFunction(
|
||||
(barSel) => {
|
||||
const bar = document.querySelector(barSel);
|
||||
return !bar || bar.style.display === 'none';
|
||||
},
|
||||
BAR_ID,
|
||||
{ timeout },
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user