mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-11 21:57:14 +03:00
tests/live-e2e/agents/llm-agent.mjs: a Claude-backed VariantAgent that
implements the same one-method interface as the fake agent
(generateVariants(event, context) → { scopedCss, variants[] }). Default
model claude-haiku-4-5; override via IMPECCABLE_E2E_LLM_MODEL.
Prompt caching is on — the system prompt (instructions + the live-mode
spec from reference/live.md) is the cacheable prefix. First call writes
~10K tokens to cache; subsequent fixtures pay only the cache-read rate.
JSON output is validated for shape (scopedCss, variants[N].innerHtml),
with light error messages on parse failure.
tests/live-e2e.test.mjs: read IMPECCABLE_E2E_AGENT (fake|llm). When 'llm',
construct the LLM agent and skip the case cleanly if ANTHROPIC_API_KEY is
unset. Param-manifest assertions are gated to fake mode (LLM may emit
zero-param "fixed point" variants per the live.md spec). The accepted-h1
class assertion now allows hero-title as one of multiple classes so an
LLM agent that adds classes alongside the original still passes.
Test timeouts widen for LLM mode: 25s first-pass on conditional-render
fixtures (vs 5s for fake), 60s on direct waits (vs 30s). Without these,
the LLM's 3-8s generate latency races the orchestration's state-loss
recovery window.
tests/live-e2e/ui.mjs: clickGo retries up to 3× on stability failures.
Required because conditional-render fixtures (modal/tabs) animate the bar
mid-transition when preActions trigger framework HMR; a single click can
land during a re-render and Playwright's stability gate times out.
Pass rate on a typical sweep: 18/19 in LLM mode, 19/19 in fake mode.
The modal fixture's intrinsic state-loss flake (Fast Refresh resetting
useState(open) when source changes) is amplified by LLM latency and may
need a re-run; documented in CLAUDE.md.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
193 lines
6.8 KiB
JavaScript
193 lines
6.8 KiB
JavaScript
/**
|
||
* Playwright helpers that drive the live-mode bar UI exactly the way a user
|
||
* would: pick an element, configure, Go, cycle, accept.
|
||
*
|
||
* Selector strategy: live-browser.js uses deterministic ids (`impeccable-live-*`)
|
||
* for the global bar, per-element bar, action picker, and params panel. Buttons
|
||
* inside the per-element bar are matched by visible text or unicode glyph
|
||
* (`Go →`, `← / →`, `✓ Accept`, `✕`). All selectors below come from
|
||
* source/skills/impeccable/scripts/live-browser.js — keep this file in sync if
|
||
* the bar's text content changes.
|
||
*/
|
||
|
||
const BAR_ID = '#impeccable-live-bar';
|
||
const GLOBAL_BAR_ID = '#impeccable-live-global-bar';
|
||
const PICKER_ID = '#impeccable-live-picker';
|
||
|
||
/**
|
||
* Wait for the live handshake to complete:
|
||
* - window.__IMPECCABLE_LIVE_INIT__ set
|
||
* - global bar mounted
|
||
* - SSE connection established (state transitioned to PICKING)
|
||
*
|
||
* Times out generously since some frameworks delay first render.
|
||
*/
|
||
export async function waitForHandshake(page, { timeout = 20_000 } = {}) {
|
||
await page.waitForFunction(
|
||
() => window.__IMPECCABLE_LIVE_INIT__ === true,
|
||
{ timeout },
|
||
);
|
||
await page.waitForSelector(GLOBAL_BAR_ID, { timeout });
|
||
// Wait for the picker mode to be active (live.js flips state PICKING after
|
||
// SSE 'connected' arrives). We can detect it via the global bar's pick
|
||
// toggle being in its ready state. Soft wait — fall through after a beat
|
||
// even if the toggle hasn't visibly shifted.
|
||
await page.waitForTimeout(250);
|
||
}
|
||
|
||
/**
|
||
* Click an in-page element to select it. live-browser.js's picker only acts
|
||
* when state === 'PICKING' AND pickActive is true; pickActive starts true on
|
||
* connect. The handler reads the hovered element from `mousemove`, so we
|
||
* dispatch a hover before the click.
|
||
*/
|
||
export async function pickElement(page, selector) {
|
||
const el = await page.waitForSelector(selector, { timeout: 5_000 });
|
||
await el.hover();
|
||
// Tiny settle: live-browser updates `hoveredElement` on mousemove, and the
|
||
// click handler reads from it.
|
||
await page.waitForTimeout(50);
|
||
await el.click();
|
||
// Per-element bar mounts on click → wait for it.
|
||
await page.waitForSelector(BAR_ID, { state: 'visible', timeout: 5_000 });
|
||
// Wait specifically for the Configure-row Go button to be in the bar.
|
||
// pickElement returning before that race-conditions with clickGo on
|
||
// fixtures whose framework re-renders right after pick (modal open, tab
|
||
// switch). Anchoring the wait on the Go button's text is robust: the bar
|
||
// can be visible-but-empty (state=PICKING) before showBar('configure')
|
||
// populates the row.
|
||
await page.waitForFunction(
|
||
(barSel) => {
|
||
const bar = document.querySelector(barSel);
|
||
if (!bar) return false;
|
||
const btns = [...bar.querySelectorAll('button')];
|
||
return btns.some((b) => /Go\b/.test(b.textContent || ''));
|
||
},
|
||
BAR_ID,
|
||
{ timeout: 5_000 },
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Set the variant count by clicking the count button (cycles 2 → 3 → 4 → 2).
|
||
* Default is 3. If the desired count is already showing, this is a no-op.
|
||
*/
|
||
export async function setCount(page, count) {
|
||
if (count < 2 || count > 4) throw new Error('count must be 2..4');
|
||
for (let i = 0; i < 4; i++) {
|
||
const current = await page.evaluate((barSel) => {
|
||
const bar = document.querySelector(barSel);
|
||
if (!bar) return null;
|
||
const btns = [...bar.querySelectorAll('button')];
|
||
const btn = btns.find((b) => /^×\d+$/.test((b.textContent || '').trim()));
|
||
if (!btn) return null;
|
||
return parseInt((btn.textContent || '').trim().slice(1), 10);
|
||
}, BAR_ID);
|
||
if (current === count) return;
|
||
await page.locator(`${BAR_ID} button`, { hasText: /^×\d+$/ }).click();
|
||
}
|
||
throw new Error(`could not cycle count to ${count}`);
|
||
}
|
||
|
||
/**
|
||
* Click Go. Browser POSTs the generate event; the agent picks it up.
|
||
*
|
||
* On fixtures whose preActions triggered a layout shift (modal/tab opening)
|
||
* the bar's open animation can still be running when we click, and
|
||
* Playwright's stability gate occasionally times out on the first attempt.
|
||
* Retry up to three times with a settle in between so a single race doesn't
|
||
* fail the test.
|
||
*/
|
||
export async function clickGo(page) {
|
||
const go = page.locator(`${BAR_ID} button`, { hasText: /Go\b/ });
|
||
let lastErr;
|
||
for (let attempt = 0; attempt < 3; attempt++) {
|
||
try {
|
||
await go.click({ timeout: 5_000 });
|
||
return;
|
||
} catch (err) {
|
||
lastErr = err;
|
||
await page.waitForTimeout(500);
|
||
}
|
||
}
|
||
throw lastErr;
|
||
}
|
||
|
||
/**
|
||
* Wait for the bar to enter CYCLING state — happens after the agent's
|
||
* variants land in the DOM via HMR and the MutationObserver counts them.
|
||
*
|
||
* The cycling row has the visible counter `N/M` in monospaced font; we
|
||
* detect it by content. The bar can also auto-reload if HMR was slow, so
|
||
* we give it a generous window.
|
||
*/
|
||
export async function waitForCycling(page, expectedCount, { timeout = 30_000 } = {}) {
|
||
await page.waitForFunction(
|
||
({ barSel, expected }) => {
|
||
const bar = document.querySelector(barSel);
|
||
if (!bar) return false;
|
||
const text = bar.textContent || '';
|
||
// Counter format: "1/3", "2/3" etc. Look for any "i/N" with N matching.
|
||
const m = text.match(/(\d+)\s*\/\s*(\d+)/);
|
||
if (!m) return false;
|
||
return parseInt(m[2], 10) === expected;
|
||
},
|
||
{ barSel: BAR_ID, expected: expectedCount },
|
||
{ timeout },
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Click the next variant button (right arrow).
|
||
*/
|
||
export async function clickNext(page) {
|
||
await page.locator(`${BAR_ID} button`, { hasText: '→' }).click();
|
||
}
|
||
|
||
export async function clickPrev(page) {
|
||
await page.locator(`${BAR_ID} button`, { hasText: '←' }).click();
|
||
}
|
||
|
||
/**
|
||
* Read the currently visible variant index (the "i" in "i/N").
|
||
*/
|
||
export async function getVisibleVariant(page) {
|
||
return page.evaluate((barSel) => {
|
||
const bar = document.querySelector(barSel);
|
||
if (!bar) return null;
|
||
const m = (bar.textContent || '').match(/(\d+)\s*\/\s*(\d+)/);
|
||
return m ? parseInt(m[1], 10) : null;
|
||
}, BAR_ID);
|
||
}
|
||
|
||
/**
|
||
* Click Accept — sends accept event with current variantId + paramValues.
|
||
* The bar transitions to a "Saving..." spinner, then a green confirmed row.
|
||
*/
|
||
export async function clickAccept(page) {
|
||
await page.locator(`${BAR_ID} button`, { hasText: /Accept/ }).click();
|
||
}
|
||
|
||
/**
|
||
* Click Discard — sends discard event. live-accept.mjs unwinds the wrapper
|
||
* and restores the original.
|
||
*/
|
||
export async function clickDiscard(page) {
|
||
// The discard button has just a "✕" glyph as text content.
|
||
await page.locator(`${BAR_ID} button`, { hasText: '✕' }).click();
|
||
}
|
||
|
||
/**
|
||
* Wait for the bar to go away (after accept/discard the bar hides on confirm).
|
||
*/
|
||
export async function waitForBarHidden(page, { timeout = 10_000 } = {}) {
|
||
await page.waitForFunction(
|
||
(barSel) => {
|
||
const bar = document.querySelector(barSel);
|
||
return !bar || bar.style.display === 'none';
|
||
},
|
||
BAR_ID,
|
||
{ timeout },
|
||
);
|
||
}
|