/** * End-to-end live-mode tests — full click-to-accept cycle. * * For every framework fixture with a `runtime` block in fixture.json, this * runner exercises the entire user-visible chain: * * 1. Stage → install → start live-server + dev server → inject script tag * 2. Open Playwright Chromium, assert the live handshake fires * 3. Spawn a deterministic fake-agent polling loop in this same process * 4. Drive the bar UI: pick element → Go → wait CYCLING → cycle → Accept * 5. Assert source rewrite (variants block, then accepted-only after accept) * 6. Assert DOM reflects the accepted variant via getComputedStyle * 7. Tear down (browser, dev server, agent loop, live-server, tmp) * * The fake agent is pluggable — see tests/live-e2e/agent.mjs. A future * LLM-backed agent slots in by implementing the same VariantAgent interface. * * Run with: bun run test:live-e2e */ import { describe, it, before, after } from 'node:test'; import assert from 'node:assert/strict'; import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createFakeAgent } from './live-e2e/agent.mjs'; import { createLlmAgent } from './live-e2e/agents/llm-agent.mjs'; import { bootFixtureSession, FIXTURES_DIR } from './live-e2e/session.mjs'; import { clickAccept, clickGo, clickNext, getVisibleVariant, pickElement, waitForCycling, waitForHandshake, } from './live-e2e/ui.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); // Discover fixtures that opt into the runtime E2E pass. function listRuntimeFixtures() { const names = readdirSync(FIXTURES_DIR, { withFileTypes: true }) .filter((e) => e.isDirectory()) .map((e) => e.name); const out = []; for (const name of names) { const fixturePath = join(FIXTURES_DIR, name, 'fixture.json'); if (!existsSync(fixturePath)) continue; const fixture = JSON.parse(readFileSync(fixturePath, 'utf-8')); if (fixture.runtime) out.push({ name, fixture }); } return out; } const allFixtures = listRuntimeFixtures(); // During development of the full-cycle test, a single fixture is much faster // to iterate on. Set IMPECCABLE_E2E_ONLY= to scope the run. const onlyName = process.env.IMPECCABLE_E2E_ONLY; const fixtures = onlyName ? allFixtures.filter((f) => f.name === onlyName) : allFixtures; if (fixtures.length === 0) { describe('live-e2e (no runtime fixtures registered)', () => { it('is a no-op', () => assert.ok(true)); }); } let playwright; let browser; before(async () => { if (fixtures.length === 0) return; try { playwright = await import('playwright'); } catch (err) { throw new Error( `Playwright is required for live-e2e tests (${err.message}). Run: npx playwright install chromium`, ); } try { browser = await playwright.chromium.launch({ headless: true }); } catch (err) { throw new Error(`Failed to launch Chromium (${err.message}). Run: npx playwright install chromium`); } }); after(async () => { if (browser) await browser.close(); }); for (const { name, fixture } of fixtures) { describe(`live-e2e · ${name} (${fixture.runtime.styling || 'unknown-styling'})`, () => { it('drives the full click → Go → cycle → accept cycle', async (t) => { // Fixtures may declare `runtime.knownLimitation` to flag a scenario // that exposes a genuine live-mode gap rather than a test bug. The // test still attempts the full chain but does not fail the suite when // the documented failure mode appears — it surfaces the diagnostic so // the limitation is visible in the run output. const knownLimitation = fixture.runtime.knownLimitation; // Pick the agent. `IMPECCABLE_E2E_AGENT=llm` opts into the real Claude // API; everything else uses the deterministic fake. Skip rather than // fail when LLM is requested but no API key is set so default suite // runs in unauthenticated environments still pass. const agentMode = process.env.IMPECCABLE_E2E_AGENT || 'fake'; let agent; if (agentMode === 'llm') { agent = await createLlmAgent({ model: process.env.IMPECCABLE_E2E_LLM_MODEL, log: (m) => t.diagnostic('[llm] ' + m), }); if (!agent) { t.skip('IMPECCABLE_E2E_AGENT=llm requires ANTHROPIC_API_KEY'); return; } t.diagnostic(`Using LLM agent (model=${process.env.IMPECCABLE_E2E_LLM_MODEL || 'claude-haiku-4-5'})`); } else { agent = createFakeAgent(); } t.diagnostic(`Booting fixture ${name}`); const session = await bootFixtureSession({ name, fixture, browser, agent, log: (m) => t.diagnostic(m), }); const { page, tmp, consoleErrors, teardown } = session; const expectedCount = 3; const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title'; try { // 1. Handshake t.diagnostic('Waiting for live handshake'); await waitForHandshake(page); // 2. preActions — fixtures with hidden/conditional content (modals, // tabs, routes) drive the page into the right state before pick. if (fixture.runtime.preActions) { t.diagnostic(`Running ${fixture.runtime.preActions.length} preAction(s)`); await runPreActions(page, fixture.runtime.preActions); } // 3. Pick the target element t.diagnostic(`Picking ${pickSelector}`); await pickElement(page, pickSelector); if (process.env.IMPECCABLE_E2E_DEBUG) { const barText = await page.evaluate(() => { const bar = document.querySelector('#impeccable-live-bar'); return bar ? { display: bar.style.display, text: bar.textContent || '', html: bar.innerHTML.slice(0, 500) } : null; }); t.diagnostic(`Bar after pick: ${JSON.stringify(barText)}`); } // 3. Click Go (default action 'impeccable', default count 3 — fixture-stable) t.diagnostic('Clicking Go'); await clickGo(page); // 4. Wait for the agent's variants to land (HMR + MutationObserver). // For fixtures whose picked element lives inside a conditional // render (modal, tab, route), HMR can remount the parent and lose // the open/active state — the wrapper exists in source but isn't // in the DOM, so MutationObserver never sees it. Live mode now // surfaces a toast asking the user to retrace the path; we mirror // that here by re-running preActions on the first short timeout. // // The first-pass timeout has to be long enough to cover the agent's // generate latency before declaring "state was lost, retrace." A // fake agent finishes in <100ms; an LLM agent typically lands in // 3-8s. Scale the gate accordingly. t.diagnostic(`Waiting for CYCLING state with ${expectedCount} variants`); const firstPassTimeoutMs = agentMode === 'llm' ? 25_000 : 5_000; let cyclingReached = false; if (fixture.runtime.preActions) { try { await waitForCycling(page, expectedCount, { timeout: firstPassTimeoutMs }); cyclingReached = true; } catch { t.diagnostic(`Cycling not reached in ${firstPassTimeoutMs}ms — retracing preActions`); await runPreActions(page, fixture.runtime.preActions); } } try { if (!cyclingReached) { // Default 30s; LLM mode bumps to 60s to absorb API latency on // top of HMR settle time. const finalTimeoutMs = agentMode === 'llm' ? 60_000 : 30_000; await waitForCycling(page, expectedCount, { timeout: finalTimeoutMs }); } } catch (err) { if (process.env.IMPECCABLE_E2E_DEBUG) { const variantCount = await page.evaluate(() => document.querySelectorAll('[data-impeccable-variant]').length, ); const barInfo = await page.evaluate(() => { const bars = document.querySelectorAll('#impeccable-live-bar'); return { count: bars.length, bars: [...bars].map((bar) => ({ display: bar.style.display, opacity: bar.style.opacity, text: bar.textContent || '', innerHtml: bar.innerHTML.slice(0, 600), })), __init: window.__IMPECCABLE_LIVE_INIT__, }; }); t.diagnostic(`waitForCycling failed; variants in DOM: ${variantCount}`); t.diagnostic(`Bar state: ${JSON.stringify(barInfo)}`); t.diagnostic(`--- dev server tail ---\n${session.dev.log()}`); } throw err; } // 5. Source-side check: wrapper + style + variants are present const sourceFile = await locateSessionFile(tmp); const after = readFileSync(sourceFile, 'utf-8'); assert.match(after, /data-impeccable-variants="/, 'wrapper inserted'); assert.match(after, /