/** * End-to-end live-mode tests — full click-to-accept cycle. * * For every framework fixture with a `runtime` block in fixture.json, this * runner exercises the entire user-visible chain: * * 1. Stage → install → start live-server + dev server → inject script tag * 2. Open Playwright Chromium, assert the live handshake fires * 3. Spawn a deterministic fake-agent polling loop in this same process * 4. Steer smoke: submit page-level chat → agent steer_done → bar unlocks * 5. Drive the bar UI: pick element → Go → wait CYCLING → cycle → Accept * 6. Assert source rewrite (variants block, then accepted-only after accept) * 7. Assert DOM reflects the accepted variant via getComputedStyle * 8. Tear down (browser, dev server, agent loop, live-server, tmp) * * The fake and LLM agents share one interface — see tests/live-e2e/agent.mjs * and tests/live-e2e/agents/llm-agent.mjs. * * Run with: bun run test:live-e2e */ import { describe, it, before, after } from 'node:test'; import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createFakeAgent } from './live-e2e/agent.mjs'; import { createLlmAgent, resolveLlmAgentConfig } from './live-e2e/agents/llm-agent.mjs'; import { bootFixtureSession, FIXTURES_DIR } from './live-e2e/session.mjs'; import { assertApplyDockVisible, assertApplyDockLoading, assertSourceApplied, clickAccept, clickApplyEdits, clickEditCopy, clickSaveEdit, clickGo, clickNext, editTextLeaf, getVisibleVariant, pickElement, waitForApplyDockHidden, waitForBarHidden, waitForCycling, runInsertFlow, waitForHandshake, } from './live-e2e/ui.mjs'; import { runSteerSmoke } from './live-e2e/steer.mjs'; import { runPreActions, waitForCyclingRobust } from './live-e2e/preactions.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); // Discover fixtures that opt into the runtime E2E pass. function listRuntimeFixtures() { const names = readdirSync(FIXTURES_DIR, { withFileTypes: true }) .filter((e) => e.isDirectory()) .map((e) => e.name); const out = []; for (const name of names) { const fixturePath = join(FIXTURES_DIR, name, 'fixture.json'); if (!existsSync(fixturePath)) continue; const fixture = JSON.parse(readFileSync(fixturePath, 'utf-8')); if (fixture.runtime) out.push({ name, fixture }); } return out; } const allFixtures = listRuntimeFixtures(); // During development of the full-cycle test, a single fixture is much faster // to iterate on. Set IMPECCABLE_E2E_ONLY= to scope the run. const onlyName = process.env.IMPECCABLE_E2E_ONLY; const fixtures = onlyName ? allFixtures.filter((f) => f.name === onlyName) : allFixtures; const manualOnly = process.env.IMPECCABLE_E2E_MANUAL_ONLY === '1' || process.env.IMPECCABLE_E2E_MANUAL_ONLY === 'true'; if (fixtures.length === 0) { describe('live-e2e (no runtime fixtures registered)', () => { it('is a no-op', () => assert.ok(true)); }); } let playwright; let browser; before(async () => { if (fixtures.length === 0) return; try { playwright = await import('playwright'); } catch (err) { throw new Error( `Playwright is required for live-e2e tests (${err.message}). Run: npx playwright install chromium`, ); } try { browser = await playwright.chromium.launch({ headless: true }); } catch (err) { throw new Error(`Failed to launch Chromium (${err.message}). Run: npx playwright install chromium`); } }); after(async () => { if (browser) await browser.close(); }); for (const { name, fixture } of fixtures) { describe(`live-e2e · ${name} (${fixture.runtime.styling || 'unknown-styling'})`, () => { it('drives the full click → Go → cycle → accept cycle', async (t) => { if (manualOnly || process.env.IMPECCABLE_E2E_MANUAL_SCENARIO) { t.skip('manual scenario filter is active'); return; } // Fixtures may declare `runtime.knownLimitation` to flag a scenario // that exposes a genuine live-mode gap rather than a test bug. The // test still attempts the full chain but does not fail the suite when // the documented failure mode appears — it surfaces the diagnostic so // the limitation is visible in the run output. const knownLimitation = fixture.runtime.knownLimitation; // Pick the agent. `IMPECCABLE_E2E_AGENT=llm` opts into Claude first, // with DeepSeek as the secondary fallback/override; everything else // uses the deterministic fake. Skip rather than fail when LLM is // requested but the selected provider key is missing so default suite // runs in unauthenticated environments still pass. const agentMode = process.env.IMPECCABLE_E2E_AGENT || 'fake'; let agent; if (agentMode === 'llm') { const llmConfig = resolveLlmAgentConfig({ model: process.env.IMPECCABLE_E2E_LLM_MODEL, }); agent = await createLlmAgent({ config: llmConfig, log: (m) => t.diagnostic('[llm] ' + m), }); if (!agent) { t.skip(`IMPECCABLE_E2E_AGENT=llm with provider=${llmConfig.provider} requires ${llmConfig.requiredEnv}`); return; } t.diagnostic(`Using LLM agent (provider=${llmConfig.provider} model=${llmConfig.model})`); } else { agent = createFakeAgent(); } t.diagnostic(`Booting fixture ${name}`); const session = await bootFixtureSession({ name, fixture, browser, agent, wrapTarget: agentMode === 'llm' ? wrapTargetFromPickedElement : undefined, log: (m) => t.diagnostic(m), }); const { page, tmp, consoleErrors, teardown } = session; const expectedCount = 3; const isInsert = fixture.runtime.mode === 'insert'; const insertCfg = fixture.runtime.insert || {}; const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title'; const domSelector = isInsert ? (insertCfg.expectSelector || '.inserted-strip') : pickSelector; try { // 1. Handshake t.diagnostic('Waiting for live handshake'); await waitForHandshake(page); // 1b. Steer smoke — page-level chat before the heavier generate cycle. if (fixture.runtime.steer !== false) { const steerTimeouts = agentMode === 'llm' ? { unlockTimeoutMs: 90_000, selectorTimeoutMs: 45_000, runPreActions } : { runPreActions }; await runSteerSmoke(page, tmp, fixture, (m) => t.diagnostic(m), steerTimeouts); } // 2. preActions — fixtures with hidden/conditional content (modals, // tabs, routes) drive the page into the right state before pick. if (fixture.runtime.preActions) { t.diagnostic(`Running ${fixture.runtime.preActions.length} preAction(s)`); await runPreActions(page, fixture.runtime.preActions); } // 3. Start generate — replace picks an element; insert places a placeholder. if (isInsert) { t.diagnostic(`Insert after ${insertCfg.anchorSelector || 'anchor'}`); await runInsertFlow(page, { anchorSelector: insertCfg.anchorSelector || 'section#features', position: insertCfg.position || 'after', prompt: insertCfg.prompt || 'Add new content', }); } else { t.diagnostic(`Picking ${pickSelector}`); await pickElement(page, pickSelector); if (process.env.IMPECCABLE_E2E_DEBUG) { const barText = await page.evaluate(() => { const bar = document.querySelector('#impeccable-live-bar'); return bar ? { display: bar.style.display, text: bar.textContent || '', html: bar.innerHTML.slice(0, 500) } : null; }); t.diagnostic(`Bar after pick: ${JSON.stringify(barText)}`); } t.diagnostic('Clicking Go'); await clickGo(page); } // 4. Wait for the agent's variants to land (HMR + MutationObserver). // For fixtures whose picked element lives inside a conditional // render (modal, tab, route), HMR can remount the parent and lose // the open/active state — the wrapper exists in source but isn't // in the DOM, so MutationObserver never sees it. Live mode now // surfaces a toast asking the user to retrace the path; we mirror // that here by re-running preActions on the first short timeout. // // The first-pass timeout has to be long enough to cover the agent's // generate latency before declaring "state was lost, retrace." A // fake agent finishes in <100ms. The real LLM path usually lands // quickly too, but full-matrix runs can see minute-scale API or // install pressure, so keep this gate patient enough that we do // not retrace while the agent is still writing the variants. t.diagnostic(`Waiting for CYCLING state with ${expectedCount} variants`); await waitForCyclingRobust(page, expectedCount, { agentMode, preActions: fixture.runtime.preActions, log: (m) => t.diagnostic(m), }); // 5. Source-side check: wrapper + style + variants are present const sourceFile = await locateSessionFile(tmp); const after = readFileSync(sourceFile, 'utf-8'); assert.match(after, /data-impeccable-variants="/, 'wrapper inserted'); if (isInsert) { assert.match(after, /data-impeccable-mode="insert"/, 'insert mode wrapper'); assert.doesNotMatch(after, /data-impeccable-variant="original"/, 'insert has no original variant'); if (insertCfg.assertAnchorContains) { assert.match(after, new RegExp(insertCfg.assertAnchorContains), 'anchor section untouched'); } } if (sourceFile.endsWith('.astro')) { assert.match(after, /