/** * End-to-end live-mode tests — full click-to-accept cycle. * * For every framework fixture with a `runtime` block in fixture.json, this * runner exercises the entire user-visible chain: * * 1. Stage → install → start live-server + dev server → inject script tag * 2. Open Playwright Chromium, assert the live handshake fires * 3. Spawn a deterministic fake-agent polling loop in this same process * 4. Steer smoke: submit page-level chat → agent steer_done → bar unlocks * 5. Drive the bar UI: pick element → Go → wait CYCLING → cycle → Accept * 6. Assert source rewrite (variants block, then accepted-only after accept) * 7. Assert DOM reflects the accepted variant via getComputedStyle * 8. Tear down (browser, dev server, agent loop, live-server, tmp) * * The fake and LLM agents share one interface — see tests/live-e2e/agent.mjs * and tests/live-e2e/agents/llm-agent.mjs. * * Run with: bun run test:live-e2e */ import { describe, it, before, after } from 'node:test'; import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; import { appendFileSync, cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, join, relative } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createFakeAgent } from './live-e2e/agent.mjs'; import { createLlmAgent, resolveLlmAgentConfig } from './live-e2e/agents/llm-agent.mjs'; import { bootFixtureSession, FIXTURES_DIR } from './live-e2e/session.mjs'; import { assertApplyDockVisible, assertApplyDockLoading, assertAnnotationUploadEvent, assertSourceApplied, clickExitLiveMode, clickAccept, clickApplyEdits, clickEditCopy, clickDiscard, clickSaveEdit, clickGo, clickNext, clickPrev, editTextLeaf, drawAnnotationPinAndStroke, getVisibleVariant, installLiveQueryHelpers, pickElement, runLiveChromeBottomBarSmoke, waitForApplyDockHidden, waitForBarHidden, waitForCycling, runInsertFlow, waitForHandshake, } from './live-e2e/ui.mjs'; import { runSteerSmoke } from './live-e2e/steer.mjs'; import { runPreActions, waitForCyclingRobust } from './live-e2e/preactions.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); // Discover fixtures that opt into the runtime E2E pass. function listRuntimeFixtures() { const names = readdirSync(FIXTURES_DIR, { withFileTypes: true }) .filter((e) => e.isDirectory()) .map((e) => e.name); const out = []; for (const name of names) { const fixturePath = join(FIXTURES_DIR, name, 'fixture.json'); if (!existsSync(fixturePath)) continue; const fixture = JSON.parse(readFileSync(fixturePath, 'utf-8')); if (fixture.runtime) out.push({ name, fixture }); } return out; } const allFixtures = listRuntimeFixtures(); // During development of the full-cycle test, a fixture subset is much faster // to iterate on. Set IMPECCABLE_E2E_ONLY=[,...] to scope the run. const onlyNames = parseFixtureFilter(process.env.IMPECCABLE_E2E_ONLY); const fixtures = onlyNames.size > 0 ? allFixtures.filter((f) => onlyNames.has(f.name)) : allFixtures; const missingOnlyNames = [...onlyNames].filter((name) => !allFixtures.some((f) => f.name === name)); if (missingOnlyNames.length > 0) { throw new Error(`Unknown IMPECCABLE_E2E_ONLY fixture(s): ${missingOnlyNames.join(', ')}`); } const manualOnly = process.env.IMPECCABLE_E2E_MANUAL_ONLY === '1' || process.env.IMPECCABLE_E2E_MANUAL_ONLY === 'true'; const reloadVariants = process.env.IMPECCABLE_E2E_RELOAD_VARIANTS === '1' || process.env.IMPECCABLE_E2E_RELOAD_VARIANTS === 'true'; const scenarioNames = parseFixtureFilter(process.env.IMPECCABLE_E2E_SCENARIOS); const liveE2eTestTimeoutMs = readPositiveIntEnv('IMPECCABLE_E2E_TEST_TIMEOUT_MS'); const liveE2eTestOptions = liveE2eTestTimeoutMs ? { timeout: liveE2eTestTimeoutMs } : {}; if (fixtures.length === 0) { describe('live-e2e (no runtime fixtures registered)', () => { it('is a no-op', () => assert.ok(true)); }); } let playwright; let browser; function parseFixtureFilter(value) { return new Set( String(value || '') .split(/[,\s]+/) .map((name) => name.trim()) .filter(Boolean), ); } function readPositiveIntEnv(name) { const raw = process.env[name]; if (raw == null || raw === '') return null; const parsed = Number(raw); return Number.isFinite(parsed) && parsed > 0 ? parsed : null; } function shouldRunScenario(name) { return scenarioNames.size === 0 || scenarioNames.has('all') || scenarioNames.has(name); } before(async () => { if (fixtures.length === 0) return; try { playwright = await import('playwright'); } catch (err) { throw new Error( `Playwright is required for live-e2e tests (${err.message}). Run: npx playwright install chromium`, ); } try { browser = await launchLiveE2eBrowser(); } catch (err) { throw new Error(`Failed to launch Chromium (${err.message}). Run: npx playwright install chromium`); } }); after(async () => { if (browser) await browser.close(); }); async function launchLiveE2eBrowser() { return playwright.chromium.launch({ headless: true }); } async function teardownAndResetBrowser(teardown) { try { await teardown(); } finally { if (browser) await browser.close().catch(() => {}); browser = await launchLiveE2eBrowser(); } } for (const { name, fixture } of fixtures) { describe(`live-e2e · ${name} (${fixture.runtime.styling || 'unknown-styling'})`, () => { it('drives the full click → Go → cycle → accept cycle', liveE2eTestOptions, async (t) => { if (!shouldRunScenario('core')) { t.skip('scenario filter excludes core'); return; } if (manualOnly || process.env.IMPECCABLE_E2E_MANUAL_SCENARIO) { t.skip('manual scenario filter is active'); return; } // Fixtures may declare `runtime.knownLimitation` to flag a scenario // that exposes a genuine live-mode gap rather than a test bug. The // test still attempts the full chain but does not fail the suite when // the documented failure mode appears — it surfaces the diagnostic so // the limitation is visible in the run output. const knownLimitation = fixture.runtime.knownLimitation; // Pick the agent. `IMPECCABLE_E2E_AGENT=llm` opts into Claude first, // with DeepSeek as the secondary fallback/override; everything else // uses the deterministic fake. Skip rather than fail when LLM is // requested but the selected provider key is missing so default suite // runs in unauthenticated environments still pass. const agentMode = process.env.IMPECCABLE_E2E_AGENT || 'fake'; let agent; if (agentMode === 'llm') { const llmConfig = resolveLlmAgentConfig({ model: process.env.IMPECCABLE_E2E_LLM_MODEL, }); agent = await createLlmAgent({ config: llmConfig, log: (m) => t.diagnostic('[llm] ' + m), }); if (!agent) { t.skip(`IMPECCABLE_E2E_AGENT=llm with provider=${llmConfig.provider} requires ${llmConfig.requiredEnv}`); return; } t.diagnostic(`Using LLM agent (provider=${llmConfig.provider} model=${llmConfig.model})`); } else { agent = createFakeAgent(); } t.diagnostic(`Booting fixture ${name}`); const session = await bootFixtureSession({ name, fixture, browser, agent, wrapTarget: wrapTargetFromPickedElement, log: (m) => t.diagnostic(m), }); const { page, tmp, consoleErrors, teardown } = session; const expectedCount = 3; const isInsert = fixture.runtime.mode === 'insert'; const insertCfg = fixture.runtime.insert || {}; const pickSelector = fixture.runtime.pickSelector || 'h1.hero-title'; const insertDomSelector = agentMode === 'llm' && insertCfg.expectSelectorLlm ? insertCfg.expectSelectorLlm : (insertCfg.expectSelector || '.inserted-strip'); const domSelector = isInsert ? insertDomSelector : pickSelector; const usesSvelteComponentPreview = fixtureUsesSvelteKitAdapter(fixture) || name === 'nuxt-vite7'; const variantContentSelector = isInsert ? (usesSvelteComponentPreview ? '.inserted-copy' : '[data-impeccable-variant="2"] .inserted-copy') : usesSvelteComponentPreview ? pickSelector : '[data-impeccable-variant="2"] > :first-child'; let stateProbeBaseline = null; let sourceFile = null; try { // 1. Handshake t.diagnostic('Waiting for live handshake'); await waitForHandshake(page); if (fixture.runtime.liveChrome?.bottomBar) { t.diagnostic('Running live chrome bottom-bar smoke'); await runLiveChromeBottomBarSmoke(page, { expectDetectMinCount: fixture.runtime.liveChrome.detect?.expectMinCount || 1, designTitle: fixture.runtime.liveChrome.design?.title || '', designRawText: fixture.runtime.liveChrome.design?.rawText || '', }); } // 1b. Steer smoke — page-level chat before the heavier generate cycle. if (fixture.runtime.steer !== false) { const steerTimeouts = agentMode === 'llm' ? { unlockTimeoutMs: 90_000, selectorTimeoutMs: 45_000, runPreActions } : { runPreActions }; await runSteerSmoke(page, tmp, fixture, (m) => t.diagnostic(m), steerTimeouts); } // 2. preActions — fixtures with hidden/conditional content (modals, // tabs, routes) drive the page into the right state before pick. if (fixture.runtime.preActions) { t.diagnostic(`Running ${fixture.runtime.preActions.length} preAction(s)`); await runPreActions(page, fixture.runtime.preActions); if (fixture.runtime.stateProbe) { stateProbeBaseline = await assertStateProbe(page, fixture.runtime.stateProbe, 'after preActions'); } } // 3. Start generate — replace picks an element; insert places a placeholder. if (isInsert) { t.diagnostic(`Insert after ${insertCfg.anchorSelector || 'anchor'}`); await runInsertFlow(page, { anchorSelector: insertCfg.anchorSelector || 'section#features', position: insertCfg.position || 'after', prompt: insertCfg.prompt || 'Add new content', }); } else { t.diagnostic(`Picking ${pickSelector}`); await pickElement(page, pickSelector); if (process.env.IMPECCABLE_E2E_DEBUG) { const barText = await page.evaluate(() => { const bar = document.querySelector('#impeccable-live-bar'); return bar ? { display: bar.style.display, text: bar.textContent || '', html: bar.innerHTML.slice(0, 500) } : null; }); t.diagnostic(`Bar after pick: ${JSON.stringify(barText)}`); } t.diagnostic('Clicking Go'); await clickGo(page); } // 4. Wait for the agent's variants to land (HMR + MutationObserver). // For fixtures whose picked element lives inside a conditional // render (modal, tab, route), HMR can remount the parent and lose // the open/active state — the wrapper exists in source but isn't // in the DOM, so MutationObserver never sees it. Live mode now // surfaces a toast asking the user to retrace the path; we mirror // that here by re-running preActions on the first short timeout. // // The first-pass timeout has to be long enough to cover the agent's // generate latency before declaring "state was lost, retrace." A // fake agent finishes in <100ms. The real LLM path usually lands // quickly too, but full-matrix runs can see minute-scale API or // install pressure, so keep this gate patient enough that we do // not retrace while the agent is still writing the variants. t.diagnostic(`Waiting for CYCLING state with ${expectedCount} variants`); await waitForCyclingRobust(page, expectedCount, { agentMode, preActions: fixture.runtime.preActions, log: (m) => t.diagnostic(m), }); if (fixture.runtime.stateProbe) { await assertStateProbe(page, fixture.runtime.stateProbe, 'after variants', { baseline: stateProbeBaseline }); } // 5. Source-side check: wrapper + style + variants are present sourceFile = await locateSessionFile(tmp); const after = readFileSync(sourceFile, 'utf-8'); const svelteComponentSession = svelteComponentTargetFor(sourceFile); if (svelteComponentSession) { const componentExtension = svelteComponentSession.manifest.componentExtension || 'svelte'; const variantFile = join(tmp, svelteComponentSession.manifest.componentDir, `v2.${componentExtension}`); const variantBody = readFileSync(variantFile, 'utf-8'); const routeBody = readFileSync(join(tmp, svelteComponentSession.manifest.sourceFile), 'utf-8'); assert.match(after, /"previewMode": "(?:svelte|vue)-component"/, 'framework component manifest inserted'); if (isInsert) { assert.equal(svelteComponentSession.manifest.mode, 'insert', 'Svelte insert manifest marks insert mode'); if (agentMode === 'fake') { assert.match(variantBody, /inserted-strip/, 'Svelte insert variant component contains inserted content'); } else if (insertCfg.expectSourcePattern) { assert.match(variantBody, new RegExp(insertCfg.expectSourcePattern, 'i'), 'Svelte insert variant component contains prompt-matching content'); } else { assert.match(variantBody, /<([a-z][\w:-]*)\b[\s\S]*<\/\1>|<[a-z][\w:-]*\b[^>]*\/>/i, 'Svelte insert variant component contains a root element'); } } else { assert.match(variantBody, new RegExp(`<${svelteComponentSession.expectedTag}\\b`), 'component variant contains target element'); } assert.doesNotMatch(routeBody, /data-impeccable-variants="/, 'route source is not edited during component preview'); } else { assert.match(after, /data-impeccable-variants="/, 'wrapper inserted'); } if (isInsert) { if (svelteComponentSession) { assert.equal(svelteComponentSession.manifest.mode, 'insert', 'Svelte insert uses component preview mode'); } else { assert.match(after, /data-impeccable-mode="insert"/, 'insert mode wrapper'); assert.doesNotMatch(after, /data-impeccable-variant="original"/, 'insert has no original variant'); } if (insertCfg.assertAnchorContains) { const anchorSource = svelteComponentSession ? readFileSync(join(tmp, svelteComponentSession.manifest.sourceFile), 'utf-8') : after; assert.match(anchorSource, new RegExp(insertCfg.assertAnchorContains), 'anchor section untouched'); } } if (svelteComponentSession) { const componentExtension = svelteComponentSession.manifest.componentExtension || 'svelte'; assert.match(readFileSync(join(tmp, svelteComponentSession.manifest.componentDir, `v2.${componentExtension}`), 'utf-8'), /\s*(?:h1|\.[\w-]+)/, 'event=live_e2e.astro_css_prefix actor=agent operation=write_variants risk=astro_scopes_preview_css_away expected=variant-prefixed global selector actual=missing suggestion=inspect fake agent styleMode handling', ); assert.doesNotMatch(after, /@scope \(\[data-impeccable-variant="1"\]\)/, 'Astro live CSS does not use raw @scope'); } else { assert.match(after, /