diff --git a/package.json b/package.json index fb360c785..d1ce59b66 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "test:cli-remote-e2e": "node scripts/run-tests.mjs cli-remote-e2e", "test:live-e2e": "node scripts/run-tests.mjs live-e2e", "test:live-e2e-accept-cleanup": "node scripts/run-tests.mjs live-e2e-accept-cleanup", + "test:new-work-e2e": "node scripts/run-tests.mjs new-work-e2e", "test:live-e2e-agent": "node scripts/run-tests.mjs live-e2e-agent", "test:skill-behavior": "node scripts/run-tests.mjs skill-behavior", "test:live-svelte-adapter-deepseek": "node scripts/run-tests.mjs live-svelte-adapter-deepseek", diff --git a/scripts/test-suites.mjs b/scripts/test-suites.mjs index 365f1dea4..b911c75d4 100644 --- a/scripts/test-suites.mjs +++ b/scripts/test-suites.mjs @@ -6,6 +6,7 @@ export const OPT_IN_SUITES = [ 'cli-remote-e2e', 'live-e2e', 'live-e2e-accept-cleanup', + 'new-work-e2e', 'skill-behavior', 'live-svelte-adapter-deepseek', ]; @@ -222,6 +223,24 @@ export const SUITES = { }, ], }, + 'new-work-e2e': { + description: 'Playwright smoke sweep of the new-work concept/serve-question decision page plus the offline fake image generator.', + optIn: true, + needsPlaywright: true, + triggers: [ + ...COMMON_INFRA_PATTERNS, + /^skill\/scripts\/(serve-question|generate-image|concept-seed)\.mjs$/, + /^tests\/new-work-e2e(\.test\.mjs|\/)/, + ], + commands: [ + { + runner: 'node', + timeoutMs: 600000, + forceExit: true, + files: ['tests/new-work-e2e.test.mjs'], + }, + ], + }, 'live-e2e-accept-cleanup': { description: 'Provider-backed post-accept cleanup regression.', optIn: true, diff --git a/skill/scripts/generate-image.mjs b/skill/scripts/generate-image.mjs index 17501e3dc..873bc84c8 100644 --- a/skill/scripts/generate-image.mjs +++ b/skill/scripts/generate-image.mjs @@ -12,6 +12,7 @@ * node generate-image.mjs --prompt-file prompt.txt --out mock.png */ import fs from 'node:fs'; +import zlib from 'node:zlib'; function arg(name, fallback = null) { const i = process.argv.indexOf(`--${name}`); @@ -20,6 +21,183 @@ function arg(name, fallback = null) { return v && !v.startsWith('--') ? v : fallback; } +// --------------------------------------------------------------------------- +// Fake mode (IMPECCABLE_IMAGE_GEN_FAKE=1) +// +// Deterministic offline stand-in for the OpenAI call: same prompt -> identical +// bytes, no network, no key, cost line reads $0.00. Used by the new-work smoke +// suite so the concept/serve-question/image chain can run without spend. The +// output renders the prompt over a 2-3 color palette hashed from the prompt, +// plus a "SYNTHETIC COMP" corner label. SVG carries the readable text; the +// raster (.png/.webp/.jpg) fallback carries palette stripes and stows the +// prompt + marker in a PNG tEXt chunk so downstream stays a valid image. +// --------------------------------------------------------------------------- + +// FNV-1a 32-bit: tiny, dependency-free, stable across runs and platforms. +function hash32(str) { + let h = 0x811c9dc5; + for (let i = 0; i < str.length; i++) { + h ^= str.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + return h >>> 0; +} + +function hslToRgb(hDeg, s, l) { + const h = ((hDeg % 360) + 360) % 360 / 360; + const q = l < 0.5 ? l * (1 + s) : l + s - l * s; + const p = 2 * l - q; + const hue = (t) => { + let tt = t; + if (tt < 0) tt += 1; + if (tt > 1) tt -= 1; + if (tt < 1 / 6) return p + (q - p) * 6 * tt; + if (tt < 1 / 2) return q; + if (tt < 2 / 3) return p + (q - p) * (2 / 3 - tt) * 6; + return p; + }; + return [hue(h + 1 / 3), hue(h), hue(h - 1 / 3)].map((c) => Math.round(c * 255)); +} + +const toHex = ([r, g, b]) => + '#' + [r, g, b].map((c) => c.toString(16).padStart(2, '0')).join(''); + +// Two or three deterministic swatches derived from the prompt hash. The band +// count itself is prompt-derived, so different prompts differ in palette. +function palette(prompt) { + const h = hash32(prompt); + const base = h % 360; + const bands = 2 + (h >>> 9) % 2; // 2 or 3 + const spread = 40 + (h >>> 3) % 120; + const out = []; + for (let i = 0; i < bands; i++) { + const hue = base + i * spread; + const light = 0.32 + ((h >>> (i * 5)) % 40) / 100; // 0.32 - 0.71 + out.push(hslToRgb(hue, 0.55, light)); + } + return out; +} + +function svgFake(prompt, [w, h]) { + const colors = palette(prompt).map(toHex); + const stops = colors + .map((c, i) => ``) + .join(''); + // Greedy word wrap tuned to the canvas width so the prompt stays legible. + const perLine = Math.max(12, Math.floor(w / 26)); + const words = String(prompt).replace(/\s+/g, ' ').trim().split(' '); + const lines = []; + let cur = ''; + for (const word of words) { + if ((cur + ' ' + word).trim().length > perLine) { + if (cur) lines.push(cur); + cur = word; + } else { + cur = (cur + ' ' + word).trim(); + } + if (lines.length >= 10) break; + } + if (cur && lines.length < 11) lines.push(cur); + const escape = (s) => String(s).replace(/[&<>]/g, (c) => ({ '&': '&', '<': '<', '>': '>' }[c])); + const fontSize = Math.round(w / 24); + const startY = h / 2 - ((lines.length - 1) * fontSize * 1.3) / 2; + const text = lines + .map((line, i) => `${escape(line)}`) + .join(''); + return ` + + ${stops} + + + ${text} + + SYNTHETIC COMP + +`; +} + +// Minimal valid PNG: palette stripes plus a tEXt chunk carrying the marker and +// prompt, so a .png/.webp fake stays a decodable image and still contains the +// "SYNTHETIC" bytes downstream tools look for. +function crc32(buf) { + let c = 0xffffffff; + for (let i = 0; i < buf.length; i++) { + c ^= buf[i]; + for (let k = 0; k < 8; k++) c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1); + } + return (c ^ 0xffffffff) >>> 0; +} + +function pngChunk(type, data) { + const typeBuf = Buffer.from(type, 'latin1'); + const body = Buffer.concat([typeBuf, data]); + const len = Buffer.alloc(4); + len.writeUInt32BE(data.length, 0); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(body), 0); + return Buffer.concat([len, body, crc]); +} + +function pngFake(prompt, [w, h]) { + const colors = palette(prompt); // [[r,g,b], ...] + const bandH = Math.ceil(h / colors.length); + // Raw image: each scanline prefixed with a 0 filter byte, RGB pixels. + const stride = w * 3; + const raw = Buffer.alloc(h * (stride + 1)); + for (let y = 0; y < h; y++) { + const rowStart = y * (stride + 1); + raw[rowStart] = 0; + const [r, g, b] = colors[Math.min(colors.length - 1, Math.floor(y / bandH))]; + for (let x = 0; x < w; x++) { + const p = rowStart + 1 + x * 3; + raw[p] = r; + raw[p + 1] = g; + raw[p + 2] = b; + } + } + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(w, 0); + ihdr.writeUInt32BE(h, 4); + ihdr[8] = 8; // bit depth + ihdr[9] = 2; // color type: truecolor RGB + const idat = zlib.deflateSync(raw, { level: 9 }); + const textData = Buffer.concat([ + Buffer.from('Comment', 'latin1'), + Buffer.from([0]), + Buffer.from(`SYNTHETIC COMP: ${String(prompt).replace(/\s+/g, ' ').trim()}`, 'latin1'), + ]); + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk('IHDR', ihdr), + pngChunk('tEXt', textData), + pngChunk('IDAT', idat), + pngChunk('IEND', Buffer.alloc(0)), + ]); +} + +function parseSize(sizeStr) { + const m = String(sizeStr).match(/^(\d+)x(\d+)$/); + if (!m) return [1536, 1024]; + return [Number(m[1]), Number(m[2])]; +} + +if (process.env.IMPECCABLE_IMAGE_GEN_FAKE) { + const fakePromptFile = arg('prompt-file'); + const fakePrompt = fakePromptFile ? fs.readFileSync(fakePromptFile, 'utf8') : arg('prompt'); + const fakeOut = arg('out'); + if (!fakePrompt || !fakeOut) { + console.error('generate-image: --prompt (or --prompt-file) and --out are required.'); + process.exit(1); + } + const dims = parseSize(arg('size', '1536x1024')); + const bytes = fakeOut.endsWith('.svg') + ? Buffer.from(svgFake(fakePrompt, dims), 'utf8') + : pngFake(fakePrompt, dims); + fs.writeFileSync(fakeOut, bytes); + console.log(`IMAGE: ${fakeOut} (${dims[0]}x${dims[1]}, fake synthetic comp, $0.00, no API call)`); + process.exit(0); +} + const key = process.env.OPENAI_API_KEY; if (!key) { console.error('generate-image: OPENAI_API_KEY is not set; use the harness-native image tool instead.'); diff --git a/tests/new-work-e2e.test.mjs b/tests/new-work-e2e.test.mjs new file mode 100644 index 000000000..0cbde6567 --- /dev/null +++ b/tests/new-work-e2e.test.mjs @@ -0,0 +1,357 @@ +/** + * Deterministic smoke tests for the new-work interactive flow. + * + * Covers the parts a user actually touches: the serve-question decision page + * (pick, re-roll + steer + re-deal, canon, tab close) driven through a real + * browser by the scripted user bot, plus the offline fake image generator. + * No LLM calls; a real Chromium via Playwright supplies full page fidelity + * (heartbeats, re-roll reload, tab close). Kept OUT of `bun run test` like + * live-e2e; run it with `bun run test:new-work-e2e`. + * + * The concept-seed direction roll (challengers, ASSIGNED INDEX, the no + * PRODUCT.md gate) is already covered by tests/concept-seed.test.mjs and is + * not repeated here. + * + * One-time setup: npx playwright install chromium + */ + +import { describe, it, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { mkdtempSync, writeFileSync, mkdirSync, readFileSync, existsSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { runUserBot } from './new-work-e2e/user-bot.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const SERVE = path.join(ROOT, 'skill', 'scripts', 'serve-question.mjs'); +const GENERATE = path.join(ROOT, 'skill', 'scripts', 'generate-image.mjs'); +const CATALOG_DIR = path.join(ROOT, 'tests', 'fixtures', 'concept-catalog'); + +let playwright; +let browser; + +before(async () => { + try { + playwright = await import('playwright'); + } catch (err) { + throw new Error( + `Playwright is required for new-work-e2e tests (${err.message}). Run: npx playwright install chromium`, + ); + } + try { + browser = await playwright.chromium.launch({ headless: true }); + } catch (err) { + throw new Error(`Failed to launch Chromium (${err.message}). Run: npx playwright install chromium`); + } +}); + +after(async () => { + if (browser) await browser.close(); +}); + +// -------------------------------------------------------------------------- +// Workspace + serve-question helpers +// -------------------------------------------------------------------------- +function makeWorkspace() { + const dir = mkdtempSync(path.join(tmpdir(), 'new-work-e2e-')); + writeFileSync( + path.join(dir, 'PRODUCT.md'), + '# Product\n\n## Register\n\nbrand\n\n## Platform\n\nweb\n', + ); + return dir; +} + +// serve-question writes its state under cwd; run everything from the workspace. +function run(args, cwd) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [SERVE, ...args], { + cwd, + env: { ...process.env, IMPECCABLE_QUESTION_FORCE: '1', IMPECCABLE_CATALOG_DIR: CATALOG_DIR }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let out = ''; + let err = ''; + child.stdout.on('data', (c) => { out += c; }); + child.stderr.on('data', (c) => { err += c; }); + child.on('exit', (code) => resolve({ code, out, err })); + }); +} + +async function startDaemon(cwd, payload, key) { + const payloadPath = path.join(cwd, `${key}.payload.json`); + writeFileSync(payloadPath, JSON.stringify(payload)); + const started = await run(['--start', '--payload', payloadPath, '--no-open', '--key', key], cwd); + assert.equal(started.code, 0, `--start failed: ${started.out} ${started.err}`); + const url = started.out.match(/QUESTION URL: (\S+)/)?.[1]; + assert.ok(url, `no URL from --start: ${started.out}`); + return { url, payloadPath }; +} + +// Poll --wait until it settles on a terminal exit code (0 answered, 2 gone, +// 4 page closed); loop while it reports WAITING (3). +async function waitLoop(cwd, key, { poll = 30, max = 20 } = {}) { + for (let i = 0; i < max; i++) { + const res = await run(['--wait', '--key', key, '--poll', String(poll)], cwd); + if (res.code !== 3) return res; + } + throw new Error('waitLoop exceeded max iterations'); +} + +async function stopDaemon(cwd, key) { + await run(['--stop', '--key', key], cwd).catch(() => {}); +} + +function makeFakeImage(cwd, prompt, outName) { + const out = path.join(cwd, outName); + const res = spawnSyncGen(prompt, out); + assert.equal(res.status, 0, `generate-image fake failed: ${res.stderr}`); + return out; +} + +function spawnSyncGen(prompt, out, size = null) { + const args = [GENERATE, '--prompt', prompt, '--out', out]; + if (size) args.push('--size', size); + return spawnSync(process.execPath, args, { + env: { ...process.env, IMPECCABLE_IMAGE_GEN_FAKE: '1' }, + encoding: 'buffer', + }); +} + +// -------------------------------------------------------------------------- +// serve-question interactive cycles +// -------------------------------------------------------------------------- +describe('new-work-e2e: serve-question decision page', () => { + it('(a) pick assigned returns the option, hero/board fields, and the CHOSEN CARD directive', async () => { + const cwd = makeWorkspace(); + const key = 'pick'; + const hero = makeFakeImage(cwd, 'Fillmore handbill hero', 'hero.png'); + const board = makeFakeImage(cwd, 'Fillmore handbill board', 'board.png'); + const payload = { + title: 'Choose the visual world', + question: 'The roll assigned Fillmore Handbill.', + options: [ + { id: 'assigned', label: 'Fillmore Handbill', kicker: 'THE ROLL', hero, board }, + { id: 'challenger-teletext', label: 'Teletext Service', body: 'block-mosaic pages' }, + ], + reroll: true, + canon: true, + steer: true, + }; + await startDaemon(cwd, payload, key); + try { + const bot = await runUserBot({ + workspaceDir: cwd, key, browser, + policy: [{ pick: 'assigned', steer: 'warmer palette' }], + }); + assert.equal(bot.results[0].action, 'pick'); + const collected = await waitLoop(cwd, key); + assert.equal(collected.code, 0, collected.out); + assert.match(collected.out, /ANSWER: /); + const answer = JSON.parse(collected.out.match(/ANSWER: (\{.*\})/)[1]); + assert.equal(answer.optionId, 'assigned'); + assert.equal(answer.steer, 'warmer palette'); + assert.ok(answer.hero, 'answer carries the chosen hero path'); + assert.ok(answer.board, 'answer carries the chosen board path'); + assert.match(collected.out, /CHOSEN CARD:/); + } finally { + await stopDaemon(cwd, key); + rmSync(cwd, { recursive: true, force: true }); + } + }); + + it('(b) re-roll with steer keeps the server alive; --update re-deals; the next pick is terminal', async () => { + const cwd = makeWorkspace(); + const key = 'reroll'; + const payload1 = { + title: 'Choose the visual world', + options: [ + { id: 'assigned', label: 'First Hand', kicker: 'THE ROLL' }, + { id: 'challenger-a', label: 'Alt One' }, + ], + reroll: true, steer: true, canon: true, + }; + const payload2 = { + title: 'Choose the visual world', + options: [ + { id: 'assigned', label: 'Second Hand', kicker: 'THE ROLL' }, + { id: 'challenger-b', label: 'Alt Two' }, + ], + reroll: true, steer: true, + }; + await startDaemon(cwd, payload1, key); + try { + // Bot drives the whole page: re-roll (with steer) then, after the page + // reloads into the next hand, pick the assigned card. + const botPromise = runUserBot({ + workspaceDir: cwd, key, browser, + policy: [{ reroll: true, steer: 'colder, more restraint' }, { pick: 'assigned' }], + }); + + // First answer: the re-roll. Server must stay alive afterwards. + const first = await waitLoop(cwd, key); + assert.equal(first.code, 0, first.out); + assert.match(first.out, /"optionId":"reroll"/); + assert.match(first.out, /colder, more restraint/); + assert.ok(existsSync(path.join(cwd, '.impeccable', 'questions', `${key}.state.json`)), + 'server state file survives a re-roll'); + + // Deliver the next hand; the live page reloads itself. + const nextPayloadPath = path.join(cwd, 'next.json'); + writeFileSync(nextPayloadPath, JSON.stringify(payload2)); + const updated = await run(['--update', '--key', key, '--payload', nextPayloadPath], cwd); + assert.equal(updated.code, 0, updated.out); + + // Second answer: the terminal pick on the re-dealt hand. + const second = await waitLoop(cwd, key); + assert.equal(second.code, 0, second.out); + assert.match(second.out, /"optionId":"assigned"/); + + const bot = await botPromise; + assert.equal(bot.results[0].action, 'reroll'); + assert.ok(bot.results[0].reloaded, 'page reloaded into the next hand'); + assert.equal(bot.results[1].action, 'pick'); + + // Terminal pick cleans the state file up. + assert.ok(!existsSync(path.join(cwd, '.impeccable', 'questions', `${key}.state.json`)), + 'terminal pick removes the server state file'); + } finally { + await stopDaemon(cwd, key); + rmSync(cwd, { recursive: true, force: true }); + } + }); + + it('(c) canon click returns optionId canon and prints the CANON CHOSEN directive', async () => { + const cwd = makeWorkspace(); + const key = 'canon'; + const payload = { + title: 'Choose the visual world', + options: [{ id: 'assigned', label: 'Fillmore Handbill', kicker: 'THE ROLL' }], + reroll: true, canon: true, steer: true, + }; + await startDaemon(cwd, payload, key); + try { + await runUserBot({ workspaceDir: cwd, key, browser, policy: [{ canon: true }] }); + const collected = await waitLoop(cwd, key); + assert.equal(collected.code, 0, collected.out); + assert.match(collected.out, /"optionId":"canon"/); + assert.match(collected.out, /CANON CHOSEN:/); + } finally { + await stopDaemon(cwd, key); + rmSync(cwd, { recursive: true, force: true }); + } + }); + + it('(d) closing the tab makes --wait exit 4 PAGE CLOSED', async () => { + const cwd = makeWorkspace(); + const key = 'close'; + const payload = { + title: 'Choose the visual world', + options: [{ id: 'assigned', label: 'Fillmore Handbill', kicker: 'THE ROLL' }], + reroll: true, steer: true, + }; + await startDaemon(cwd, payload, key); + try { + const bot = await runUserBot({ workspaceDir: cwd, key, browser, policy: [{ close: true }] }); + assert.equal(bot.results[0].action, 'close'); + assert.ok(bot.results[0].beat > 0, 'a heartbeat landed before the tab closed'); + // --wait must observe the stale heartbeat and report the closed page. + const res = await run(['--wait', '--key', key, '--poll', '30'], cwd); + assert.equal(res.code, 4, `expected exit 4, got ${res.code}: ${res.out}`); + assert.match(res.out, /PAGE CLOSED/); + } finally { + await stopDaemon(cwd, key); + rmSync(cwd, { recursive: true, force: true }); + } + }); + + it('(e) an option with no hero renders a text-only card (no .media element)', async () => { + const cwd = makeWorkspace(); + const key = 'textonly'; + const hero = makeFakeImage(cwd, 'has a hero', 'hero.png'); + const payload = { + title: 'Choose the visual world', + options: [ + { id: 'assigned', label: 'Text Only Direction', body: 'a grounded direction, no comp' }, + { id: 'challenger-hero', label: 'Has A Card', hero }, + ], + reroll: true, steer: true, + }; + const { url } = await startDaemon(cwd, payload, key); + try { + const context = await browser.newContext(); + const page = await context.newPage(); + await page.goto(url, { waitUntil: 'load' }); + await page.waitForSelector('button.choose'); + const textOnlyMedia = await page.$('.card[data-id="assigned"] .media'); + const heroMedia = await page.$('.card[data-id="challenger-hero"] .media'); + const textOnlyFace = await page.$('.card[data-id="assigned"] .face.text-only'); + await context.close(); + assert.equal(textOnlyMedia, null, 'text-only card has no .media region'); + assert.ok(textOnlyFace, 'text-only card carries the .text-only face class'); + assert.ok(heroMedia, 'the hero card still renders its .media region'); + } finally { + await stopDaemon(cwd, key); + rmSync(cwd, { recursive: true, force: true }); + } + }); +}); + +// -------------------------------------------------------------------------- +// Fake image generation +// -------------------------------------------------------------------------- +describe('new-work-e2e: fake image generation', () => { + it('is deterministic per prompt and encodes the SYNTHETIC marker', () => { + const cwd = mkdtempSync(path.join(tmpdir(), 'new-work-img-')); + try { + const a = path.join(cwd, 'a.png'); + const b = path.join(cwd, 'b.png'); + const r1 = spawnSyncGen('Fillmore psychedelic handbill, warm ink', a); + const r2 = spawnSyncGen('Fillmore psychedelic handbill, warm ink', b); + assert.equal(r1.status, 0, r1.stderr?.toString()); + assert.equal(r2.status, 0, r2.stderr?.toString()); + assert.ok(existsSync(a) && existsSync(b), 'both files exist'); + assert.match(r1.stdout.toString(), /\$0\.00/, 'cost line reads $0.00'); + const bytesA = readFileSync(a); + const bytesB = readFileSync(b); + assert.ok(bytesA.equals(bytesB), 'same prompt yields identical bytes'); + // Valid PNG signature + the SYNTHETIC marker (in the tEXt chunk). + assert.equal(bytesA.slice(0, 8).toString('hex'), '89504e470d0a1a0a'); + assert.ok(bytesA.includes(Buffer.from('SYNTHETIC')), 'PNG carries the SYNTHETIC marker'); + } finally { + rmSync(cwd, { recursive: true, force: true }); + } + }); + + it('renders a different palette for a different prompt', () => { + const cwd = mkdtempSync(path.join(tmpdir(), 'new-work-img-')); + try { + const a = path.join(cwd, 'a.png'); + const c = path.join(cwd, 'c.png'); + spawnSyncGen('Fillmore psychedelic handbill, warm ink', a); + spawnSyncGen('Teletext broadcast mosaic, cold blue', c); + const bytesA = readFileSync(a); + const bytesC = readFileSync(c); + assert.ok(!bytesA.equals(bytesC), 'different prompts produce different images'); + } finally { + rmSync(cwd, { recursive: true, force: true }); + } + }); + + it('the SVG variant carries the readable prompt text and SYNTHETIC COMP label', () => { + const cwd = mkdtempSync(path.join(tmpdir(), 'new-work-img-')); + try { + const svg = path.join(cwd, 'comp.svg'); + const res = spawnSyncGen('teletext broadcast mosaic', svg, '800x600'); + assert.equal(res.status, 0, res.stderr?.toString()); + const text = readFileSync(svg, 'utf8'); + assert.match(text, /^<\?xml/, 'is an SVG document'); + assert.match(text, /SYNTHETIC COMP/); + assert.match(text, /teletext/i, 'the prompt text is rendered'); + } finally { + rmSync(cwd, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/new-work-e2e/README.md b/tests/new-work-e2e/README.md new file mode 100644 index 000000000..8f1561447 --- /dev/null +++ b/tests/new-work-e2e/README.md @@ -0,0 +1,65 @@ +# new-work E2E + +A cheap, deterministic smoke suite for the interactive parts of new-work: the +serve-question decision page and the offline image generator. It is kept out of +`bun run test` and runs on demand. + +```bash +bun run test:new-work-e2e +``` + +One-time setup: `npx playwright install chromium` (the suite drives a real +Chromium so the page runs its own JS, exactly as a user's tab would). + +## What it covers + +`tests/new-work-e2e.test.mjs` opens the served decision page with a real +browser and drives it through the scripted user bot, then asserts on the +serve-question protocol output: + +- **pick assigned** returns the chosen `optionId`, the typed steer, the + `hero`/`board` fields, and the `CHOSEN CARD` directive printed by `--wait`. +- **re-roll with steer** keeps the daemon alive, `--update` re-deals the next + hand, the page reloads itself, and the following pick is terminal (state file + cleaned up). +- **canon** returns `optionId: canon` and prints the `CANON CHOSEN` directive. +- **tab close** stops the page heartbeats so `--wait` exits 4 `PAGE CLOSED`. +- **text-only card** renders with no `.media` region when an option has no hero. +- **fake image generation**: same prompt yields identical bytes, the file + exists, the `SYNTHETIC` marker is present, and different prompts produce + different palettes. + +The concept-seed direction roll (challengers, `ASSIGNED INDEX`, the no +PRODUCT.md gate) is already covered by `tests/concept-seed.test.mjs` and is not +repeated here. + +## Pieces + +- `user-bot.mjs` is a module plus CLI. Given a workspace dir it resolves the + running daemon from `.impeccable/questions/.state.json`, opens the page, + and runs a JSON policy of real clicks: `{"pick":"assigned"}`, + `{"reroll":true,"steer":"warmer"}`, `{"pick":"challenger-*"}`, + `{"canon":true}`, `{"close":true}`. The deterministic tier passes an + already-launched browser in; the CLI launches its own Chromium. +- `IMPECCABLE_IMAGE_GEN_FAKE=1` switches `skill/scripts/generate-image.mjs` to + the offline stand-in: no OpenAI call, no key, a `$0.00` cost line, and a + deterministic image (SVG for `.svg` out with the wrapped prompt text and a + `SYNTHETIC COMP` label; a valid palette-stripe PNG otherwise, with the prompt + and marker in a PNG `tEXt` chunk). + +## Planned LLM tier (not built yet) + +The same scaffolding supports an opt-in LLM tier later, mirroring the two-layer +pattern in `tests/live-e2e`: + +- A real model plays the user through the same scripted `user-bot.mjs` policy, + choosing and steering instead of following canned actions. +- `IMPECCABLE_IMAGE_GEN_FAKE` still stands in for image spend, so a full + concept-to-card cycle runs without paying per render. +- Assertions run against the tool-call trace via the skill-behavior harness, + the same way `tests/skill-behavior` keys on the trace rather than free-form + output. + +Cost posture: the deterministic tier is free (no API calls, local Chromium). +The LLM tier hits a provider and costs money, so it stays opt-in and out of CI, +matching how `test:live-e2e` and `test:skill-behavior` are gated today. diff --git a/tests/new-work-e2e/user-bot.mjs b/tests/new-work-e2e/user-bot.mjs new file mode 100644 index 000000000..2fbae07ae --- /dev/null +++ b/tests/new-work-e2e/user-bot.mjs @@ -0,0 +1,196 @@ +/** + * Scripted user bot for the new-work interactive smoke suite. + * + * Given a workspace directory, it discovers a running serve-question daemon + * from `.impeccable/questions/.state.json`, opens the served page in a + * real browser, and drives it through a scripted policy: it clicks the real + * `button.choose`, `#reroll`, and `#canon` controls, types into `#steer`, and + * closes the tab for the exit-4 path. Because a real page runs the page's own + * JS, heartbeats fire and re-roll reloads behave exactly as a user's would. + * + * The deterministic tier passes an already-launched Playwright browser in. + * Run as a CLI (`--workspace DIR --policy ''`) it launches its own + * Chromium. The policy is an ordered list of actions: + * + * { "pick": "assigned" } click the assigned card + * { "pick": "challenger-*" } click the first matching card + * { "pickIndex": 1 } click the Nth choose button + * { "reroll": true, "steer": "warmer" } type the steer, click Re-roll + * { "canon": true } click Play it straight + * { "close": true } close the tab (stops heartbeats) + * + * A `steer` on any action is typed into `#steer` first when the field exists. + * After a re-roll the bot waits for the page to reload into the next hand + * (delivered out of band by `serve-question --update`) before the next action. + */ + +import { readdirSync, readFileSync, existsSync } from 'node:fs'; +import path from 'node:path'; + +function questionsDir(workspaceDir) { + return path.join(workspaceDir, '.impeccable', 'questions'); +} + +// Resolve the served URL from the daemon state file. When no key is given and +// several exist, the newest wins. +export function resolveQuestion(workspaceDir, key = null) { + const dir = questionsDir(workspaceDir); + if (!existsSync(dir)) throw new Error(`no questions dir at ${dir}`); + const stateFiles = readdirSync(dir).filter((f) => f.endsWith('.state.json')); + if (stateFiles.length === 0) throw new Error(`no *.state.json in ${dir}`); + let file; + if (key) { + file = `${key}.state.json`; + if (!stateFiles.includes(file)) throw new Error(`no state file for key ${key}`); + } else { + file = stateFiles + .map((f) => ({ f, mtime: readFileSync(path.join(dir, f), 'utf8') && f })) + .sort() + .pop().f; + } + const resolvedKey = file.replace(/\.state\.json$/, ''); + const state = JSON.parse(readFileSync(path.join(dir, file), 'utf8')); + return { key: resolvedKey, url: state.url, port: state.port, pid: state.pid }; +} + +function stateLastBeat(workspaceDir, key) { + try { + const state = JSON.parse(readFileSync(path.join(questionsDir(workspaceDir), `${key}.state.json`), 'utf8')); + return state.lastBeat || 0; + } catch { + return 0; + } +} + +async function typeSteer(page, action) { + if (action.steer == null) return; + const steer = await page.$('#steer'); + if (steer) await steer.fill(String(action.steer)); +} + +function chooseSelector(pick) { + if (pick.endsWith('*')) { + const prefix = pick.slice(0, -1); + return `button.choose[data-id^="${prefix}"]`; + } + return `button.choose[data-id="${pick}"]`; +} + +async function runAction(page, action, { workspaceDir, key }) { + await typeSteer(page, action); + + if (action.reroll) { + await Promise.all([ + page.waitForNavigation({ waitUntil: 'load', timeout: 60000 }).catch(() => {}), + page.click('#reroll'), + ]); + // Fresh hand loaded: wait for the interactive controls of the next round. + await page.waitForSelector('button.choose', { timeout: 30000 }); + return { action: 'reroll', reloaded: true }; + } + + if (action.canon) { + await page.click('#canon'); + return { action: 'canon' }; + } + + if (action.close) { + // Make sure at least one heartbeat has been recorded so the --wait poll can + // later see the beat go stale (the exit-4 PAGE CLOSED path). + const deadline = Date.now() + 8000; + while (Date.now() < deadline && !stateLastBeat(workspaceDir, key)) { + await page.waitForTimeout(200); + } + await page.close(); + return { action: 'close', beat: stateLastBeat(workspaceDir, key) }; + } + + if (action.pickIndex != null) { + const buttons = await page.$$('button.choose'); + const btn = buttons[action.pickIndex]; + if (!btn) throw new Error(`no choose button at index ${action.pickIndex}`); + await btn.click(); + return { action: 'pick', index: action.pickIndex }; + } + + if (action.pick) { + await page.click(chooseSelector(action.pick)); + return { action: 'pick', id: action.pick }; + } + + throw new Error(`unknown action: ${JSON.stringify(action)}`); +} + +/** + * Drive the served question page through the policy. Pass a launched + * Playwright `browser` (deterministic tier) or omit it to launch Chromium. + */ +export async function runUserBot({ workspaceDir, key = null, policy = [], browser = null }) { + let ownBrowser = null; + let pw = null; + if (!browser) { + pw = await import('playwright'); + ownBrowser = await pw.chromium.launch({ headless: true }); + browser = ownBrowser; + } + const question = resolveQuestion(workspaceDir, key); + const context = await browser.newContext(); + const page = await context.newPage(); + await page.goto(question.url, { waitUntil: 'load' }); + await page.waitForSelector('button.choose', { timeout: 30000 }); + + const results = []; + let closed = false; + try { + for (const action of policy) { + const result = await runAction(page, action, { workspaceDir, key: question.key }); + results.push(result); + if (result.action === 'close') { closed = true; break; } + // Give the answer POST time to land before the process may exit. + if (result.action === 'pick' || result.action === 'canon') { + await page.waitForTimeout(300); + } + } + } finally { + if (!closed) await context.close().catch(() => {}); + if (ownBrowser) await ownBrowser.close().catch(() => {}); + } + return { key: question.key, url: question.url, results }; +} + +// -------------------------------------------------------------------------- +// CLI +// -------------------------------------------------------------------------- +function cliArg(name, fallback = null) { + const i = process.argv.indexOf(`--${name}`); + if (i === -1) return fallback; + const v = process.argv[i + 1]; + return v && !v.startsWith('--') ? v : fallback; +} + +const isMain = import.meta.url === `file://${process.argv[1]}`; +if (isMain) { + const workspaceDir = cliArg('workspace'); + const key = cliArg('key'); + const policyRaw = cliArg('policy'); + if (!workspaceDir || !policyRaw) { + console.error('user-bot: --workspace and --policy are required.'); + process.exit(1); + } + let policy; + try { + policy = JSON.parse(policyRaw); + } catch (err) { + console.error(`user-bot: --policy must be JSON (${err.message})`); + process.exit(1); + } + runUserBot({ workspaceDir, key, policy }) + .then((out) => { + console.log(JSON.stringify(out)); + process.exit(0); + }) + .catch((err) => { + console.error(`user-bot: ${err.message}`); + process.exit(1); + }); +}