mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-11 21:57:14 +03:00
Add deterministic new-work interactive smoke suite
A cheap, LLM-free E2E tier for the interactive parts of new-work, mirroring the two-layer live-e2e pattern (deterministic now, opt-in LLM tier later). - generate-image.mjs: IMPECCABLE_IMAGE_GEN_FAKE=1 writes a deterministic offline image (SVG with wrapped prompt + SYNTHETIC COMP label, or a valid palette-stripe PNG carrying the prompt/marker in a tEXt chunk). Same CLI contract, no key, no network, $0.00 cost line. - tests/new-work-e2e/user-bot.mjs: scripted user bot (module + CLI) that resolves the serve-question daemon from the workspace and drives the real page via Playwright (pick, re-roll + steer, canon, tab close). - tests/new-work-e2e.test.mjs: node --test coverage of the serve-question cycles (pick + CHOSEN CARD, re-roll + --update re-deal, canon + CANON CHOSEN, tab-close exit-4, text-only card) plus fake image determinism. - Registered as the opt-in new-work-e2e suite; added test:new-work-e2e. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Paul Bakaus
co-authored by
Claude Fable 5
Claude Fable 5
parent
bcdf38881e
commit
6ece0e588f
@@ -58,6 +58,7 @@
|
||||
"test:cli-remote-e2e": "node scripts/run-tests.mjs cli-remote-e2e",
|
||||
"test:live-e2e": "node scripts/run-tests.mjs live-e2e",
|
||||
"test:live-e2e-accept-cleanup": "node scripts/run-tests.mjs live-e2e-accept-cleanup",
|
||||
"test:new-work-e2e": "node scripts/run-tests.mjs new-work-e2e",
|
||||
"test:live-e2e-agent": "node scripts/run-tests.mjs live-e2e-agent",
|
||||
"test:skill-behavior": "node scripts/run-tests.mjs skill-behavior",
|
||||
"test:live-svelte-adapter-deepseek": "node scripts/run-tests.mjs live-svelte-adapter-deepseek",
|
||||
|
||||
@@ -6,6 +6,7 @@ export const OPT_IN_SUITES = [
|
||||
'cli-remote-e2e',
|
||||
'live-e2e',
|
||||
'live-e2e-accept-cleanup',
|
||||
'new-work-e2e',
|
||||
'skill-behavior',
|
||||
'live-svelte-adapter-deepseek',
|
||||
];
|
||||
@@ -222,6 +223,24 @@ export const SUITES = {
|
||||
},
|
||||
],
|
||||
},
|
||||
'new-work-e2e': {
|
||||
description: 'Playwright smoke sweep of the new-work concept/serve-question decision page plus the offline fake image generator.',
|
||||
optIn: true,
|
||||
needsPlaywright: true,
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^skill\/scripts\/(serve-question|generate-image|concept-seed)\.mjs$/,
|
||||
/^tests\/new-work-e2e(\.test\.mjs|\/)/,
|
||||
],
|
||||
commands: [
|
||||
{
|
||||
runner: 'node',
|
||||
timeoutMs: 600000,
|
||||
forceExit: true,
|
||||
files: ['tests/new-work-e2e.test.mjs'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'live-e2e-accept-cleanup': {
|
||||
description: 'Provider-backed post-accept cleanup regression.',
|
||||
optIn: true,
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
* node generate-image.mjs --prompt-file prompt.txt --out mock.png
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import zlib from 'node:zlib';
|
||||
|
||||
function arg(name, fallback = null) {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
@@ -20,6 +21,183 @@ function arg(name, fallback = null) {
|
||||
return v && !v.startsWith('--') ? v : fallback;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fake mode (IMPECCABLE_IMAGE_GEN_FAKE=1)
|
||||
//
|
||||
// Deterministic offline stand-in for the OpenAI call: same prompt -> identical
|
||||
// bytes, no network, no key, cost line reads $0.00. Used by the new-work smoke
|
||||
// suite so the concept/serve-question/image chain can run without spend. The
|
||||
// output renders the prompt over a 2-3 color palette hashed from the prompt,
|
||||
// plus a "SYNTHETIC COMP" corner label. SVG carries the readable text; the
|
||||
// raster (.png/.webp/.jpg) fallback carries palette stripes and stows the
|
||||
// prompt + marker in a PNG tEXt chunk so downstream stays a valid image.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// FNV-1a 32-bit: tiny, dependency-free, stable across runs and platforms.
|
||||
function hash32(str) {
|
||||
let h = 0x811c9dc5;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
h ^= str.charCodeAt(i);
|
||||
h = Math.imul(h, 0x01000193);
|
||||
}
|
||||
return h >>> 0;
|
||||
}
|
||||
|
||||
function hslToRgb(hDeg, s, l) {
|
||||
const h = ((hDeg % 360) + 360) % 360 / 360;
|
||||
const q = l < 0.5 ? l * (1 + s) : l + s - l * s;
|
||||
const p = 2 * l - q;
|
||||
const hue = (t) => {
|
||||
let tt = t;
|
||||
if (tt < 0) tt += 1;
|
||||
if (tt > 1) tt -= 1;
|
||||
if (tt < 1 / 6) return p + (q - p) * 6 * tt;
|
||||
if (tt < 1 / 2) return q;
|
||||
if (tt < 2 / 3) return p + (q - p) * (2 / 3 - tt) * 6;
|
||||
return p;
|
||||
};
|
||||
return [hue(h + 1 / 3), hue(h), hue(h - 1 / 3)].map((c) => Math.round(c * 255));
|
||||
}
|
||||
|
||||
const toHex = ([r, g, b]) =>
|
||||
'#' + [r, g, b].map((c) => c.toString(16).padStart(2, '0')).join('');
|
||||
|
||||
// Two or three deterministic swatches derived from the prompt hash. The band
|
||||
// count itself is prompt-derived, so different prompts differ in palette.
|
||||
function palette(prompt) {
|
||||
const h = hash32(prompt);
|
||||
const base = h % 360;
|
||||
const bands = 2 + (h >>> 9) % 2; // 2 or 3
|
||||
const spread = 40 + (h >>> 3) % 120;
|
||||
const out = [];
|
||||
for (let i = 0; i < bands; i++) {
|
||||
const hue = base + i * spread;
|
||||
const light = 0.32 + ((h >>> (i * 5)) % 40) / 100; // 0.32 - 0.71
|
||||
out.push(hslToRgb(hue, 0.55, light));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function svgFake(prompt, [w, h]) {
|
||||
const colors = palette(prompt).map(toHex);
|
||||
const stops = colors
|
||||
.map((c, i) => `<stop offset="${Math.round((i / (colors.length - 1)) * 100)}%" stop-color="${c}"/>`)
|
||||
.join('');
|
||||
// Greedy word wrap tuned to the canvas width so the prompt stays legible.
|
||||
const perLine = Math.max(12, Math.floor(w / 26));
|
||||
const words = String(prompt).replace(/\s+/g, ' ').trim().split(' ');
|
||||
const lines = [];
|
||||
let cur = '';
|
||||
for (const word of words) {
|
||||
if ((cur + ' ' + word).trim().length > perLine) {
|
||||
if (cur) lines.push(cur);
|
||||
cur = word;
|
||||
} else {
|
||||
cur = (cur + ' ' + word).trim();
|
||||
}
|
||||
if (lines.length >= 10) break;
|
||||
}
|
||||
if (cur && lines.length < 11) lines.push(cur);
|
||||
const escape = (s) => String(s).replace(/[&<>]/g, (c) => ({ '&': '&', '<': '<', '>': '>' }[c]));
|
||||
const fontSize = Math.round(w / 24);
|
||||
const startY = h / 2 - ((lines.length - 1) * fontSize * 1.3) / 2;
|
||||
const text = lines
|
||||
.map((line, i) => `<text x="${w / 2}" y="${Math.round(startY + i * fontSize * 1.3)}" font-family="Helvetica, Arial, sans-serif" font-size="${fontSize}" fill="#ffffff" text-anchor="middle" dominant-baseline="middle">${escape(line)}</text>`)
|
||||
.join('');
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="${w}" height="${h}" viewBox="0 0 ${w} ${h}">
|
||||
<defs><linearGradient id="g" x1="0" y1="0" x2="1" y2="1">${stops}</linearGradient></defs>
|
||||
<rect width="${w}" height="${h}" fill="url(#g)"/>
|
||||
<rect x="0" y="0" width="${w}" height="${h}" fill="#000000" fill-opacity="0.22"/>
|
||||
${text}
|
||||
<rect x="${w - Math.round(w / 4.2)}" y="${h - Math.round(h / 16)}" width="${Math.round(w / 4.2)}" height="${Math.round(h / 16)}" fill="#000000" fill-opacity="0.55"/>
|
||||
<text x="${w - Math.round(w / 8.4)}" y="${h - Math.round(h / 32)}" font-family="Helvetica, Arial, sans-serif" font-size="${Math.round(w / 60)}" letter-spacing="2" fill="#ffffff" text-anchor="middle" dominant-baseline="middle">SYNTHETIC COMP</text>
|
||||
</svg>
|
||||
`;
|
||||
}
|
||||
|
||||
// Minimal valid PNG: palette stripes plus a tEXt chunk carrying the marker and
|
||||
// prompt, so a .png/.webp fake stays a decodable image and still contains the
|
||||
// "SYNTHETIC" bytes downstream tools look for.
|
||||
function crc32(buf) {
|
||||
let c = 0xffffffff;
|
||||
for (let i = 0; i < buf.length; i++) {
|
||||
c ^= buf[i];
|
||||
for (let k = 0; k < 8; k++) c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1);
|
||||
}
|
||||
return (c ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function pngChunk(type, data) {
|
||||
const typeBuf = Buffer.from(type, 'latin1');
|
||||
const body = Buffer.concat([typeBuf, data]);
|
||||
const len = Buffer.alloc(4);
|
||||
len.writeUInt32BE(data.length, 0);
|
||||
const crc = Buffer.alloc(4);
|
||||
crc.writeUInt32BE(crc32(body), 0);
|
||||
return Buffer.concat([len, body, crc]);
|
||||
}
|
||||
|
||||
function pngFake(prompt, [w, h]) {
|
||||
const colors = palette(prompt); // [[r,g,b], ...]
|
||||
const bandH = Math.ceil(h / colors.length);
|
||||
// Raw image: each scanline prefixed with a 0 filter byte, RGB pixels.
|
||||
const stride = w * 3;
|
||||
const raw = Buffer.alloc(h * (stride + 1));
|
||||
for (let y = 0; y < h; y++) {
|
||||
const rowStart = y * (stride + 1);
|
||||
raw[rowStart] = 0;
|
||||
const [r, g, b] = colors[Math.min(colors.length - 1, Math.floor(y / bandH))];
|
||||
for (let x = 0; x < w; x++) {
|
||||
const p = rowStart + 1 + x * 3;
|
||||
raw[p] = r;
|
||||
raw[p + 1] = g;
|
||||
raw[p + 2] = b;
|
||||
}
|
||||
}
|
||||
const ihdr = Buffer.alloc(13);
|
||||
ihdr.writeUInt32BE(w, 0);
|
||||
ihdr.writeUInt32BE(h, 4);
|
||||
ihdr[8] = 8; // bit depth
|
||||
ihdr[9] = 2; // color type: truecolor RGB
|
||||
const idat = zlib.deflateSync(raw, { level: 9 });
|
||||
const textData = Buffer.concat([
|
||||
Buffer.from('Comment', 'latin1'),
|
||||
Buffer.from([0]),
|
||||
Buffer.from(`SYNTHETIC COMP: ${String(prompt).replace(/\s+/g, ' ').trim()}`, 'latin1'),
|
||||
]);
|
||||
return Buffer.concat([
|
||||
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
pngChunk('IHDR', ihdr),
|
||||
pngChunk('tEXt', textData),
|
||||
pngChunk('IDAT', idat),
|
||||
pngChunk('IEND', Buffer.alloc(0)),
|
||||
]);
|
||||
}
|
||||
|
||||
function parseSize(sizeStr) {
|
||||
const m = String(sizeStr).match(/^(\d+)x(\d+)$/);
|
||||
if (!m) return [1536, 1024];
|
||||
return [Number(m[1]), Number(m[2])];
|
||||
}
|
||||
|
||||
if (process.env.IMPECCABLE_IMAGE_GEN_FAKE) {
|
||||
const fakePromptFile = arg('prompt-file');
|
||||
const fakePrompt = fakePromptFile ? fs.readFileSync(fakePromptFile, 'utf8') : arg('prompt');
|
||||
const fakeOut = arg('out');
|
||||
if (!fakePrompt || !fakeOut) {
|
||||
console.error('generate-image: --prompt (or --prompt-file) and --out are required.');
|
||||
process.exit(1);
|
||||
}
|
||||
const dims = parseSize(arg('size', '1536x1024'));
|
||||
const bytes = fakeOut.endsWith('.svg')
|
||||
? Buffer.from(svgFake(fakePrompt, dims), 'utf8')
|
||||
: pngFake(fakePrompt, dims);
|
||||
fs.writeFileSync(fakeOut, bytes);
|
||||
console.log(`IMAGE: ${fakeOut} (${dims[0]}x${dims[1]}, fake synthetic comp, $0.00, no API call)`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const key = process.env.OPENAI_API_KEY;
|
||||
if (!key) {
|
||||
console.error('generate-image: OPENAI_API_KEY is not set; use the harness-native image tool instead.');
|
||||
|
||||
@@ -0,0 +1,357 @@
|
||||
/**
|
||||
* Deterministic smoke tests for the new-work interactive flow.
|
||||
*
|
||||
* Covers the parts a user actually touches: the serve-question decision page
|
||||
* (pick, re-roll + steer + re-deal, canon, tab close) driven through a real
|
||||
* browser by the scripted user bot, plus the offline fake image generator.
|
||||
* No LLM calls; a real Chromium via Playwright supplies full page fidelity
|
||||
* (heartbeats, re-roll reload, tab close). Kept OUT of `bun run test` like
|
||||
* live-e2e; run it with `bun run test:new-work-e2e`.
|
||||
*
|
||||
* The concept-seed direction roll (challengers, ASSIGNED INDEX, the no
|
||||
* PRODUCT.md gate) is already covered by tests/concept-seed.test.mjs and is
|
||||
* not repeated here.
|
||||
*
|
||||
* One-time setup: npx playwright install chromium
|
||||
*/
|
||||
|
||||
import { describe, it, before, after } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, writeFileSync, mkdirSync, readFileSync, existsSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { runUserBot } from './new-work-e2e/user-bot.mjs';
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const SERVE = path.join(ROOT, 'skill', 'scripts', 'serve-question.mjs');
|
||||
const GENERATE = path.join(ROOT, 'skill', 'scripts', 'generate-image.mjs');
|
||||
const CATALOG_DIR = path.join(ROOT, 'tests', 'fixtures', 'concept-catalog');
|
||||
|
||||
let playwright;
|
||||
let browser;
|
||||
|
||||
before(async () => {
|
||||
try {
|
||||
playwright = await import('playwright');
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`Playwright is required for new-work-e2e tests (${err.message}). Run: npx playwright install chromium`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
browser = await playwright.chromium.launch({ headless: true });
|
||||
} catch (err) {
|
||||
throw new Error(`Failed to launch Chromium (${err.message}). Run: npx playwright install chromium`);
|
||||
}
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
if (browser) await browser.close();
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Workspace + serve-question helpers
|
||||
// --------------------------------------------------------------------------
|
||||
function makeWorkspace() {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), 'new-work-e2e-'));
|
||||
writeFileSync(
|
||||
path.join(dir, 'PRODUCT.md'),
|
||||
'# Product\n\n## Register\n\nbrand\n\n## Platform\n\nweb\n',
|
||||
);
|
||||
return dir;
|
||||
}
|
||||
|
||||
// serve-question writes its state under cwd; run everything from the workspace.
|
||||
function run(args, cwd) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(process.execPath, [SERVE, ...args], {
|
||||
cwd,
|
||||
env: { ...process.env, IMPECCABLE_QUESTION_FORCE: '1', IMPECCABLE_CATALOG_DIR: CATALOG_DIR },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let out = '';
|
||||
let err = '';
|
||||
child.stdout.on('data', (c) => { out += c; });
|
||||
child.stderr.on('data', (c) => { err += c; });
|
||||
child.on('exit', (code) => resolve({ code, out, err }));
|
||||
});
|
||||
}
|
||||
|
||||
async function startDaemon(cwd, payload, key) {
|
||||
const payloadPath = path.join(cwd, `${key}.payload.json`);
|
||||
writeFileSync(payloadPath, JSON.stringify(payload));
|
||||
const started = await run(['--start', '--payload', payloadPath, '--no-open', '--key', key], cwd);
|
||||
assert.equal(started.code, 0, `--start failed: ${started.out} ${started.err}`);
|
||||
const url = started.out.match(/QUESTION URL: (\S+)/)?.[1];
|
||||
assert.ok(url, `no URL from --start: ${started.out}`);
|
||||
return { url, payloadPath };
|
||||
}
|
||||
|
||||
// Poll --wait until it settles on a terminal exit code (0 answered, 2 gone,
|
||||
// 4 page closed); loop while it reports WAITING (3).
|
||||
async function waitLoop(cwd, key, { poll = 30, max = 20 } = {}) {
|
||||
for (let i = 0; i < max; i++) {
|
||||
const res = await run(['--wait', '--key', key, '--poll', String(poll)], cwd);
|
||||
if (res.code !== 3) return res;
|
||||
}
|
||||
throw new Error('waitLoop exceeded max iterations');
|
||||
}
|
||||
|
||||
async function stopDaemon(cwd, key) {
|
||||
await run(['--stop', '--key', key], cwd).catch(() => {});
|
||||
}
|
||||
|
||||
function makeFakeImage(cwd, prompt, outName) {
|
||||
const out = path.join(cwd, outName);
|
||||
const res = spawnSyncGen(prompt, out);
|
||||
assert.equal(res.status, 0, `generate-image fake failed: ${res.stderr}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
function spawnSyncGen(prompt, out, size = null) {
|
||||
const args = [GENERATE, '--prompt', prompt, '--out', out];
|
||||
if (size) args.push('--size', size);
|
||||
return spawnSync(process.execPath, args, {
|
||||
env: { ...process.env, IMPECCABLE_IMAGE_GEN_FAKE: '1' },
|
||||
encoding: 'buffer',
|
||||
});
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// serve-question interactive cycles
|
||||
// --------------------------------------------------------------------------
|
||||
describe('new-work-e2e: serve-question decision page', () => {
|
||||
it('(a) pick assigned returns the option, hero/board fields, and the CHOSEN CARD directive', async () => {
|
||||
const cwd = makeWorkspace();
|
||||
const key = 'pick';
|
||||
const hero = makeFakeImage(cwd, 'Fillmore handbill hero', 'hero.png');
|
||||
const board = makeFakeImage(cwd, 'Fillmore handbill board', 'board.png');
|
||||
const payload = {
|
||||
title: 'Choose the visual world',
|
||||
question: 'The roll assigned Fillmore Handbill.',
|
||||
options: [
|
||||
{ id: 'assigned', label: 'Fillmore Handbill', kicker: 'THE ROLL', hero, board },
|
||||
{ id: 'challenger-teletext', label: 'Teletext Service', body: 'block-mosaic pages' },
|
||||
],
|
||||
reroll: true,
|
||||
canon: true,
|
||||
steer: true,
|
||||
};
|
||||
await startDaemon(cwd, payload, key);
|
||||
try {
|
||||
const bot = await runUserBot({
|
||||
workspaceDir: cwd, key, browser,
|
||||
policy: [{ pick: 'assigned', steer: 'warmer palette' }],
|
||||
});
|
||||
assert.equal(bot.results[0].action, 'pick');
|
||||
const collected = await waitLoop(cwd, key);
|
||||
assert.equal(collected.code, 0, collected.out);
|
||||
assert.match(collected.out, /ANSWER: /);
|
||||
const answer = JSON.parse(collected.out.match(/ANSWER: (\{.*\})/)[1]);
|
||||
assert.equal(answer.optionId, 'assigned');
|
||||
assert.equal(answer.steer, 'warmer palette');
|
||||
assert.ok(answer.hero, 'answer carries the chosen hero path');
|
||||
assert.ok(answer.board, 'answer carries the chosen board path');
|
||||
assert.match(collected.out, /CHOSEN CARD:/);
|
||||
} finally {
|
||||
await stopDaemon(cwd, key);
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('(b) re-roll with steer keeps the server alive; --update re-deals; the next pick is terminal', async () => {
|
||||
const cwd = makeWorkspace();
|
||||
const key = 'reroll';
|
||||
const payload1 = {
|
||||
title: 'Choose the visual world',
|
||||
options: [
|
||||
{ id: 'assigned', label: 'First Hand', kicker: 'THE ROLL' },
|
||||
{ id: 'challenger-a', label: 'Alt One' },
|
||||
],
|
||||
reroll: true, steer: true, canon: true,
|
||||
};
|
||||
const payload2 = {
|
||||
title: 'Choose the visual world',
|
||||
options: [
|
||||
{ id: 'assigned', label: 'Second Hand', kicker: 'THE ROLL' },
|
||||
{ id: 'challenger-b', label: 'Alt Two' },
|
||||
],
|
||||
reroll: true, steer: true,
|
||||
};
|
||||
await startDaemon(cwd, payload1, key);
|
||||
try {
|
||||
// Bot drives the whole page: re-roll (with steer) then, after the page
|
||||
// reloads into the next hand, pick the assigned card.
|
||||
const botPromise = runUserBot({
|
||||
workspaceDir: cwd, key, browser,
|
||||
policy: [{ reroll: true, steer: 'colder, more restraint' }, { pick: 'assigned' }],
|
||||
});
|
||||
|
||||
// First answer: the re-roll. Server must stay alive afterwards.
|
||||
const first = await waitLoop(cwd, key);
|
||||
assert.equal(first.code, 0, first.out);
|
||||
assert.match(first.out, /"optionId":"reroll"/);
|
||||
assert.match(first.out, /colder, more restraint/);
|
||||
assert.ok(existsSync(path.join(cwd, '.impeccable', 'questions', `${key}.state.json`)),
|
||||
'server state file survives a re-roll');
|
||||
|
||||
// Deliver the next hand; the live page reloads itself.
|
||||
const nextPayloadPath = path.join(cwd, 'next.json');
|
||||
writeFileSync(nextPayloadPath, JSON.stringify(payload2));
|
||||
const updated = await run(['--update', '--key', key, '--payload', nextPayloadPath], cwd);
|
||||
assert.equal(updated.code, 0, updated.out);
|
||||
|
||||
// Second answer: the terminal pick on the re-dealt hand.
|
||||
const second = await waitLoop(cwd, key);
|
||||
assert.equal(second.code, 0, second.out);
|
||||
assert.match(second.out, /"optionId":"assigned"/);
|
||||
|
||||
const bot = await botPromise;
|
||||
assert.equal(bot.results[0].action, 'reroll');
|
||||
assert.ok(bot.results[0].reloaded, 'page reloaded into the next hand');
|
||||
assert.equal(bot.results[1].action, 'pick');
|
||||
|
||||
// Terminal pick cleans the state file up.
|
||||
assert.ok(!existsSync(path.join(cwd, '.impeccable', 'questions', `${key}.state.json`)),
|
||||
'terminal pick removes the server state file');
|
||||
} finally {
|
||||
await stopDaemon(cwd, key);
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('(c) canon click returns optionId canon and prints the CANON CHOSEN directive', async () => {
|
||||
const cwd = makeWorkspace();
|
||||
const key = 'canon';
|
||||
const payload = {
|
||||
title: 'Choose the visual world',
|
||||
options: [{ id: 'assigned', label: 'Fillmore Handbill', kicker: 'THE ROLL' }],
|
||||
reroll: true, canon: true, steer: true,
|
||||
};
|
||||
await startDaemon(cwd, payload, key);
|
||||
try {
|
||||
await runUserBot({ workspaceDir: cwd, key, browser, policy: [{ canon: true }] });
|
||||
const collected = await waitLoop(cwd, key);
|
||||
assert.equal(collected.code, 0, collected.out);
|
||||
assert.match(collected.out, /"optionId":"canon"/);
|
||||
assert.match(collected.out, /CANON CHOSEN:/);
|
||||
} finally {
|
||||
await stopDaemon(cwd, key);
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('(d) closing the tab makes --wait exit 4 PAGE CLOSED', async () => {
|
||||
const cwd = makeWorkspace();
|
||||
const key = 'close';
|
||||
const payload = {
|
||||
title: 'Choose the visual world',
|
||||
options: [{ id: 'assigned', label: 'Fillmore Handbill', kicker: 'THE ROLL' }],
|
||||
reroll: true, steer: true,
|
||||
};
|
||||
await startDaemon(cwd, payload, key);
|
||||
try {
|
||||
const bot = await runUserBot({ workspaceDir: cwd, key, browser, policy: [{ close: true }] });
|
||||
assert.equal(bot.results[0].action, 'close');
|
||||
assert.ok(bot.results[0].beat > 0, 'a heartbeat landed before the tab closed');
|
||||
// --wait must observe the stale heartbeat and report the closed page.
|
||||
const res = await run(['--wait', '--key', key, '--poll', '30'], cwd);
|
||||
assert.equal(res.code, 4, `expected exit 4, got ${res.code}: ${res.out}`);
|
||||
assert.match(res.out, /PAGE CLOSED/);
|
||||
} finally {
|
||||
await stopDaemon(cwd, key);
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('(e) an option with no hero renders a text-only card (no .media element)', async () => {
|
||||
const cwd = makeWorkspace();
|
||||
const key = 'textonly';
|
||||
const hero = makeFakeImage(cwd, 'has a hero', 'hero.png');
|
||||
const payload = {
|
||||
title: 'Choose the visual world',
|
||||
options: [
|
||||
{ id: 'assigned', label: 'Text Only Direction', body: 'a grounded direction, no comp' },
|
||||
{ id: 'challenger-hero', label: 'Has A Card', hero },
|
||||
],
|
||||
reroll: true, steer: true,
|
||||
};
|
||||
const { url } = await startDaemon(cwd, payload, key);
|
||||
try {
|
||||
const context = await browser.newContext();
|
||||
const page = await context.newPage();
|
||||
await page.goto(url, { waitUntil: 'load' });
|
||||
await page.waitForSelector('button.choose');
|
||||
const textOnlyMedia = await page.$('.card[data-id="assigned"] .media');
|
||||
const heroMedia = await page.$('.card[data-id="challenger-hero"] .media');
|
||||
const textOnlyFace = await page.$('.card[data-id="assigned"] .face.text-only');
|
||||
await context.close();
|
||||
assert.equal(textOnlyMedia, null, 'text-only card has no .media region');
|
||||
assert.ok(textOnlyFace, 'text-only card carries the .text-only face class');
|
||||
assert.ok(heroMedia, 'the hero card still renders its .media region');
|
||||
} finally {
|
||||
await stopDaemon(cwd, key);
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Fake image generation
|
||||
// --------------------------------------------------------------------------
|
||||
describe('new-work-e2e: fake image generation', () => {
|
||||
it('is deterministic per prompt and encodes the SYNTHETIC marker', () => {
|
||||
const cwd = mkdtempSync(path.join(tmpdir(), 'new-work-img-'));
|
||||
try {
|
||||
const a = path.join(cwd, 'a.png');
|
||||
const b = path.join(cwd, 'b.png');
|
||||
const r1 = spawnSyncGen('Fillmore psychedelic handbill, warm ink', a);
|
||||
const r2 = spawnSyncGen('Fillmore psychedelic handbill, warm ink', b);
|
||||
assert.equal(r1.status, 0, r1.stderr?.toString());
|
||||
assert.equal(r2.status, 0, r2.stderr?.toString());
|
||||
assert.ok(existsSync(a) && existsSync(b), 'both files exist');
|
||||
assert.match(r1.stdout.toString(), /\$0\.00/, 'cost line reads $0.00');
|
||||
const bytesA = readFileSync(a);
|
||||
const bytesB = readFileSync(b);
|
||||
assert.ok(bytesA.equals(bytesB), 'same prompt yields identical bytes');
|
||||
// Valid PNG signature + the SYNTHETIC marker (in the tEXt chunk).
|
||||
assert.equal(bytesA.slice(0, 8).toString('hex'), '89504e470d0a1a0a');
|
||||
assert.ok(bytesA.includes(Buffer.from('SYNTHETIC')), 'PNG carries the SYNTHETIC marker');
|
||||
} finally {
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('renders a different palette for a different prompt', () => {
|
||||
const cwd = mkdtempSync(path.join(tmpdir(), 'new-work-img-'));
|
||||
try {
|
||||
const a = path.join(cwd, 'a.png');
|
||||
const c = path.join(cwd, 'c.png');
|
||||
spawnSyncGen('Fillmore psychedelic handbill, warm ink', a);
|
||||
spawnSyncGen('Teletext broadcast mosaic, cold blue', c);
|
||||
const bytesA = readFileSync(a);
|
||||
const bytesC = readFileSync(c);
|
||||
assert.ok(!bytesA.equals(bytesC), 'different prompts produce different images');
|
||||
} finally {
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('the SVG variant carries the readable prompt text and SYNTHETIC COMP label', () => {
|
||||
const cwd = mkdtempSync(path.join(tmpdir(), 'new-work-img-'));
|
||||
try {
|
||||
const svg = path.join(cwd, 'comp.svg');
|
||||
const res = spawnSyncGen('teletext broadcast mosaic', svg, '800x600');
|
||||
assert.equal(res.status, 0, res.stderr?.toString());
|
||||
const text = readFileSync(svg, 'utf8');
|
||||
assert.match(text, /^<\?xml/, 'is an SVG document');
|
||||
assert.match(text, /SYNTHETIC COMP/);
|
||||
assert.match(text, /teletext/i, 'the prompt text is rendered');
|
||||
} finally {
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
# new-work E2E
|
||||
|
||||
A cheap, deterministic smoke suite for the interactive parts of new-work: the
|
||||
serve-question decision page and the offline image generator. It is kept out of
|
||||
`bun run test` and runs on demand.
|
||||
|
||||
```bash
|
||||
bun run test:new-work-e2e
|
||||
```
|
||||
|
||||
One-time setup: `npx playwright install chromium` (the suite drives a real
|
||||
Chromium so the page runs its own JS, exactly as a user's tab would).
|
||||
|
||||
## What it covers
|
||||
|
||||
`tests/new-work-e2e.test.mjs` opens the served decision page with a real
|
||||
browser and drives it through the scripted user bot, then asserts on the
|
||||
serve-question protocol output:
|
||||
|
||||
- **pick assigned** returns the chosen `optionId`, the typed steer, the
|
||||
`hero`/`board` fields, and the `CHOSEN CARD` directive printed by `--wait`.
|
||||
- **re-roll with steer** keeps the daemon alive, `--update` re-deals the next
|
||||
hand, the page reloads itself, and the following pick is terminal (state file
|
||||
cleaned up).
|
||||
- **canon** returns `optionId: canon` and prints the `CANON CHOSEN` directive.
|
||||
- **tab close** stops the page heartbeats so `--wait` exits 4 `PAGE CLOSED`.
|
||||
- **text-only card** renders with no `.media` region when an option has no hero.
|
||||
- **fake image generation**: same prompt yields identical bytes, the file
|
||||
exists, the `SYNTHETIC` marker is present, and different prompts produce
|
||||
different palettes.
|
||||
|
||||
The concept-seed direction roll (challengers, `ASSIGNED INDEX`, the no
|
||||
PRODUCT.md gate) is already covered by `tests/concept-seed.test.mjs` and is not
|
||||
repeated here.
|
||||
|
||||
## Pieces
|
||||
|
||||
- `user-bot.mjs` is a module plus CLI. Given a workspace dir it resolves the
|
||||
running daemon from `.impeccable/questions/<key>.state.json`, opens the page,
|
||||
and runs a JSON policy of real clicks: `{"pick":"assigned"}`,
|
||||
`{"reroll":true,"steer":"warmer"}`, `{"pick":"challenger-*"}`,
|
||||
`{"canon":true}`, `{"close":true}`. The deterministic tier passes an
|
||||
already-launched browser in; the CLI launches its own Chromium.
|
||||
- `IMPECCABLE_IMAGE_GEN_FAKE=1` switches `skill/scripts/generate-image.mjs` to
|
||||
the offline stand-in: no OpenAI call, no key, a `$0.00` cost line, and a
|
||||
deterministic image (SVG for `.svg` out with the wrapped prompt text and a
|
||||
`SYNTHETIC COMP` label; a valid palette-stripe PNG otherwise, with the prompt
|
||||
and marker in a PNG `tEXt` chunk).
|
||||
|
||||
## Planned LLM tier (not built yet)
|
||||
|
||||
The same scaffolding supports an opt-in LLM tier later, mirroring the two-layer
|
||||
pattern in `tests/live-e2e`:
|
||||
|
||||
- A real model plays the user through the same scripted `user-bot.mjs` policy,
|
||||
choosing and steering instead of following canned actions.
|
||||
- `IMPECCABLE_IMAGE_GEN_FAKE` still stands in for image spend, so a full
|
||||
concept-to-card cycle runs without paying per render.
|
||||
- Assertions run against the tool-call trace via the skill-behavior harness,
|
||||
the same way `tests/skill-behavior` keys on the trace rather than free-form
|
||||
output.
|
||||
|
||||
Cost posture: the deterministic tier is free (no API calls, local Chromium).
|
||||
The LLM tier hits a provider and costs money, so it stays opt-in and out of CI,
|
||||
matching how `test:live-e2e` and `test:skill-behavior` are gated today.
|
||||
@@ -0,0 +1,196 @@
|
||||
/**
|
||||
* Scripted user bot for the new-work interactive smoke suite.
|
||||
*
|
||||
* Given a workspace directory, it discovers a running serve-question daemon
|
||||
* from `.impeccable/questions/<key>.state.json`, opens the served page in a
|
||||
* real browser, and drives it through a scripted policy: it clicks the real
|
||||
* `button.choose`, `#reroll`, and `#canon` controls, types into `#steer`, and
|
||||
* closes the tab for the exit-4 path. Because a real page runs the page's own
|
||||
* JS, heartbeats fire and re-roll reloads behave exactly as a user's would.
|
||||
*
|
||||
* The deterministic tier passes an already-launched Playwright browser in.
|
||||
* Run as a CLI (`--workspace DIR --policy '<json>'`) it launches its own
|
||||
* Chromium. The policy is an ordered list of actions:
|
||||
*
|
||||
* { "pick": "assigned" } click the assigned card
|
||||
* { "pick": "challenger-*" } click the first matching card
|
||||
* { "pickIndex": 1 } click the Nth choose button
|
||||
* { "reroll": true, "steer": "warmer" } type the steer, click Re-roll
|
||||
* { "canon": true } click Play it straight
|
||||
* { "close": true } close the tab (stops heartbeats)
|
||||
*
|
||||
* A `steer` on any action is typed into `#steer` first when the field exists.
|
||||
* After a re-roll the bot waits for the page to reload into the next hand
|
||||
* (delivered out of band by `serve-question --update`) before the next action.
|
||||
*/
|
||||
|
||||
import { readdirSync, readFileSync, existsSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
function questionsDir(workspaceDir) {
|
||||
return path.join(workspaceDir, '.impeccable', 'questions');
|
||||
}
|
||||
|
||||
// Resolve the served URL from the daemon state file. When no key is given and
|
||||
// several exist, the newest wins.
|
||||
export function resolveQuestion(workspaceDir, key = null) {
|
||||
const dir = questionsDir(workspaceDir);
|
||||
if (!existsSync(dir)) throw new Error(`no questions dir at ${dir}`);
|
||||
const stateFiles = readdirSync(dir).filter((f) => f.endsWith('.state.json'));
|
||||
if (stateFiles.length === 0) throw new Error(`no *.state.json in ${dir}`);
|
||||
let file;
|
||||
if (key) {
|
||||
file = `${key}.state.json`;
|
||||
if (!stateFiles.includes(file)) throw new Error(`no state file for key ${key}`);
|
||||
} else {
|
||||
file = stateFiles
|
||||
.map((f) => ({ f, mtime: readFileSync(path.join(dir, f), 'utf8') && f }))
|
||||
.sort()
|
||||
.pop().f;
|
||||
}
|
||||
const resolvedKey = file.replace(/\.state\.json$/, '');
|
||||
const state = JSON.parse(readFileSync(path.join(dir, file), 'utf8'));
|
||||
return { key: resolvedKey, url: state.url, port: state.port, pid: state.pid };
|
||||
}
|
||||
|
||||
function stateLastBeat(workspaceDir, key) {
|
||||
try {
|
||||
const state = JSON.parse(readFileSync(path.join(questionsDir(workspaceDir), `${key}.state.json`), 'utf8'));
|
||||
return state.lastBeat || 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async function typeSteer(page, action) {
|
||||
if (action.steer == null) return;
|
||||
const steer = await page.$('#steer');
|
||||
if (steer) await steer.fill(String(action.steer));
|
||||
}
|
||||
|
||||
function chooseSelector(pick) {
|
||||
if (pick.endsWith('*')) {
|
||||
const prefix = pick.slice(0, -1);
|
||||
return `button.choose[data-id^="${prefix}"]`;
|
||||
}
|
||||
return `button.choose[data-id="${pick}"]`;
|
||||
}
|
||||
|
||||
async function runAction(page, action, { workspaceDir, key }) {
|
||||
await typeSteer(page, action);
|
||||
|
||||
if (action.reroll) {
|
||||
await Promise.all([
|
||||
page.waitForNavigation({ waitUntil: 'load', timeout: 60000 }).catch(() => {}),
|
||||
page.click('#reroll'),
|
||||
]);
|
||||
// Fresh hand loaded: wait for the interactive controls of the next round.
|
||||
await page.waitForSelector('button.choose', { timeout: 30000 });
|
||||
return { action: 'reroll', reloaded: true };
|
||||
}
|
||||
|
||||
if (action.canon) {
|
||||
await page.click('#canon');
|
||||
return { action: 'canon' };
|
||||
}
|
||||
|
||||
if (action.close) {
|
||||
// Make sure at least one heartbeat has been recorded so the --wait poll can
|
||||
// later see the beat go stale (the exit-4 PAGE CLOSED path).
|
||||
const deadline = Date.now() + 8000;
|
||||
while (Date.now() < deadline && !stateLastBeat(workspaceDir, key)) {
|
||||
await page.waitForTimeout(200);
|
||||
}
|
||||
await page.close();
|
||||
return { action: 'close', beat: stateLastBeat(workspaceDir, key) };
|
||||
}
|
||||
|
||||
if (action.pickIndex != null) {
|
||||
const buttons = await page.$$('button.choose');
|
||||
const btn = buttons[action.pickIndex];
|
||||
if (!btn) throw new Error(`no choose button at index ${action.pickIndex}`);
|
||||
await btn.click();
|
||||
return { action: 'pick', index: action.pickIndex };
|
||||
}
|
||||
|
||||
if (action.pick) {
|
||||
await page.click(chooseSelector(action.pick));
|
||||
return { action: 'pick', id: action.pick };
|
||||
}
|
||||
|
||||
throw new Error(`unknown action: ${JSON.stringify(action)}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive the served question page through the policy. Pass a launched
|
||||
* Playwright `browser` (deterministic tier) or omit it to launch Chromium.
|
||||
*/
|
||||
export async function runUserBot({ workspaceDir, key = null, policy = [], browser = null }) {
|
||||
let ownBrowser = null;
|
||||
let pw = null;
|
||||
if (!browser) {
|
||||
pw = await import('playwright');
|
||||
ownBrowser = await pw.chromium.launch({ headless: true });
|
||||
browser = ownBrowser;
|
||||
}
|
||||
const question = resolveQuestion(workspaceDir, key);
|
||||
const context = await browser.newContext();
|
||||
const page = await context.newPage();
|
||||
await page.goto(question.url, { waitUntil: 'load' });
|
||||
await page.waitForSelector('button.choose', { timeout: 30000 });
|
||||
|
||||
const results = [];
|
||||
let closed = false;
|
||||
try {
|
||||
for (const action of policy) {
|
||||
const result = await runAction(page, action, { workspaceDir, key: question.key });
|
||||
results.push(result);
|
||||
if (result.action === 'close') { closed = true; break; }
|
||||
// Give the answer POST time to land before the process may exit.
|
||||
if (result.action === 'pick' || result.action === 'canon') {
|
||||
await page.waitForTimeout(300);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (!closed) await context.close().catch(() => {});
|
||||
if (ownBrowser) await ownBrowser.close().catch(() => {});
|
||||
}
|
||||
return { key: question.key, url: question.url, results };
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// CLI
|
||||
// --------------------------------------------------------------------------
|
||||
function cliArg(name, fallback = null) {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
if (i === -1) return fallback;
|
||||
const v = process.argv[i + 1];
|
||||
return v && !v.startsWith('--') ? v : fallback;
|
||||
}
|
||||
|
||||
const isMain = import.meta.url === `file://${process.argv[1]}`;
|
||||
if (isMain) {
|
||||
const workspaceDir = cliArg('workspace');
|
||||
const key = cliArg('key');
|
||||
const policyRaw = cliArg('policy');
|
||||
if (!workspaceDir || !policyRaw) {
|
||||
console.error('user-bot: --workspace <dir> and --policy <json> are required.');
|
||||
process.exit(1);
|
||||
}
|
||||
let policy;
|
||||
try {
|
||||
policy = JSON.parse(policyRaw);
|
||||
} catch (err) {
|
||||
console.error(`user-bot: --policy must be JSON (${err.message})`);
|
||||
process.exit(1);
|
||||
}
|
||||
runUserBot({ workspaceDir, key, policy })
|
||||
.then((out) => {
|
||||
console.log(JSON.stringify(out));
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error(`user-bot: ${err.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user