Tests: run new-work-e2e through the engine's serve-question and generate-image verbs

Prepared with AI assistance (Claude Code).
This commit is contained in:
Paul Bakaus
2026-08-31 20:00:04 -07:00
parent 166a78ad3d
commit 77eeecd26b
2 changed files with 21 additions and 68 deletions
+14 -65
View File
@@ -9,8 +9,12 @@
* live-e2e; run it with `bun run test:new-work-e2e`.
*
* The concept-seed direction roll (challengers, ASSIGNED INDEX, the no
* PRODUCT.md gate) is already covered by tests/concept-seed.test.mjs and is
* not repeated here.
* PRODUCT.md gate) is pinned by the oracle corpus (tests/oracle, `seed-*`
* cases) and is not repeated here.
*
* Both verbs (`serve-question`, `generate-image`) run through the engine
* binary from tests/lib/engine-bin.mjs (IMPECCABLE_BIN or
* skill/scripts/bin/<os>-<arch>/); the suite fails loudly without one.
*
* One-time setup: npx playwright install chromium
*/
@@ -24,17 +28,17 @@ import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { runUserBot } from './new-work-e2e/user-bot.mjs';
import { ENGINE_MISSING_MESSAGE, engineEnv, findEngineBinary } from './lib/engine-bin.mjs';
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const SERVE = path.join(ROOT, 'skill', 'scripts', 'serve-question.mjs');
const GENERATE = path.join(ROOT, 'skill', 'scripts', 'generate-image.mjs');
const EMBED_PROMPT = path.join(ROOT, 'skill', 'scripts', 'embed-prompt.mjs');
const ENGINE_BIN = findEngineBinary();
const CATALOG_DIR = path.join(ROOT, 'tests', 'fixtures', 'concept-catalog');
let playwright;
let browser;
before(async () => {
if (!ENGINE_BIN) throw new Error(ENGINE_MISSING_MESSAGE);
try {
playwright = await import('playwright');
} catch (err) {
@@ -68,9 +72,9 @@ function makeWorkspace() {
// serve-question writes its state under cwd; run everything from the workspace.
function run(args, cwd) {
return new Promise((resolve) => {
const child = spawn(process.execPath, [SERVE, ...args], {
const child = spawn(ENGINE_BIN, ['serve-question', ...args], {
cwd,
env: { ...process.env, IMPECCABLE_QUESTION_FORCE: '1', IMPECCABLE_CATALOG_DIR: CATALOG_DIR },
env: engineEnv(ENGINE_BIN, { IMPECCABLE_QUESTION_FORCE: '1', IMPECCABLE_CATALOG_DIR: CATALOG_DIR }),
stdio: ['ignore', 'pipe', 'pipe'],
});
let out = '';
@@ -113,18 +117,14 @@ function makeFakeImage(cwd, prompt, outName) {
}
function spawnSyncGen(prompt, out, size = null) {
const args = [GENERATE, '--prompt', prompt, '--out', out];
const args = ['generate-image', '--prompt', prompt, '--out', out];
if (size) args.push('--size', size);
return spawnSync(process.execPath, args, {
env: { ...process.env, IMPECCABLE_IMAGE_GEN_FAKE: '1' },
return spawnSync(ENGINE_BIN, args, {
env: engineEnv(ENGINE_BIN, { IMPECCABLE_IMAGE_GEN_FAKE: '1' }),
encoding: 'buffer',
});
}
function spawnSyncEmbed(args) {
return spawnSync(process.execPath, [EMBED_PROMPT, ...args], { encoding: 'utf8' });
}
// --------------------------------------------------------------------------
// serve-question interactive cycles
// --------------------------------------------------------------------------
@@ -1018,57 +1018,6 @@ describe('new-work-e2e: fake image generation', () => {
}
});
it('reads, scans, and idempotently replaces the prompt embedded in a PNG', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'new-work-img-'));
try {
const image = makeFakeImage(cwd, 'synthetic IEND source prompt', 'comp.png');
const original = readFileSync(image);
assert.ok(original.indexOf(Buffer.from('IEND')) < original.lastIndexOf(Buffer.from('IEND')),
'the fixture carries IEND bytes in metadata before the real terminator chunk');
const first = spawnSyncEmbed([image, '--prompt', 'first production prompt']);
assert.equal(first.status, 0, first.stderr);
assert.equal(spawnSyncEmbed([image, '--read']).stdout.trim(), 'first production prompt');
const scan = spawnSyncEmbed(['--scan', cwd]);
assert.equal(scan.status, 0, scan.stderr);
assert.match(scan.stdout, /SCAN: 1 raster, 0 missing/);
const second = spawnSyncEmbed([image, '--prompt', 'replacement production prompt']);
assert.equal(second.status, 0, second.stderr);
assert.equal(spawnSyncEmbed([image, '--read']).stdout.trim(), 'replacement production prompt');
const bytes = readFileSync(image);
assert.equal(bytes.toString().match(/impeccable:prompt/g)?.length, 1,
're-embedding replaces the existing metadata instead of accumulating chunks');
assert.ok(bytes.includes(Buffer.from('SYNTHETIC')),
'replacing the prompt preserves unrelated PNG metadata');
} finally {
rmSync(cwd, { recursive: true, force: true });
}
});
it('reads JPEG comments and sidecar fallbacks through the same scan contract', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'new-work-img-'));
try {
const jpeg = path.join(cwd, 'reference.jpg');
const webp = path.join(cwd, 'reference.webp');
writeFileSync(jpeg, Buffer.from([0xff, 0xd8, 0xff, 0xda, 0x00, 0x02]));
writeFileSync(webp, Buffer.from('RIFF placeholder WEBP'));
assert.equal(spawnSyncEmbed([jpeg, '--prompt', 'jpeg prompt']).status, 0);
assert.equal(spawnSyncEmbed([webp, '--prompt', 'sidecar prompt']).status, 0);
assert.equal(spawnSyncEmbed([jpeg, '--read']).stdout.trim(), 'jpeg prompt');
assert.equal(spawnSyncEmbed([webp, '--read']).stdout.trim(), 'sidecar prompt');
const scan = spawnSyncEmbed(['--scan', cwd]);
assert.equal(scan.status, 0, scan.stderr);
assert.match(scan.stdout, /SCAN: 2 rasters, 0 missing/);
} finally {
rmSync(cwd, { recursive: true, force: true });
}
});
it('the SVG variant carries the readable prompt text and SYNTHETIC COMP label', () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'new-work-img-'));
try {
+7 -3
View File
@@ -30,8 +30,12 @@ serve-question protocol output:
different palettes.
The concept-seed direction roll (challengers, `ASSIGNED INDEX`, the no
PRODUCT.md gate) is already covered by `tests/concept-seed.test.mjs` and is not
repeated here.
PRODUCT.md gate) is pinned by the oracle corpus (`tests/oracle`, the `seed-*`
cases) and is not repeated here.
Both verbs run through the engine binary (`impeccable serve-question`,
`impeccable generate-image`) resolved by `tests/lib/engine-bin.mjs`:
`IMPECCABLE_BIN` or `skill/scripts/bin/<os>-<arch>/` (`bun run fetch:engine`).
## Pieces
@@ -41,7 +45,7 @@ repeated here.
`{"reroll":true,"steer":"warmer"}`, `{"pick":"challenger-*"}`,
`{"canon":true}`, `{"close":true}`. The deterministic tier passes an
already-launched browser in; the CLI launches its own Chromium.
- `IMPECCABLE_IMAGE_GEN_FAKE=1` switches `skill/scripts/generate-image.mjs` to
- `IMPECCABLE_IMAGE_GEN_FAKE=1` switches `impeccable generate-image` to
the offline stand-in: no OpenAI call, no key, a `$0.00` cost line, and a
deterministic image (SVG for `.svg` out with the wrapped prompt text and a
`SYNTHETIC COMP` label; a valid palette-stripe PNG otherwise, with the prompt