mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-20 10:06:54 +03:00
Fix skill workflow regression coverage (#783)
Clarify launcher fallback and completed documentation handoffs; separate bounded protocol checkpoints from opt-in browser-backed completion diagnostics. Correct fixture containment, target syntax, and artifact assertions. AI assistance: Codex, under maintainer direction.
This commit is contained in:
Paul Bakaus
@@ -0,0 +1,50 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { sourceHash } from './source-hash.mjs';
|
||||
import { missingReferences } from '../skill-behavior/assertions.mjs';
|
||||
|
||||
export function assertDocumentationArtifacts(design, sidecarText) {
|
||||
const frontmatter = design.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/)?.[1];
|
||||
assert.ok(frontmatter, 'documentation must include machine-readable frontmatter, not prose alone');
|
||||
assert.match(frontmatter, /^colors:\s*\n[ \t]+\S/m, 'documentation must record color tokens');
|
||||
assert.match(frontmatter, /^typography:\s*\n[ \t]+\S/m, 'documentation must record typography tokens');
|
||||
const sidecar = JSON.parse(sidecarText);
|
||||
assert.equal(sidecar.schemaVersion, 2, 'documentation must write the v2 sidecar');
|
||||
for (const key of ['extensions', 'narrative']) {
|
||||
assert.ok(sidecar[key] && typeof sidecar[key] === 'object' && !Array.isArray(sidecar[key])
|
||||
&& Object.keys(sidecar[key]).length, `sidecar must contain ${key} metadata`);
|
||||
}
|
||||
}
|
||||
|
||||
// For a resumed, already-reviewed ordinary extension only. New worlds and
|
||||
// redesigns still owe real documentation writes; this is not an escape hatch.
|
||||
export function assertNoChangeDocumentation(result, { target, evidence }) {
|
||||
assertCompleted(result);
|
||||
const { trace, text } = result;
|
||||
assert.deepEqual(missingReferences(trace, ['reference/document.md', target, 'DESIGN.md']), [],
|
||||
'documentation must consult its contract and inspect the actual source and recorded system');
|
||||
assert.deepEqual(trace.toolCalls.flatMap((call) => call.mutatedPaths || []), [],
|
||||
'the resumed no-change check must not mutate project files');
|
||||
assert.match(text, /no (?:system |visual.system |documentation )?changes|unchanged|no rewrite/i,
|
||||
'documentation must explicitly report a no-change outcome');
|
||||
for (const filename of [target, 'DESIGN.md']) {
|
||||
assert.ok(text.includes(filename), `documentation must identify the checked ${filename}`);
|
||||
}
|
||||
for (const fact of evidence) {
|
||||
assert.match(text, fact, 'no-change documentation must report evidence from the fixture, not an unsupported completion claim');
|
||||
}
|
||||
}
|
||||
|
||||
export function assertCompleted(result) {
|
||||
assert.equal(result.outcome, 'complete', `workflow did not finish: ${result.outcome} after ${result.steps} steps`);
|
||||
}
|
||||
|
||||
export function assertFreshCaptures(trace, workspace, target) {
|
||||
const calls = trace.toolCalls;
|
||||
const lastEdit = calls.findLastIndex((call) => (call.mutatedPaths || []).includes(target));
|
||||
const hash = sourceHash(workspace);
|
||||
for (const viewport of ['desktop', 'mobile']) {
|
||||
assert.ok(calls.some((call, index) => index > lastEdit && call.capture?.target === target
|
||||
&& call.capture.viewport === viewport && call.capture.sourceHash === hash),
|
||||
`missing ${viewport} screenshot of the final ${target}; pre-edit captures do not count`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import { sourceHash as hashSources } from './source-hash.mjs';
|
||||
import { chromium } from 'playwright';
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
|
||||
const VIEWPORTS = { desktop: { width: 1440, height: 1000 }, mobile: { width: 390, height: 844 } };
|
||||
const TYPES = { '.html': 'text/html', '.css': 'text/css', '.js': 'text/javascript', '.mjs': 'text/javascript', '.svg': 'image/svg+xml', '.png': 'image/png', '.woff2': 'font/woff2' };
|
||||
const PNG = Buffer.from('89504e470d0a1a0a', 'hex');
|
||||
|
||||
function resolveFile(root, name) {
|
||||
if (path.isAbsolute(name)) throw new Error('Use a workspace-relative path');
|
||||
const file = path.resolve(root, name);
|
||||
const rel = path.relative(root, file);
|
||||
if (rel === '..' || rel.startsWith(`..${path.sep}`)) throw new Error('Path escapes workspace');
|
||||
const real = fs.realpathSync(file);
|
||||
const realRel = path.relative(fs.realpathSync(root), real);
|
||||
if (realRel === '..' || realRel.startsWith(`..${path.sep}`)) throw new Error('Path escapes workspace through a symlink');
|
||||
return real;
|
||||
}
|
||||
|
||||
export function imageOutput({ output }) {
|
||||
const { image, ...metadata } = output;
|
||||
return { type: 'content', value: [
|
||||
{ type: 'text', text: JSON.stringify(metadata) },
|
||||
{ type: 'file', mediaType: 'image/png', data: { type: 'data', data: image } },
|
||||
] };
|
||||
}
|
||||
|
||||
/** Preflight before any billed call; no runtime installs or browser discovery. */
|
||||
export async function prepareBrowser(root) {
|
||||
let browser;
|
||||
try {
|
||||
browser = await chromium.launch({ headless: true, timeout: 15000 });
|
||||
} catch (error) {
|
||||
throw new Error('Workflow browser preflight failed. Run `bunx playwright install chromium` before billed tests.', { cause: error });
|
||||
}
|
||||
const blockedRequests = [];
|
||||
const server = http.createServer((req, res) => {
|
||||
try {
|
||||
const name = decodeURIComponent(new URL(req.url, 'http://localhost').pathname).replace(/^\//, '') || 'index.html';
|
||||
if (name.split('/').some((part) => part.startsWith('.'))) throw new Error('Private workspace path');
|
||||
const file = resolveFile(root, name);
|
||||
if (!fs.statSync(file).isFile()) throw new Error('Not a file');
|
||||
res.writeHead(200, { 'Content-Type': TYPES[path.extname(file)] || 'application/octet-stream', 'Cache-Control': 'no-store' });
|
||||
res.end(fs.readFileSync(file));
|
||||
} catch (error) {
|
||||
res.writeHead(error.code === 'ENOENT' ? 404 : 403);
|
||||
res.end('Not available');
|
||||
}
|
||||
});
|
||||
try {
|
||||
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); });
|
||||
} catch (error) {
|
||||
await browser.close();
|
||||
throw error;
|
||||
}
|
||||
const origin = `http://127.0.0.1:${server.address().port}`;
|
||||
let context;
|
||||
try {
|
||||
context = await browser.newContext({ reducedMotion: 'reduce', serviceWorkers: 'block' });
|
||||
await context.route('**/*', (route) => {
|
||||
const url = route.request().url();
|
||||
if (new URL(url).origin === origin || url.startsWith('data:')) return route.continue();
|
||||
blockedRequests.push(url);
|
||||
return route.abort();
|
||||
});
|
||||
} catch (error) {
|
||||
await browser.close();
|
||||
await new Promise((resolve) => { server.close(resolve); server.closeAllConnections(); });
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
origin, blockedRequests,
|
||||
environment: `Workspace: ${root}. A local server and Chromium are already running. browser_snapshot renders a workspace-relative HTML path at desktop/mobile size, saves a screenshot, and returns the actual image plus DOM text. view_image opens saved PNGs. No browser installation is needed. External browser requests are blocked; this text-only fixture uses system fonts. No image-generation or subagent tools are available.`,
|
||||
tools(trace) {
|
||||
return {
|
||||
browser_snapshot: tool({
|
||||
description: 'Render and inspect an HTML file with the ready Chromium browser. Returns an actual screenshot and visible DOM text; optionally click a CSS selector before capture. Captures save to .impeccable/review/{desktop|mobile}.png.',
|
||||
inputSchema: z.object({ path: z.string(), viewport: z.enum(['desktop', 'mobile']), click: z.string().optional() }),
|
||||
execute: async ({ path: target, viewport, click }) => {
|
||||
const file = resolveFile(root, target);
|
||||
if (!/\.html?$/i.test(file)) throw new Error('Expected an HTML artifact');
|
||||
const relativeTarget = path.relative(fs.realpathSync(root), file).split(path.sep).join('/');
|
||||
const call = { name: 'browser_snapshot', input: { path: target, viewport, click }, mutatedPaths: [] };
|
||||
trace.toolCalls.push(call);
|
||||
const page = await context.newPage();
|
||||
page.setDefaultTimeout(10000);
|
||||
try {
|
||||
const sourceHash = hashSources(root);
|
||||
await page.setViewportSize(VIEWPORTS[viewport]);
|
||||
await page.goto(`${origin}/${relativeTarget.split('/').map(encodeURIComponent).join('/')}`, { waitUntil: 'load', timeout: 15000 });
|
||||
await page.evaluate(() => document.fonts.ready);
|
||||
if (click) await page.locator(click).click();
|
||||
const screenshot = `.impeccable/review/${viewport}.png`;
|
||||
if (fs.existsSync(path.join(root, '.impeccable'))) resolveFile(root, '.impeccable');
|
||||
fs.mkdirSync(path.join(root, '.impeccable/review'), { recursive: true });
|
||||
resolveFile(root, '.impeccable/review');
|
||||
if (fs.existsSync(path.join(root, screenshot))) resolveFile(root, screenshot);
|
||||
const image = await page.screenshot({ path: path.join(root, screenshot), fullPage: true, animations: 'disabled' });
|
||||
call.mutatedPaths = [screenshot];
|
||||
if (sourceHash !== hashSources(root)) throw new Error('Artifact changed during capture; retry');
|
||||
call.capture = { target: relativeTarget, viewport, screenshot, sourceHash };
|
||||
return { ...call.capture, text: (await page.locator('body').innerText()).slice(0, 12000), image: image.toString('base64') };
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
},
|
||||
toModelOutput: imageOutput,
|
||||
}),
|
||||
view_image: tool({
|
||||
description: 'Inspect an existing workspace PNG as an actual image, not raw file bytes.',
|
||||
inputSchema: z.object({ path: z.string() }),
|
||||
execute: async ({ path: name }) => {
|
||||
const bytes = fs.readFileSync(resolveFile(root, name));
|
||||
if (!bytes.subarray(0, 8).equals(PNG)) throw new Error('Expected a PNG image');
|
||||
trace.toolCalls.push({ name: 'view_image', input: { path: name }, mutatedPaths: [], loadedImages: [name] });
|
||||
return { path: name, image: bytes.toString('base64') };
|
||||
},
|
||||
toModelOutput: imageOutput,
|
||||
}),
|
||||
};
|
||||
},
|
||||
async close() {
|
||||
await browser.close();
|
||||
await new Promise((resolve) => { server.close(resolve); server.closeAllConnections(); });
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { prepareWorkspace, cleanupWorkspace, runTurn, fileLoaded, ENGINE_BIN } from '../skill-behavior/harness.mjs';
|
||||
import { getModel, detectProvider, hasKey } from '../skill-behavior/providers.mjs';
|
||||
import { assertCompleted, assertNoChangeDocumentation, assertDocumentationArtifacts } from './assertions.mjs';
|
||||
import { missingReferences } from '../skill-behavior/assertions.mjs';
|
||||
|
||||
// A synthetic post-review checkpoint, not another full-build simulation.
|
||||
// The page and system agree. A missing sidecar predates this task and is not
|
||||
// permission to repair drift or rewrite the incumbent DESIGN.md.
|
||||
const DESIGN = `# Field Manual
|
||||
|
||||
## Overview
|
||||
An established, plain reading surface. Preserve this identity.
|
||||
|
||||
## Colors
|
||||
White background (#ffffff), near-black text (#222222), blue links (#0645ad).
|
||||
|
||||
## Typography
|
||||
System-ui body at 16px, line-height 1.6. Headings at 24px, weight 700.
|
||||
|
||||
## Layout
|
||||
One column, max-width 65ch, padding 24px; no decorative containers.
|
||||
`;
|
||||
const PAGE = '<!doctype html><html lang="en"><meta charset="utf-8"><title>Keyboard guide</title><style>body{background:#fff;color:#222;font:16px/1.6 system-ui;max-width:65ch;margin:auto;padding:24px}h1{font-size:24px;font-weight:700}a{color:#0645ad}</style><main><h1>Keyboard guide</h1><p>Use Tab to move between controls. Press Enter to activate a link.</p><a href="#top" id="top">Back to top</a></main></html>';
|
||||
const BRIEF = '# Keyboard guide\n\n## Direction contract\nTHESIS: A short reading page.\nOWN-WORLD: Inherit Field Manual.\nSTORY: Read keyboard instructions.\nFIRST VIEWPORT: Title, paragraph, link.\nFORM: Direct, precisely specified page; no seed required.\nFINISH: unreviewed and undocumented is unfinished.\n';
|
||||
|
||||
for (const modelId of (process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS || 'claude-sonnet-5').split(',').map((id) => id.trim()).filter(Boolean)) {
|
||||
for (const mode of ['extension', 'new world', 'redesign']) {
|
||||
const existingSystem = mode !== 'new world';
|
||||
const preserveSystem = mode === 'extension';
|
||||
it(`post-review ${mode} ${preserveSystem ? 'preserves' : 'records'} its system :: ${modelId}`,
|
||||
{ skip: !ENGINE_BIN || !hasKey(detectProvider(modelId)) }, async (t) => {
|
||||
const files = {
|
||||
'PRODUCT.md': '# Field Manual\n\n## Platform\nweb\n\nA reference guide for keyboard users.\n',
|
||||
...(existingSystem ? { 'DESIGN.md': preserveSystem ? DESIGN : '# Old Field Manual\n\nBeige cards, serif body type, orange links.\n' } : {}),
|
||||
'index.html': PAGE,
|
||||
'.impeccable/surfaces/index-html.md': mode === 'redesign'
|
||||
? BRIEF.replace('Inherit Field Manual.', 'Approved replacement: white, blue links, system-ui, single column.') : BRIEF,
|
||||
};
|
||||
const workspace = prepareWorkspace({ files });
|
||||
try {
|
||||
const reference = fs.readFileSync(path.join(workspace, '.claude/skills/impeccable/reference/new-work.md'), 'utf8');
|
||||
const result = await runTurn({
|
||||
workspace, model: getModel(modelId), maxSteps: 12, timeoutMs: 180000, contextOnlyBash: true,
|
||||
environment: 'This is a resumed post-review checkpoint. No subagent or browser tools are available. The review is closed; no further UI edits or screenshots are needed. Read/list/write tools are available.',
|
||||
priorMessages: [
|
||||
{ role: 'user', content: preserveSystem
|
||||
? 'Use /impeccable to add the specified keyboard guide page inside the established Field Manual world. Keep the existing visual system. Do not repair unrelated project drift.'
|
||||
: mode === 'redesign'
|
||||
? 'Use /impeccable to redesign the keyboard guide. I approve replacing the old beige-card/serif/orange world with the plain single-column, system-font, white-background and blue-link identity. Update the system documentation from the finished page.'
|
||||
: 'Use /impeccable to create Field Manual’s first keyboard guide page. The chosen identity is plain, single-column, system fonts, white background and blue links.' },
|
||||
{ role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'load-new-work', toolName: 'read', input: { path: '.claude/skills/impeccable/reference/new-work.md' } }] },
|
||||
{ role: 'tool', content: [{ type: 'tool-result', toolCallId: 'load-new-work', toolName: 'read', output: { type: 'text', value: reference } }] },
|
||||
{ role: 'assistant', content: `Checkpoint: context and PRODUCT.md were loaded. The user confirmed the exact page and identity. The surface brief and index.html are written. Desktop/mobile captures were validated, the detector ran once, and the shipped finish reviewer returned ship with no open findings. ${preserveSystem
|
||||
? 'DESIGN.md was loaded. No durable system changes were requested or introduced. The pre-existing missing .impeccable/design.json was reported but not repaired.'
|
||||
: mode === 'redesign'
|
||||
? 'The approved replacement world is implemented in index.html. DESIGN.md still describes the superseded identity; no design sidecar exists yet.'
|
||||
: 'This is the first completed surface of the approved new world. No DESIGN.md or design sidecar exists yet.'}` },
|
||||
],
|
||||
userPrompt: 'Continue from this checkpoint and finish the task.',
|
||||
});
|
||||
assertCompleted(result);
|
||||
t.diagnostic(`Documentation wrapper coverage gaps (diagnostic; contract and artifacts remain required): ${missingReferences(result.trace, ['degraded/documenter.md']).join(', ') || 'none'}`);
|
||||
assert.ok(fileLoaded(result.trace, 'reference/document.md'), 'must consult the documentation contract');
|
||||
for (const name of existingSystem ? ['index.html', 'DESIGN.md'] : ['index.html']) {
|
||||
assert.ok(fileLoaded(result.trace, name), `documentation must check ${name}, not merely announce a no-op`);
|
||||
}
|
||||
for (const [name, contents] of Object.entries(files)) {
|
||||
if (mode === 'redesign' && name === 'DESIGN.md') continue;
|
||||
assert.equal(fs.readFileSync(path.join(workspace, name), 'utf8'), contents, `${name} must remain unchanged`);
|
||||
}
|
||||
if (preserveSystem) {
|
||||
assertNoChangeDocumentation(result, { target: 'index.html', evidence: [/system-ui/i, /65\s*ch/i, /#0645ad/i] });
|
||||
assert.equal(fs.existsSync(path.join(workspace, '.impeccable/design.json')), false, 'must not repair pre-existing sidecar drift unasked');
|
||||
assert.deepEqual(result.trace.toolCalls.flatMap((call) => call.mutatedPaths || []), [], 'a no-change check must not mutate other project files');
|
||||
} else {
|
||||
const design = fs.readFileSync(path.join(workspace, 'DESIGN.md'), 'utf8');
|
||||
if (mode === 'redesign') assert.notEqual(design, files['DESIGN.md'], 'approved redesign must replace the old system');
|
||||
assertDocumentationArtifacts(design, fs.readFileSync(path.join(workspace, '.impeccable/design.json'), 'utf8'));
|
||||
assert.match(design, /system-ui/);
|
||||
const writes = result.trace.toolCalls.flatMap((call) => call.mutatedPaths || []);
|
||||
assert.ok(writes.includes('DESIGN.md') && writes.includes('.impeccable/design.json'), 'both documentation artifacts must be written');
|
||||
assert.deepEqual(writes.filter((file) => !['DESIGN.md', '.impeccable/design.json'].includes(file)), [], 'documentation must stay inside its write boundary');
|
||||
}
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
/**
|
||||
* Provider-backed workflow contract tests. Unlike scenarios.test.mjs, these
|
||||
* assert the attended turns and writes that make init/redesign/refinement real.
|
||||
*/
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
prepareWorkspace,
|
||||
cleanupWorkspace,
|
||||
runTurn as runHarnessTurn,
|
||||
fileLoaded,
|
||||
summarizeTrace,
|
||||
ENGINE_BIN,
|
||||
ENGINE_MISSING_MESSAGE,
|
||||
} from '../skill-behavior/harness.mjs';
|
||||
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from '../skill-behavior/providers.mjs';
|
||||
import { assertNewWorkLifecycle } from '../skill-behavior/assertions.mjs';
|
||||
import { PRODUCT_MD_SAMPLE, DESIGN_MD_SAMPLE as ORIGINAL_DESIGN, CASE_STUDY_ANSWER } from '../skill-behavior/fixtures.mjs';
|
||||
import { prepareBrowser } from './browser.mjs';
|
||||
import { assertCompleted, assertFreshCaptures, assertDocumentationArtifacts } from './assertions.mjs';
|
||||
|
||||
const DESIGN_MD_SAMPLE = ORIGINAL_DESIGN.replace(/GT Sectra \(commercial\)/g, 'Georgia (system)').replace(/JetBrains Mono/g, 'monospace').replace(/Inter/g, 'Arial');
|
||||
|
||||
async function runTurn(options) {
|
||||
// Preflight happens before the first provider call. These are text-only
|
||||
// HTML fixtures: no dependencies, font downloads, or browser discovery.
|
||||
const browser = await prepareBrowser(options.workspace);
|
||||
try {
|
||||
const result = await runHarnessTurn({
|
||||
...options, maxSteps: 50, timeoutMs: 840000,
|
||||
userPrompt: `${options.userPrompt}\nUse system fonts and no external assets for this text-only fixture. The browser_snapshot and view_image tools are ready for visual review.`,
|
||||
environment: browser.environment,
|
||||
additionalTools: (trace) => browser.tools(trace),
|
||||
});
|
||||
assertCompleted(result);
|
||||
const contextCalls = result.trace.toolCalls.filter(({ name, input }) => name === 'bash' && /impeccable\s+context\b/.test(input.command));
|
||||
assert.equal(contextCalls.length, 1, 'completed workflow must load context exactly once');
|
||||
return result;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
const LEGACY_DESIGN = `# Design
|
||||
|
||||
## Identity
|
||||
BORING_BEIGE_CARDS. Quiet beige panels, timid scale, rounded cards everywhere.
|
||||
|
||||
## Color
|
||||
Warm gray background with a muted tan accent.
|
||||
`;
|
||||
|
||||
const EXISTING_PAGE = `<!doctype html>
|
||||
<html><head><style>
|
||||
:root { --legacy-beige: #e8e1d5; --legacy-tan: #a78969; }
|
||||
body { background: var(--legacy-beige); color: #3c3833; font-family: Arial, sans-serif; }
|
||||
.card { border: 1px solid #cfc5b6; border-radius: 18px; padding: 24px; }
|
||||
</style></head><body>
|
||||
<header data-untouched="header"><a href="/">Harbor Desk</a></header>
|
||||
<main><section id="case-study" class="card"><h1>Harbor Desk</h1><p>Challenge. Approach. Outcome.</p><p>Image placeholder</p></section></main>
|
||||
<footer data-untouched="footer">Operational since 1987</footer>
|
||||
</body></html>`;
|
||||
|
||||
// Deliberately broken enough that any honest critique lists three or more
|
||||
// Priority Issues, so the run cannot reach the "fewer than 3" skip branch by
|
||||
// merit. Low contrast, an icon-tile stack, a kicker over the heading, dead
|
||||
// hierarchy, and a placeholder CTA.
|
||||
const FLAWED_PAGE = `<!doctype html>
|
||||
<html><head><style>
|
||||
body { background:#f4f4f5; color:#b9b9c0; font-family: Arial, sans-serif; font-size:15px; }
|
||||
h1, h2, h3, p { font-size:15px; font-weight:400; margin:8px 0; }
|
||||
.tile { width:48px; height:48px; background:#e6e6ea; border-radius:12px; }
|
||||
.card { border:1px solid #e6e6ea; border-radius:12px; padding:16px; }
|
||||
</style></head><body>
|
||||
<main>
|
||||
<p class="kicker">INTRODUCING</p>
|
||||
<h1>Harbor Desk</h1>
|
||||
<p>A platform that helps teams do more of what matters, faster.</p>
|
||||
<section class="card"><div class="tile"></div><h3>Lightning Fast</h3><p>Blazing performance.</p></section>
|
||||
<section class="card"><div class="tile"></div><h3>Rock Solid</h3><p>Enterprise grade.</p></section>
|
||||
<section class="card"><div class="tile"></div><h3>Fully Secure</h3><p>Bank level security.</p></section>
|
||||
<button style="background:#e6e6ea;color:#c9c9d0;border:none;padding:8px 12px">Learn More</button>
|
||||
</main>
|
||||
</body></html>`;
|
||||
|
||||
/**
|
||||
* Flatten assistant output into ordered parts.
|
||||
*
|
||||
* `generateText` only returns `text` for the FINAL step, which is empty when a
|
||||
* turn ends on a tool call. Reading the report out of that field silently tests
|
||||
* nothing. Walking responseMessages instead preserves emission order, which is
|
||||
* the point: critique's invariant is that report prose precedes the question
|
||||
* inside the message, since prose after a structured question is withheld until
|
||||
* the user answers.
|
||||
*/
|
||||
function assistantParts(responseMessages) {
|
||||
const parts = [];
|
||||
for (const message of responseMessages) {
|
||||
if (message.role !== 'assistant') continue;
|
||||
const content = message.content;
|
||||
if (typeof content === 'string') {
|
||||
parts.push({ kind: 'text', value: content });
|
||||
continue;
|
||||
}
|
||||
for (const part of content ?? []) {
|
||||
if (part.type === 'text') parts.push({ kind: 'text', value: part.text ?? '' });
|
||||
else if (part.type === 'tool-call') parts.push({ kind: 'tool', value: part.toolName ?? '' });
|
||||
}
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
|
||||
function firstCall(trace, predicate) {
|
||||
return trace.toolCalls.findIndex(predicate);
|
||||
}
|
||||
|
||||
function firstMutation(trace, pattern) {
|
||||
return firstCall(trace, ({ mutatedPaths = [] }) => mutatedPaths.some((file) => pattern.test(file)));
|
||||
}
|
||||
|
||||
function workflowTraceMessage(trace) {
|
||||
return JSON.stringify(summarizeTrace(trace), null, 2);
|
||||
}
|
||||
|
||||
// Full builds are separately opt-in and default to one provider. The existing
|
||||
// model selection variable can explicitly request a cross-provider sweep.
|
||||
for (const modelId of process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS ? resolveModelList() : ['claude-sonnet-5']) {
|
||||
const provider = detectProvider(modelId);
|
||||
const keyPresent = hasKey(provider);
|
||||
|
||||
describe(`skill workflow contract :: ${modelId}`, () => {
|
||||
if (!keyPresent) {
|
||||
it(`skipped — ${PROVIDERS[provider].envKey} is unset`, { skip: true }, () => {});
|
||||
return;
|
||||
}
|
||||
if (!ENGINE_BIN) {
|
||||
it(`skipped — ${ENGINE_MISSING_MESSAGE}`, { skip: true }, () => {});
|
||||
return;
|
||||
}
|
||||
const model = getModel(modelId);
|
||||
|
||||
it('fresh init asks and writes PRODUCT without inventing a visual system', async () => {
|
||||
const workspace = prepareWorkspace({ files: {} });
|
||||
try {
|
||||
const { trace } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable init for a harbor operations product, then finish setup.',
|
||||
});
|
||||
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
||||
const productWrite = firstMutation(trace, /(^|\/)PRODUCT\.md$/i);
|
||||
assert.ok(fileLoaded(trace, 'init.md'), `init.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(question >= 0, `structured user was never asked.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(productWrite > question, `PRODUCT.md must follow a user answer.\n${workflowTraceMessage(trace)}`);
|
||||
const product = fs.readFileSync(path.join(workspace, 'PRODUCT.md'), 'utf8');
|
||||
assert.doesNotMatch(product, /^## Register\s*$/im);
|
||||
assert.match(product, /ferry|dispatch|harbor/i, 'PRODUCT.md should incorporate the simulated user context');
|
||||
assert.equal(fs.existsSync(path.join(workspace, 'DESIGN.md')), false, 'init must not create DESIGN.md');
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
|
||||
it('an initialized natural build request asks for the task concept before implementation', async () => {
|
||||
const workspace = prepareWorkspace({
|
||||
files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE, 'DESIGN.md': DESIGN_MD_SAMPLE },
|
||||
});
|
||||
try {
|
||||
const { trace } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable create a concise evidence-led case-study page. Leave it at index.html.',
|
||||
simulatedUser: { answer: () => CASE_STUDY_ANSWER },
|
||||
});
|
||||
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
||||
assert.ok(fileLoaded(trace, 'new-work.md'), `new-work.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(question >= 0, `task concept was never put to the user.\n${workflowTraceMessage(trace)}`);
|
||||
assertNewWorkLifecycle(trace, { target: 'index.html' });
|
||||
assertFreshCaptures(trace, workspace, 'index.html');
|
||||
assert.ok(fileLoaded(trace, 'finish-reviewer.md'), 'new-work must run the shipped finish review');
|
||||
assert.ok(fileLoaded(trace, 'documenter.md'), 'new-work must run the shipped documentation pass');
|
||||
assert.equal(fs.existsSync(path.join(workspace, 'index.html')), true, 'new-work must still produce the requested artifact');
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
|
||||
it('redesign approves and records the direction before code, then documents the built world', async () => {
|
||||
const workspace = prepareWorkspace({
|
||||
files: {
|
||||
'PRODUCT.md': PRODUCT_MD_SAMPLE,
|
||||
'DESIGN.md': LEGACY_DESIGN,
|
||||
'current.html': EXISTING_PAGE,
|
||||
},
|
||||
});
|
||||
try {
|
||||
const { trace } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable redesign current.html for this product. Leave the result at current.html.',
|
||||
});
|
||||
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
||||
assert.ok(fileLoaded(trace, 'new-work.md'), `redesign did not route through new-work.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(question >= 0, `replacement world was not put to the user.\n${workflowTraceMessage(trace)}`);
|
||||
assertNewWorkLifecycle(trace, { target: 'current.html', redesign: true });
|
||||
assertFreshCaptures(trace, workspace, 'current.html');
|
||||
assert.ok(fileLoaded(trace, 'finish-reviewer.md'), 'redesign must run the shipped finish review');
|
||||
assert.ok(fileLoaded(trace, 'documenter.md'), 'redesign must run the shipped documentation pass');
|
||||
const design = fs.readFileSync(path.join(workspace, 'DESIGN.md'), 'utf8');
|
||||
assert.notEqual(design.trim(), LEGACY_DESIGN.trim(), 'redesign preserved the old visual world verbatim');
|
||||
assertDocumentationArtifacts(design, fs.readFileSync(path.join(workspace, '.impeccable/design.json'), 'utf8'));
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
|
||||
it('bolder refinement preserves the world and everything outside scope', async () => {
|
||||
const workspace = prepareWorkspace({
|
||||
files: {
|
||||
'PRODUCT.md': PRODUCT_MD_SAMPLE,
|
||||
'DESIGN.md': DESIGN_MD_SAMPLE,
|
||||
'current.html': EXISTING_PAGE,
|
||||
},
|
||||
});
|
||||
try {
|
||||
const { trace } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable bolder current.html, only the #case-study section. Keep everything else untouched.',
|
||||
});
|
||||
const productWrite = firstMutation(trace, /(^|\/)PRODUCT\.md$/i);
|
||||
const designWrite = firstMutation(trace, /(^|\/)DESIGN\.md$/i);
|
||||
const implementation = firstMutation(trace, /(^|\/)current\.html$/i);
|
||||
assert.ok(fileLoaded(trace, 'bolder.md'), `bolder.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
||||
assert.equal(productWrite, -1, `refinement rewrote PRODUCT.md.\n${workflowTraceMessage(trace)}`);
|
||||
assert.equal(designWrite, -1, `refinement rewrote DESIGN.md.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(implementation >= 0, `refinement did not write current.html.\n${workflowTraceMessage(trace)}`);
|
||||
assertFreshCaptures(trace, workspace, 'current.html');
|
||||
const artifact = fs.readFileSync(path.join(workspace, 'current.html'), 'utf8');
|
||||
assert.match(artifact, /data-untouched="header"/);
|
||||
assert.match(artifact, /data-untouched="footer"/);
|
||||
assert.match(artifact, /id="case-study"/);
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
|
||||
// Regression guard for the failure mode that shipped in PR #576: the report
|
||||
// landed and the run then stopped, asking nothing and printing no skip
|
||||
// line. The close is the deliverable's other half, so a critique that ends
|
||||
// on the report is incomplete. Asserted on the trace rather than on prose
|
||||
// because the model's own account of why it skipped is not evidence.
|
||||
it('critique closes with the question or an explicit skip line', async () => {
|
||||
const workspace = prepareWorkspace({
|
||||
files: {
|
||||
'PRODUCT.md': PRODUCT_MD_SAMPLE,
|
||||
'DESIGN.md': DESIGN_MD_SAMPLE,
|
||||
'current.html': FLAWED_PAGE,
|
||||
},
|
||||
});
|
||||
try {
|
||||
const { trace, responseMessages } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable critique current.html',
|
||||
});
|
||||
assert.ok(fileLoaded(trace, 'critique.md'), `critique.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
||||
assertFreshCaptures(trace, workspace, 'current.html');
|
||||
|
||||
const parts = assistantParts(responseMessages);
|
||||
const allText = parts.filter((p) => p.kind === 'text').map((p) => p.value).join('\n');
|
||||
const reportPattern = /priority issue|heuristic|design health/i;
|
||||
assert.match(allText, reportPattern, `no report reached the user.\n${workflowTraceMessage(trace)}`);
|
||||
|
||||
const askIndex = parts.findIndex((p) => p.kind === 'tool' && p.value === 'ask_user_question');
|
||||
const skipped = /Questions skipped:/i.test(allText);
|
||||
assert.ok(
|
||||
askIndex >= 0 || skipped,
|
||||
`critique ended without the questions and without a "Questions skipped: <reason>" line.\n` +
|
||||
`This is the PR #576 regression: the report is not the finish, the close is.\n${workflowTraceMessage(trace)}`,
|
||||
);
|
||||
|
||||
// The ordering invariant. Only meaningful when a question was actually
|
||||
// asked; a skip-line close has nothing to order against.
|
||||
if (askIndex >= 0) {
|
||||
const reportIndex = parts.findIndex((p) => p.kind === 'text' && reportPattern.test(p.value));
|
||||
assert.ok(
|
||||
reportIndex >= 0 && reportIndex < askIndex,
|
||||
`the question was emitted before the report text, so the report stays hidden until the user answers.\n` +
|
||||
`${workflowTraceMessage(trace)}`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
// Include local styles/scripts/assets too: an unchanged HTML file is not
|
||||
// evidence of a current capture when an external stylesheet changed.
|
||||
export function sourceHash(root) {
|
||||
const hash = crypto.createHash('sha256');
|
||||
function visit(directory, prefix = '') {
|
||||
for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
if (entry.name.startsWith('.') || entry.name === 'node_modules') continue;
|
||||
const relative = `${prefix}${entry.name}`;
|
||||
const file = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) visit(file, `${relative}/`);
|
||||
else if (entry.isFile() && /\.(html?|css|m?js|svg|png|jpe?g|webp|woff2?)$/i.test(entry.name)) {
|
||||
hash.update(relative).update('\0').update(fs.readFileSync(file)).update('\0');
|
||||
}
|
||||
}
|
||||
}
|
||||
visit(root);
|
||||
return hash.digest('hex');
|
||||
}
|
||||
Reference in New Issue
Block a user