mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-12 06:06:37 +03:00
Refresh the Impeccable product experience
Rework the landing page proof, steering demo, feature grid, slop catalog, detector coverage, theming, Live workflow, and responsive behavior.\n\nAI-assisted implementation by OpenAI Codex.
This commit is contained in:
@@ -40,18 +40,18 @@ The trace is the source of truth, not the model's free-form reply.
|
||||
|
||||
| # | Setup | Assertion |
|
||||
|---|---|---|
|
||||
| 1 | empty workspace | runs `context.mjs`; loads `reference/init.md` when it treats the run as attended or `reference/new-work.md` when it recognizes the one-shot exception; resolves that gate before implementation |
|
||||
| 2 | PRODUCT.md only | runs `context.mjs` 1-3 times; loads `reference/new-work.md` because no committed design system exists |
|
||||
| 3 | PRODUCT.md + DESIGN.md | runs `context.mjs` 1-3 times; receives or explores the committed design system |
|
||||
| 1 | empty workspace | runs `context.mjs`; loads `reference/init.md` before implementation; automation is not an init bypass |
|
||||
| 2 | PRODUCT.md only | runs `context.mjs` 1-3 times; loads `reference/init.md` to establish DESIGN.md with the user before surface work |
|
||||
| 3 | PRODUCT.md + DESIGN.md | runs `context.mjs` 1-3 times; receives the committed design system and loads `reference/new-work.md` for the task-scoped concept |
|
||||
| 4 | PRODUCT.md + DESIGN.md, context already loaded in turn 1 | turn 2 does **not** re-run `context.mjs` |
|
||||
| 5 | PRODUCT.md without the legacy `## Register` field | runs `context.mjs`; greenfield craft still loads `reference/new-work.md` |
|
||||
| 5 | PRODUCT.md without the legacy `## Register` field and no DESIGN.md | runs `context.mjs`; greenfield craft still loads `reference/init.md` to establish the missing world |
|
||||
| 6 | PRODUCT.md + DESIGN.md + a minimal `index.html`; prompt is `/impeccable polish` | loads `reference/polish.md` |
|
||||
| 7 | same fixture; prompt is `/impeccable audit` | loads `reference/audit.md` |
|
||||
| 8 | PRODUCT.md + DESIGN.md + a SvelteKit scaffold (`src/app.css`, components, `+page.svelte`); prompt is `/impeccable polish src/routes/+page.svelte` | reads at least one project code file (CSS / component / page) — not just the skill's reference files |
|
||||
| 9 | PRODUCT.md + `index.html` + a seeded update cache with a newer version (`skillVersion` copy-mode so `context.mjs` has a `SKILL.md` to version-check against); prompt is `/impeccable polish index.html` | `context.mjs` runs and its output carries the `UPDATE_AVAILABLE` directive (proven via captured bash output); the agent does **not** auto-run `npx impeccable update` (it must ask first) |
|
||||
| 10 | no PRODUCT.md + a minimal `index.html`; prompt is `/impeccable polish index.html` | runs `context.mjs`, loads `reference/polish.md`, and does **not** divert into `reference/init.md` |
|
||||
| 11 | empty workspace; prompt is `/impeccable shape ...` | runs `context.mjs`; resolves `reference/init.md` (attended) or `reference/new-work.md` (unattended) before implementation |
|
||||
| 12 | empty workspace; prompt is natural-language build intent with no command word | runs `context.mjs`; resolves `reference/init.md` (attended) or `reference/new-work.md` (unattended) before implementation |
|
||||
| 11 | empty workspace; prompt is `/impeccable shape ...` | runs `context.mjs`; resolves `reference/init.md` before planning the surface |
|
||||
| 12 | empty workspace; prompt is natural-language build intent with no command word | runs `context.mjs`; resolves `reference/init.md` before implementation |
|
||||
| 13 | empty workspace; prompt is `/impeccable teach` | runs `context.mjs` and diverts into `reference/init.md` because `teach` aliases `init` |
|
||||
| 14 | PRODUCT.md with `## Register: product` + `## Platform: ios` (native iOS app); prompt is `/impeccable craft a tide detail screen` | `context.mjs` runs and emits a NEXT STEP pointing at `reference/ios.md` (proven via captured bash output); agent loads `reference/ios.md` (Setup step 5, native conventions on top of the register reference) |
|
||||
| 15 | same iOS fixture; prompt is `/impeccable audit` | agent loads `reference/audit.native.md` (the Commands-table native variant, routed instead of `audit.md`) |
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
*/
|
||||
export const PRODUCT_MD_SAMPLE = `# Acme Notes
|
||||
|
||||
## Register
|
||||
brand
|
||||
## Platform
|
||||
web
|
||||
|
||||
## Product Purpose
|
||||
Acme Notes is a marketing-driven landing page for a research-grade note-taking
|
||||
@@ -21,7 +21,23 @@ Working researchers (PhD students, postdocs, principal investigators) who
|
||||
already maintain disciplined note-taking systems and are choosing between
|
||||
ours and rolling their own in a Zettelkasten plugin.
|
||||
|
||||
## Brand
|
||||
## Positioning
|
||||
The research notebook that preserves a scientist's chain of thought instead
|
||||
of flattening it into generic documents and folders.
|
||||
|
||||
## Audience World
|
||||
Lab notebooks, margin annotations, citation trails, preprint PDFs, index cards,
|
||||
and the quiet ritual of reconstructing why a conclusion was reached months ago.
|
||||
|
||||
## Cultural Context
|
||||
Research monographs and working laboratory archives: precise, annotated,
|
||||
accumulative, and visibly handled rather than pristine lifestyle publishing.
|
||||
|
||||
## Pinned Direction
|
||||
Type-led and evidence-first. Never lead with product screenshots or generic
|
||||
startup chrome.
|
||||
|
||||
## Brand Personality
|
||||
Editorial, considered, technical. The product is for people who quote
|
||||
Knuth. The voice is closer to a long-read magazine than to a startup
|
||||
landing page.
|
||||
@@ -31,20 +47,25 @@ landing page.
|
||||
- Obsidian (too community-cottagecore)
|
||||
- Any SaaS landing page with a hero-metric grid
|
||||
|
||||
## Strategic Principles
|
||||
## Design Principles
|
||||
- Type does most of the work. The hero is words, not chrome.
|
||||
- One named accent color, used sparingly.
|
||||
- Never lead with screenshots. Lead with the idea.
|
||||
|
||||
## Accessibility & Inclusion
|
||||
WCAG AA, fully keyboard accessible, readable at 200% zoom, and calm under
|
||||
reduced motion.
|
||||
`;
|
||||
|
||||
/**
|
||||
* Same project shape as PRODUCT_MD_SAMPLE but with no `## Register` field.
|
||||
* Exercises the cascade fallback (task cue then surface in focus) in
|
||||
* scenarios where context.mjs cannot detect the register and the agent
|
||||
* must follow the SKILL.md priority list to pick brand.md.
|
||||
* Legacy product context with the modern strategic fields but no visual
|
||||
* world. Exercises init completion without a deprecated brand/product field.
|
||||
*/
|
||||
export const PRODUCT_MD_SAMPLE_NO_REGISTER = `# Acme Notes
|
||||
|
||||
## Platform
|
||||
web
|
||||
|
||||
## Product Purpose
|
||||
Acme Notes is a marketing-driven landing page for a research-grade note-taking
|
||||
tool aimed at independent scientists and graduate students. The site needs to
|
||||
@@ -56,7 +77,21 @@ Working researchers (PhD students, postdocs, principal investigators) who
|
||||
already maintain disciplined note-taking systems and are choosing between
|
||||
ours and rolling their own in a Zettelkasten plugin.
|
||||
|
||||
## Brand
|
||||
## Positioning
|
||||
The research notebook that preserves a scientist's chain of thought instead
|
||||
of flattening it into generic documents and folders.
|
||||
|
||||
## Audience World
|
||||
Lab notebooks, margin annotations, citation trails, preprint PDFs, index cards,
|
||||
and the ritual of reconstructing a conclusion months later.
|
||||
|
||||
## Cultural Context
|
||||
Research monographs and working laboratory archives.
|
||||
|
||||
## Pinned Direction
|
||||
Type-led and evidence-first; no startup chrome.
|
||||
|
||||
## Brand Personality
|
||||
Editorial, considered, technical. The product is for people who quote
|
||||
Knuth. The voice is closer to a long-read magazine than to a startup
|
||||
landing page.
|
||||
@@ -66,23 +101,22 @@ landing page.
|
||||
- Obsidian (too community-cottagecore)
|
||||
- Any SaaS landing page with a hero-metric grid
|
||||
|
||||
## Strategic Principles
|
||||
## Design Principles
|
||||
- Type does most of the work. The hero is words, not chrome.
|
||||
- One named accent color, used sparingly.
|
||||
- Never lead with screenshots. Lead with the idea.
|
||||
|
||||
## Accessibility & Inclusion
|
||||
WCAG AA, keyboard access, 200% zoom, and reduced motion support.
|
||||
`;
|
||||
|
||||
/**
|
||||
* Native iOS app fixture: product register, `## Platform` set to `ios`.
|
||||
* Exercises Setup step 5 — when context.mjs reports the platform is native,
|
||||
* the agent must also load `reference/ios.md` (Apple HIG) on top of the
|
||||
* register reference. Product register because this is app UI, not marketing.
|
||||
* Native iOS app fixture with `## Platform` set to `ios`. Exercises Setup
|
||||
* step 5 — the agent must also load `reference/ios.md` (Apple HIG) on top of
|
||||
* the task-scoped visitor-mode guidance.
|
||||
*/
|
||||
export const PRODUCT_MD_SAMPLE_IOS = `# Tideline
|
||||
|
||||
## Register
|
||||
product
|
||||
|
||||
## Platform
|
||||
ios
|
||||
|
||||
@@ -97,7 +131,18 @@ Saltwater anglers checking conditions dockside on an iPhone, often one-handed
|
||||
in bright sun and sometimes offline. They live in Apple Weather, Notes, and
|
||||
Maps and expect the same gestures and controls here.
|
||||
|
||||
## Brand
|
||||
## Positioning
|
||||
The fastest trustworthy read on whether the next coastal window is worth the trip.
|
||||
|
||||
## Audience World
|
||||
Tide tables, chartplotters, dock logs, weather radar, tackle trays, wet gloves,
|
||||
and the repeated glance from water to phone in hard daylight.
|
||||
|
||||
## Pinned Direction
|
||||
Native iOS controls and navigation are non-negotiable; the logbook may carry
|
||||
the product's distinctive character.
|
||||
|
||||
## Brand Personality
|
||||
Calm, legible, marine. Identity shows through color, type accent, and the
|
||||
logbook's character — never by reinventing the navigation bar or the back
|
||||
gesture.
|
||||
@@ -107,10 +152,14 @@ gesture.
|
||||
- Custom toggles and bespoke tab bars that fight the platform
|
||||
- Cluttered, metric-theater home screens
|
||||
|
||||
## Strategic Principles
|
||||
## Design Principles
|
||||
- Platform conformance is the structural bar; brand lives in the expressive layer.
|
||||
- Standard navigation, SF Symbols, Dynamic Type, Dark Mode first-class.
|
||||
- One accent tint drives interactive elements.
|
||||
|
||||
## Accessibility & Inclusion
|
||||
Dynamic Type, VoiceOver, reduced motion, high contrast in direct sun, and
|
||||
targets usable one-handed with wet hands.
|
||||
`;
|
||||
|
||||
/**
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
* 4. Inlines SKILL.md as the system prompt (placeholders stripped to
|
||||
* neutral values so the same body works for all providers).
|
||||
* 5. Runs Vercel AI SDK generateText with workspace-scoped tools
|
||||
* (bash, read, write, list).
|
||||
* (bash, read, write, list, ask_user_question).
|
||||
* 6. Captures every tool call and returns a trace + the raw response
|
||||
* messages (so multi-turn scenarios can append to them).
|
||||
*
|
||||
@@ -49,7 +49,7 @@ function loadSkillBody() {
|
||||
md = md
|
||||
.replaceAll('{{model}}', 'the assistant')
|
||||
.replaceAll('{{command_prefix}}', '/')
|
||||
.replaceAll('{{ask_instruction}}', 'Ask the user')
|
||||
.replaceAll('{{ask_instruction}}', 'Use the ask_user_question tool.')
|
||||
.replaceAll('{{config_file}}', 'AGENTS.md')
|
||||
.replaceAll('{{scripts_path}}', '.claude/skills/impeccable/scripts')
|
||||
.replaceAll('{{command_hint}}', 'command');
|
||||
@@ -159,7 +159,26 @@ function execBash(workspace, command, timeoutMs = 20_000, extraEnv = {}) {
|
||||
* Build the workspace-scoped tool set + the trace it writes into.
|
||||
* Returns `{ tools, trace }`. The trace mutates in place as the agent runs.
|
||||
*/
|
||||
export function makeTools(workspace, extraEnv = {}) {
|
||||
function defaultSimulatedAnswer(question) {
|
||||
const text = String(question?.question ?? '').toLowerCase();
|
||||
const options = Array.isArray(question?.options) ? question.options : [];
|
||||
const firstOption = options.find((option) => typeof option?.label === 'string')?.label;
|
||||
|
||||
// Option labels are model-authored and therefore the most faithful answer
|
||||
// when the agent is asking the user to choose a proposed world or concept.
|
||||
if (firstOption) return firstOption;
|
||||
if (/platform|web|ios|android|adaptive/.test(text)) return 'Web.';
|
||||
if (/who|audience|user|people/.test(text)) return 'Night-shift ferry dispatchers working from noisy control rooms.';
|
||||
if (/purpose|job|problem|outcome|success/.test(text)) return 'Help dispatchers resolve berth conflicts before they delay the overnight crossing.';
|
||||
if (/position|different|claim|only/.test(text)) return 'It turns fragmented radio calls into one trustworthy handoff record.';
|
||||
if (/world|tool|place|object|ritual|context/.test(text)) return 'Harbor logs, tide tables, grease-pencil berth boards, radio call signs, and sodium-lit terminals.';
|
||||
if (/direction|feel|personality|reference|look/.test(text)) return 'Decisive, maritime, and operational; avoid generic SaaS dashboards and nautical decoration.';
|
||||
if (/accessib|motion|contrast/.test(text)) return 'WCAG AA, keyboard access, reduced motion, and high contrast for dim control rooms.';
|
||||
if (/scope|fidelity|breadth|interactiv|polish/.test(text)) return 'One production-ready responsive surface with working interactions.';
|
||||
return 'Use the brief, preserve real operational content, and make the primary decision obvious.';
|
||||
}
|
||||
|
||||
export function makeTools(workspace, extraEnv = {}, simulatedUser = {}) {
|
||||
const trace = {
|
||||
toolCalls: [],
|
||||
bashCommands: [],
|
||||
@@ -167,6 +186,8 @@ export function makeTools(workspace, extraEnv = {}) {
|
||||
readPaths: [],
|
||||
writePaths: [],
|
||||
listPaths: [],
|
||||
questionCalls: [],
|
||||
questionAnswers: [],
|
||||
};
|
||||
function record(name, input) {
|
||||
trace.toolCalls.push({ name, input });
|
||||
@@ -174,6 +195,7 @@ export function makeTools(workspace, extraEnv = {}) {
|
||||
if (name === 'read' && typeof input?.path === 'string') trace.readPaths.push(input.path);
|
||||
if (name === 'write' && typeof input?.path === 'string') trace.writePaths.push(input.path);
|
||||
if (name === 'list' && typeof input?.path === 'string') trace.listPaths.push(input.path);
|
||||
if (name === 'ask_user_question') trace.questionCalls.push(input);
|
||||
}
|
||||
const tools = {
|
||||
bash: tool({
|
||||
@@ -241,6 +263,34 @@ export function makeTools(workspace, extraEnv = {}) {
|
||||
return entries.length ? entries.join('\n') : '(empty)';
|
||||
},
|
||||
}),
|
||||
ask_user_question: tool({
|
||||
description:
|
||||
'Ask the user 1-4 structured questions and wait for answers. Use this for required Impeccable init, visual-world selection, and task-concept checkpoints instead of asking in prose.',
|
||||
inputSchema: z.object({
|
||||
questions: z.array(z.object({
|
||||
header: z.string().optional(),
|
||||
question: z.string(),
|
||||
options: z.array(z.object({
|
||||
label: z.string(),
|
||||
description: z.string().optional(),
|
||||
})).optional(),
|
||||
multiSelect: z.boolean().optional(),
|
||||
})).min(1).max(4),
|
||||
}),
|
||||
execute: async ({ questions }) => {
|
||||
record('ask_user_question', { questions });
|
||||
const answers = {};
|
||||
for (let index = 0; index < questions.length; index++) {
|
||||
const question = questions[index];
|
||||
const custom = typeof simulatedUser.answer === 'function'
|
||||
? await simulatedUser.answer(question, index, { workspace, trace })
|
||||
: undefined;
|
||||
answers[question.question] = custom ?? defaultSimulatedAnswer(question);
|
||||
}
|
||||
trace.questionAnswers.push(answers);
|
||||
return JSON.stringify({ answers });
|
||||
},
|
||||
}),
|
||||
};
|
||||
return { tools, trace };
|
||||
}
|
||||
@@ -251,8 +301,8 @@ export function makeTools(workspace, extraEnv = {}) {
|
||||
* `priorMessages` lets multi-turn scenarios chain context from a previous
|
||||
* call (append the SDK's response messages between turns).
|
||||
*/
|
||||
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8, env = {} }) {
|
||||
const { tools, trace } = makeTools(workspace, env);
|
||||
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8, env = {}, simulatedUser = {} }) {
|
||||
const { tools, trace } = makeTools(workspace, env, simulatedUser);
|
||||
const messages = [
|
||||
...priorMessages,
|
||||
{ role: 'user', content: userPrompt },
|
||||
@@ -306,5 +356,7 @@ export function summarizeTrace(trace) {
|
||||
bashCommands: trace.bashCommands,
|
||||
readPaths: trace.readPaths,
|
||||
writePaths: trace.writePaths,
|
||||
questionCalls: trace.questionCalls,
|
||||
questionAnswers: trace.questionAnswers,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ const SHAPE_PROMPT = '/impeccable shape a landing page for the project in this w
|
||||
const NATURAL_BUILD_PROMPT = 'Build a landing page for the project in this workspace.';
|
||||
const TEACH_PROMPT = '/impeccable teach';
|
||||
const PRIMER_PROMPT =
|
||||
'Take a quick look at the project. What register is this? Run the impeccable context loader once if you need to.';
|
||||
'Take a quick look at the project. What context should guide later design work? Run the impeccable context loader once if you need to.';
|
||||
|
||||
const VERBOSE = process.env.IMPECCABLE_SKILL_BEHAVIOR_VERBOSE === '1';
|
||||
|
||||
@@ -109,19 +109,14 @@ for (const modelId of resolveModelList()) {
|
||||
`expected agent to run context.mjs at least once; got ${loadCalls.length}.\n` +
|
||||
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
|
||||
);
|
||||
const resolvedBuildGate =
|
||||
fileLoaded(trace, 'init.md') || fileLoaded(trace, 'new-work.md');
|
||||
assert.ok(
|
||||
resolvedBuildGate,
|
||||
`craft should load init.md for an attended run or new-work.md when it treats the harness as unattended.\n` +
|
||||
fileLoaded(trace, 'init.md'),
|
||||
`craft should load init.md when no product or visual world exists; an automated harness is not a bypass.\n` +
|
||||
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
|
||||
);
|
||||
const gatePrecededImplementation =
|
||||
loadedBeforeImplementationWrite(trace, 'init.md') ||
|
||||
loadedBeforeImplementationWrite(trace, 'new-work.md');
|
||||
assert.ok(
|
||||
gatePrecededImplementation,
|
||||
`agent should resolve the init/new-work gate before writing implementation files.\n` +
|
||||
loadedBeforeImplementationWrite(trace, 'init.md'),
|
||||
`agent should resolve init before writing implementation files.\n` +
|
||||
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
|
||||
);
|
||||
} finally {
|
||||
@@ -148,8 +143,8 @@ for (const modelId of resolveModelList()) {
|
||||
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
|
||||
);
|
||||
assert.ok(
|
||||
fileLoaded(trace, 'new-work.md'),
|
||||
`greenfield craft should load new-work.md when PRODUCT.md exists without a committed design system.\n` +
|
||||
fileLoaded(trace, 'init.md'),
|
||||
`greenfield craft should load init.md Step 5 when PRODUCT.md exists without a committed design system.\n` +
|
||||
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
|
||||
);
|
||||
} finally {
|
||||
@@ -175,6 +170,11 @@ for (const modelId of resolveModelList()) {
|
||||
`expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` +
|
||||
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
|
||||
);
|
||||
assert.ok(
|
||||
fileLoaded(trace, 'new-work.md'),
|
||||
`craft inside a committed PRODUCT.md + DESIGN.md world should load new-work.md for the task-specific concept.\n` +
|
||||
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
|
||||
);
|
||||
// The skill tells the agent to also familiarize with the existing
|
||||
// design system. DESIGN.md is bundled in context.mjs output, but
|
||||
// exploring CSS / tokens / theme files or a directory listing
|
||||
@@ -236,7 +236,7 @@ for (const modelId of resolveModelList()) {
|
||||
}
|
||||
});
|
||||
|
||||
it('scenario 5: PRODUCT.md without legacy register metadata still follows new-work', async () => {
|
||||
it('scenario 5: legacy PRODUCT.md still completes init when DESIGN.md is missing', async () => {
|
||||
const workspace = prepareWorkspace({
|
||||
files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_NO_REGISTER },
|
||||
});
|
||||
@@ -247,7 +247,7 @@ for (const modelId of resolveModelList()) {
|
||||
userPrompt: CRAFT_PROMPT,
|
||||
maxSteps: setupMaxSteps,
|
||||
});
|
||||
logTrace('S5', 'no-register-field', modelId, trace, { textSample: text.slice(0, 400) });
|
||||
logTrace('S5', 'legacy-product', modelId, trace, { textSample: text.slice(0, 400) });
|
||||
const loadCalls = bashCommandsMatching(trace, 'context.mjs');
|
||||
assert.ok(
|
||||
loadCalls.length >= 1,
|
||||
@@ -255,8 +255,8 @@ for (const modelId of resolveModelList()) {
|
||||
`bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
|
||||
);
|
||||
assert.ok(
|
||||
fileLoaded(trace, 'new-work.md'),
|
||||
`greenfield craft should load new-work.md regardless of legacy register metadata.\n` +
|
||||
fileLoaded(trace, 'init.md'),
|
||||
`greenfield craft should load init.md for legacy product context when DESIGN.md is missing.\n` +
|
||||
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
|
||||
);
|
||||
} finally {
|
||||
@@ -462,12 +462,9 @@ for (const modelId of resolveModelList()) {
|
||||
`expected agent to run context.mjs at least once.\n` +
|
||||
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
|
||||
);
|
||||
const gatePrecededImplementation =
|
||||
loadedBeforeImplementationWrite(trace, 'init.md') ||
|
||||
loadedBeforeImplementationWrite(trace, 'new-work.md');
|
||||
assert.ok(
|
||||
gatePrecededImplementation,
|
||||
`shape should resolve init.md (attended) or new-work.md (unattended) before implementation.\n` +
|
||||
loadedBeforeImplementationWrite(trace, 'init.md'),
|
||||
`shape should resolve init.md before implementation when no world exists.\n` +
|
||||
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
|
||||
);
|
||||
} finally {
|
||||
@@ -490,12 +487,9 @@ for (const modelId of resolveModelList()) {
|
||||
`expected agent to run context.mjs at least once.\n` +
|
||||
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
|
||||
);
|
||||
const gatePrecededImplementation =
|
||||
loadedBeforeImplementationWrite(trace, 'init.md') ||
|
||||
loadedBeforeImplementationWrite(trace, 'new-work.md');
|
||||
assert.ok(
|
||||
gatePrecededImplementation,
|
||||
`build intent should resolve init.md (attended) or new-work.md (unattended) before implementation.\n` +
|
||||
loadedBeforeImplementationWrite(trace, 'init.md'),
|
||||
`build intent should resolve init.md before implementation when no world exists.\n` +
|
||||
`Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
|
||||
);
|
||||
} finally {
|
||||
@@ -533,10 +527,10 @@ for (const modelId of resolveModelList()) {
|
||||
}
|
||||
});
|
||||
|
||||
it('scenario 14: native iOS project (agent loads ios.md on top of register)', async () => {
|
||||
it('scenario 14: native iOS project (agent loads ios.md)', async () => {
|
||||
// PRODUCT.md sets `## Platform` to `ios`. context.mjs emits a NEXT STEP
|
||||
// directive to read reference/ios.md for native conventions. Setup step 5
|
||||
// requires it on top of the register reference. The detector / live mode
|
||||
// requires it on top of the visitor-mode guidance. The detector / live mode
|
||||
// are web-only, so the only platform-specific obligation is loading the
|
||||
// native reference — that's what this asserts.
|
||||
const workspace = prepareWorkspace({
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
/**
|
||||
* Provider-backed workflow contract tests. Unlike scenarios.test.mjs, these
|
||||
* assert the attended turns and writes that make init/redesign/refinement real.
|
||||
*/
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
prepareWorkspace,
|
||||
cleanupWorkspace,
|
||||
runTurn,
|
||||
fileLoaded,
|
||||
summarizeTrace,
|
||||
} from './harness.mjs';
|
||||
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
|
||||
import { PRODUCT_MD_SAMPLE, DESIGN_MD_SAMPLE } from './fixtures.mjs';
|
||||
|
||||
const LEGACY_DESIGN = `# Design
|
||||
|
||||
## Identity
|
||||
BORING_BEIGE_CARDS. Quiet beige panels, timid scale, rounded cards everywhere.
|
||||
|
||||
## Color
|
||||
Warm gray background with a muted tan accent.
|
||||
`;
|
||||
|
||||
const EXISTING_PAGE = `<!doctype html>
|
||||
<html><head><style>
|
||||
:root { --legacy-beige: #e8e1d5; --legacy-tan: #a78969; }
|
||||
body { background: var(--legacy-beige); color: #3c3833; font-family: Arial, sans-serif; }
|
||||
.card { border: 1px solid #cfc5b6; border-radius: 18px; padding: 24px; }
|
||||
</style></head><body>
|
||||
<header data-untouched="header"><a href="/">Harbor Desk</a></header>
|
||||
<main><section id="case-study" class="card"><h1>Harbor Desk</h1><p>Challenge. Approach. Outcome.</p><p>Image placeholder</p></section></main>
|
||||
<footer data-untouched="footer">Operational since 1987</footer>
|
||||
</body></html>`;
|
||||
|
||||
function firstCall(trace, predicate) {
|
||||
return trace.toolCalls.findIndex(predicate);
|
||||
}
|
||||
|
||||
function firstWrite(trace, pattern) {
|
||||
return firstCall(trace, ({ name, input }) => name === 'write' && pattern.test(input?.path ?? ''));
|
||||
}
|
||||
|
||||
function workflowTraceMessage(trace) {
|
||||
return JSON.stringify(summarizeTrace(trace), null, 2);
|
||||
}
|
||||
|
||||
for (const modelId of resolveModelList()) {
|
||||
const provider = detectProvider(modelId);
|
||||
const keyPresent = hasKey(provider);
|
||||
|
||||
describe(`skill workflow contract :: ${modelId}`, () => {
|
||||
if (!keyPresent) {
|
||||
it(`skipped — ${PROVIDERS[provider].envKey} is unset`, { skip: true }, () => {});
|
||||
return;
|
||||
}
|
||||
const model = getModel(modelId);
|
||||
|
||||
it('fresh init asks, writes PRODUCT without Register, then establishes DESIGN', async () => {
|
||||
const workspace = prepareWorkspace({ files: {} });
|
||||
try {
|
||||
const { trace } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable init for a harbor operations product, then finish setup.',
|
||||
maxSteps: 24,
|
||||
});
|
||||
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
||||
const productWrite = firstWrite(trace, /(^|\/)PRODUCT\.md$/i);
|
||||
const designWrite = firstWrite(trace, /(^|\/)DESIGN\.md$/i);
|
||||
assert.ok(fileLoaded(trace, 'init.md'), `init.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(question >= 0, `structured user was never asked.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(productWrite > question, `PRODUCT.md must follow a user answer.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(designWrite > productWrite, `DESIGN.md must follow PRODUCT.md.\n${workflowTraceMessage(trace)}`);
|
||||
const product = fs.readFileSync(path.join(workspace, 'PRODUCT.md'), 'utf8');
|
||||
assert.doesNotMatch(product, /^## Register\s*$/im);
|
||||
assert.match(product, /ferry|dispatch|harbor/i, 'PRODUCT.md should incorporate the simulated user context');
|
||||
assert.ok(fs.existsSync(path.join(workspace, 'DESIGN.md')));
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
|
||||
it('initialized craft asks for the task concept before implementation', async () => {
|
||||
const workspace = prepareWorkspace({
|
||||
files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE, 'DESIGN.md': DESIGN_MD_SAMPLE },
|
||||
});
|
||||
try {
|
||||
const { trace } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable craft a concise evidence-led case-study page. Leave it at index.html.',
|
||||
maxSteps: 22,
|
||||
});
|
||||
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
||||
const implementation = firstWrite(trace, /\.(?:html?|astro|svelte|jsx?|tsx?)$/i);
|
||||
assert.ok(fileLoaded(trace, 'new-work.md'), `new-work.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(question >= 0, `task concept was never put to the user.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(implementation > question, `implementation began before the attended concept checkpoint.\n${workflowTraceMessage(trace)}`);
|
||||
const artifact = fs.readFileSync(path.join(workspace, 'index.html'), 'utf8');
|
||||
assert.match(artifact.slice(0, 1400), /DIRECTION CONTRACT/i);
|
||||
for (const field of ['UNIQUE', 'NOT-TEMPLATE', 'OWN-WORLD', 'STORY', 'FIRST VIEWPORT', 'FORM']) {
|
||||
assert.match(artifact.slice(0, 1800), new RegExp(`${field}:`, 'i'));
|
||||
}
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
|
||||
it('redesign replaces DESIGN before touching the existing page', async () => {
|
||||
const workspace = prepareWorkspace({
|
||||
files: {
|
||||
'PRODUCT.md': PRODUCT_MD_SAMPLE,
|
||||
'DESIGN.md': LEGACY_DESIGN,
|
||||
'current.html': EXISTING_PAGE,
|
||||
},
|
||||
});
|
||||
try {
|
||||
const { trace } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable craft redesign current.html for this product. Leave the result at current.html.',
|
||||
maxSteps: 26,
|
||||
});
|
||||
const question = firstCall(trace, ({ name }) => name === 'ask_user_question');
|
||||
const designWrite = firstWrite(trace, /(^|\/)DESIGN\.md$/i);
|
||||
const implementation = firstWrite(trace, /(^|\/)current\.html$/i);
|
||||
assert.ok(fileLoaded(trace, 'init.md'), `redesign did not route through init.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(question >= 0, `replacement world was not put to the user.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(designWrite > question, `replacement DESIGN.md must follow user choice.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(implementation > designWrite, `redesign touched the page before replacing DESIGN.md.\n${workflowTraceMessage(trace)}`);
|
||||
const design = fs.readFileSync(path.join(workspace, 'DESIGN.md'), 'utf8');
|
||||
assert.notEqual(design.trim(), LEGACY_DESIGN.trim(), 'redesign preserved the old visual world verbatim');
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
|
||||
it('bolder refinement preserves the world and everything outside scope', async () => {
|
||||
const workspace = prepareWorkspace({
|
||||
files: {
|
||||
'PRODUCT.md': PRODUCT_MD_SAMPLE,
|
||||
'DESIGN.md': DESIGN_MD_SAMPLE,
|
||||
'current.html': EXISTING_PAGE,
|
||||
},
|
||||
});
|
||||
try {
|
||||
const { trace } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable bolder current.html, only the #case-study section. Keep everything else untouched.',
|
||||
maxSteps: 16,
|
||||
});
|
||||
const productWrite = firstWrite(trace, /(^|\/)PRODUCT\.md$/i);
|
||||
const designWrite = firstWrite(trace, /(^|\/)DESIGN\.md$/i);
|
||||
const implementation = firstWrite(trace, /(^|\/)current\.html$/i);
|
||||
assert.ok(fileLoaded(trace, 'bolder.md'), `bolder.md was not loaded.\n${workflowTraceMessage(trace)}`);
|
||||
assert.equal(productWrite, -1, `refinement rewrote PRODUCT.md.\n${workflowTraceMessage(trace)}`);
|
||||
assert.equal(designWrite, -1, `refinement rewrote DESIGN.md.\n${workflowTraceMessage(trace)}`);
|
||||
assert.ok(implementation >= 0, `refinement did not write current.html.\n${workflowTraceMessage(trace)}`);
|
||||
const artifact = fs.readFileSync(path.join(workspace, 'current.html'), 'utf8');
|
||||
assert.match(artifact, /data-untouched="header"/);
|
||||
assert.match(artifact, /data-untouched="footer"/);
|
||||
assert.match(artifact, /id="case-study"/);
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user