From 65ef0c7fe7dd4b219221c5ae902066ffc3f14343 Mon Sep 17 00:00:00 2001 From: Paul Bakaus Date: Mon, 17 Aug 2026 22:09:27 -0700 Subject: [PATCH] Tests: point the skill-behavior harness at the launcher and engine binary The bash tool exports IMPECCABLE_BIN so the staged skill's launcher runs without a download; scenarios assert on 'impeccable context' instead of context.mjs and skip without a binary. Prepared with AI assistance (Claude Code). --- tests/skill-behavior/README.md | 46 +++++++----- tests/skill-behavior/harness.mjs | 26 +++++-- tests/skill-behavior/scenarios.test.mjs | 72 ++++++++++--------- .../skill-behavior/workflow-contract.test.mjs | 6 ++ 4 files changed, 94 insertions(+), 56 deletions(-) diff --git a/tests/skill-behavior/README.md b/tests/skill-behavior/README.md index c9bb5038a..463fa0756 100644 --- a/tests/skill-behavior/README.md +++ b/tests/skill-behavior/README.md @@ -22,12 +22,26 @@ Requires `.env` at repo root with at least one of `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_CLOUD_API_KEY`, `DEEPSEEK_API_KEY`. Providers without a key are skipped, not failed. +Also requires the engine binary (`bun run fetch:engine`, or `IMPECCABLE_BIN`). +The staged skill dir ships the launcher (`scripts/impeccable`); the harness +exports `IMPECCABLE_BIN` into every bash call the agent makes, so the launcher +resolves the binary in both symlink and copy mode without a download. Without a +binary the suites skip. + +To run a single scenario against one model: + +```bash +IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-sonnet-5 IMPECCABLE_SKILL_BEHAVIOR_VERBOSE=1 \ + node --test --test-timeout=600000 --test-name-pattern="scenario 6" tests/skill-behavior/scenarios.test.mjs +``` + ## How it works Each scenario: 1. `prepareWorkspace()` mints a temp dir, symlinks the canonical skill - into `/.claude/skills/impeccable`, and optionally writes + into `/.claude/skills/impeccable` (so its launcher is at + `.claude/skills/impeccable/scripts/impeccable`), and optionally writes `PRODUCT.md` / `DESIGN.md` fixtures. 2. `runTurn()` inlines `SKILL.md` (placeholders neutralized) as the system prompt and runs Vercel AI SDK `generateText` with four @@ -43,20 +57,20 @@ The trace is the source of truth, not the model's free-form reply. | # | Setup | Assertion | |---|---|---| -| 1 | empty workspace | runs `context.mjs`; loads `reference/init.md` before implementation; automation is not an init bypass | -| 2 | PRODUCT.md only | runs `context.mjs` 1-3 times; loads `reference/new-work.md` to resolve visual authority, establish a world when needed, and develop the surface | -| 3 | PRODUCT.md + DESIGN.md | runs `context.mjs` 1-3 times; receives the committed design system and loads `reference/new-work.md` for the task-scoped concept | -| 4 | PRODUCT.md + DESIGN.md, context already loaded in turn 1 | turn 2 does **not** re-run `context.mjs` | -| 5 | PRODUCT.md without the legacy `## Register` field and no DESIGN.md | runs `context.mjs`; greenfield craft loads `reference/new-work.md`, not init, to establish the missing world | +| 1 | empty workspace | runs `impeccable context`; loads `reference/init.md` before implementation; automation is not an init bypass | +| 2 | PRODUCT.md only | runs `impeccable context` 1-3 times; loads `reference/new-work.md` to resolve visual authority, establish a world when needed, and develop the surface | +| 3 | PRODUCT.md + DESIGN.md | runs `impeccable context` 1-3 times; receives the committed design system and loads `reference/new-work.md` for the task-scoped concept | +| 4 | PRODUCT.md + DESIGN.md, context already loaded in turn 1 | turn 2 does **not** re-run `impeccable context` | +| 5 | PRODUCT.md without the legacy `## Register` field and no DESIGN.md | runs `impeccable context`; greenfield craft loads `reference/new-work.md`, not init, to establish the missing world | | 6 | PRODUCT.md + DESIGN.md + a minimal `index.html`; prompt is `/impeccable polish` | loads `reference/polish.md` | | 7 | same fixture; prompt is `/impeccable audit` | loads `reference/audit.md` | | 8 | PRODUCT.md + DESIGN.md + a SvelteKit scaffold (`src/app.css`, components, `+page.svelte`); prompt is `/impeccable polish src/routes/+page.svelte` | reads at least one project code file (CSS / component / page) — not just the skill's reference files | -| 9 | PRODUCT.md + `index.html` + a seeded update cache with a newer version (`skillVersion` copy-mode so `context.mjs` has a `SKILL.md` to version-check against); prompt is `/impeccable polish index.html` | `context.mjs` runs and its output carries the `UPDATE_AVAILABLE` directive (proven via captured bash output); the agent does **not** auto-run `npx impeccable update` (it must ask first) | -| 10 | no PRODUCT.md + a minimal `index.html`; prompt is `/impeccable polish index.html` | runs `context.mjs`, loads `reference/polish.md`, and does **not** divert into `reference/init.md` | -| 11 | empty workspace; prompt is `/impeccable shape ...` | runs `context.mjs`; resolves `reference/init.md` before planning the surface | -| 12 | empty workspace; prompt is natural-language build intent with no command word | runs `context.mjs`; resolves `reference/init.md` before implementation | -| 13 | empty workspace; prompt is `/impeccable teach` | runs `context.mjs` and diverts into `reference/init.md` because `teach` aliases `init` | -| 14 | PRODUCT.md with `## Platform: ios` (native iOS app); prompt is `/impeccable craft a tide detail screen` | `context.mjs` runs and emits the contents of `reference/ios.md` directly, placing native conventions in context without a second model-directed read | +| 9 | PRODUCT.md + `index.html` + a seeded update cache with a newer version (`skillVersion` copy-mode so `impeccable context` has a `SKILL.md` to version-check against); prompt is `/impeccable polish index.html` | `impeccable context` runs and its output carries the `UPDATE_AVAILABLE` directive (proven via captured bash output); the agent does **not** auto-run `npx impeccable update` (it must ask first) | +| 10 | no PRODUCT.md + a minimal `index.html`; prompt is `/impeccable polish index.html` | runs `impeccable context`, loads `reference/polish.md`, and does **not** divert into `reference/init.md` | +| 11 | empty workspace; prompt is `/impeccable shape ...` | runs `impeccable context`; resolves `reference/init.md` before planning the surface | +| 12 | empty workspace; prompt is natural-language build intent with no command word | runs `impeccable context`; resolves `reference/init.md` before implementation | +| 13 | empty workspace; prompt is `/impeccable teach` | runs `impeccable context` and diverts into `reference/init.md` because `teach` aliases `init` | +| 14 | PRODUCT.md with `## Platform: ios` (native iOS app); prompt is `/impeccable craft a tide detail screen` | `impeccable context` runs and emits the contents of `reference/ios.md` directly, placing native conventions in context without a second model-directed read | | 15 | same iOS fixture; prompt is `/impeccable audit` | agent loads `reference/audit.native.md` (the Commands-table native variant, routed instead of `audit.md`) | The workflow-contract file adds end-to-end assertions for attended fresh init, @@ -170,7 +184,7 @@ lineup. Only the scenarios under investigation were scoped per model. The rows are worth keeping anyway, since a scenario absent from the table is easy to mistake for a scenario that passed. -**`bolder refinement`, deepseek-v4-flash.** The model runs `context.mjs`, reads +**`bolder refinement`, deepseek-v4-flash.** The model runs `impeccable context`, reads `bolder.md`, `craft-floor.md`, and `current.html`, then ends its turn without editing anything: empty `writePaths`, no `ask_user_question` call, well short of the 16-step cap. Confirmed identical on HEAD with `bolder.md` reverted, so it is @@ -241,7 +255,7 @@ IMPECCABLE_QUESTION_DISABLED=1 CI=1 IMPECCABLE_SKILL_BEHAVIOR_MODELS=deepseek-v4 Keep `--test-timeout` at 300000. A tighter cap turns claude-sonnet-5's slower runs into timeouts that look like failures. Set `IMPECCABLE_QUESTION_DISABLED=1` -and `CI=1` so `serve-question.mjs` cannot open a browser window on the host. Pipe +and `CI=1` so `impeccable serve-question` cannot open a browser window on the host. Pipe to a file rather than `tail`; node prints the failing-test summary at the end, and truncating it costs you the per-model attribution. @@ -256,14 +270,14 @@ and truncating it costs you the per-model attribution. Captured after moving sub-command reference loading from step 4 to step 2 of Setup (so the agent loads `reference/.md` right after -`context.mjs`, before "doing the work" preempts it), and tightening +`impeccable context`, before "doing the work" preempts it), and tightening step 3 to require at least one project code read even when a sub-command reference loads first. Use this table when comparing pre/post refactor: a regression is "more failures than baseline", not "any failures at all". | Scenario | claude-haiku-4-5 | gpt-5.4-mini | gemini-3.1-flash-lite | |---|---|---|---| -| 1 (no context) | pass (rare flake — agent stops after `context.mjs` without loading `init.md`) | pass | pass | +| 1 (no context) | pass (rare flake — agent stops after `impeccable context` without loading `init.md`) | pass | pass | | 2 (product only) | pass | pass | pass | | 3 (product + design) | pass | pass | pass (rare flake — sub-command ref loads but world ref doesn't) | | 4 (already loaded) | pass | pass | pass | diff --git a/tests/skill-behavior/harness.mjs b/tests/skill-behavior/harness.mjs index e9f4187b1..98898a6fb 100644 --- a/tests/skill-behavior/harness.mjs +++ b/tests/skill-behavior/harness.mjs @@ -4,8 +4,8 @@ * Each scenario: * 1. Creates a temp workspace. * 2. Symlinks the real .claude/skills/impeccable into the workspace so - * scripts (context.mjs, etc.) resolve from the canonical path - * the skill references. + * the launcher (`scripts/impeccable`) resolves from the canonical path + * the skill references, and points it at an engine binary. * 3. Optionally writes PRODUCT.md / DESIGN.md fixtures. * 4. Inlines SKILL.md as the system prompt (placeholders stripped to * neutral values so the same body works for all providers). @@ -27,6 +27,7 @@ import path from 'node:path'; import { spawn } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { getProviderOptions } from './providers.mjs'; +import { ENGINE_MISSING_MESSAGE, findEngineBinary } from '../lib/engine-bin.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(__dirname, '..', '..'); @@ -98,10 +99,18 @@ export const SKILL_BODY = loadSkillBody(); * their content. * - `files` lets the test seed PRODUCT.md / DESIGN.md (or anything else). * - `skillVersion` switches from symlink to a real COPY of the skill dir and - * writes a `SKILL.md` carrying that version. context.mjs reads its own - * version from that sibling file, so this is required for any scenario that - * exercises the update-check path (the source dir has only SKILL.src.md). + * writes a `SKILL.md` carrying that version. `impeccable context` reads its + * own version from that sibling file, so this is required for any scenario + * that exercises the update-check path (the source dir has only SKILL.src.md). + * + * The launcher in the staged scripts dir needs an engine binary. Every bash + * call the agent makes gets `IMPECCABLE_BIN` (tests/lib/engine-bin.mjs: + * `IMPECCABLE_BIN` or `skill/scripts/bin/-/`), which the launcher + * honors first, so the symlink and copy modes both work without a download. */ +export const ENGINE_BIN = findEngineBinary(); +export { ENGINE_MISSING_MESSAGE }; + export function prepareWorkspace({ files = {}, skillVersion = null } = {}) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-skill-test-')); const skillDest = path.join(dir, '.claude', 'skills', 'impeccable'); @@ -145,7 +154,10 @@ function safeResolve(root, userPath) { function execBash(workspace, command, timeoutMs = 20_000, extraEnv = {}) { return new Promise((resolve) => { - const proc = spawn('bash', ['-lc', command], { cwd: workspace, env: { ...process.env, ...extraEnv } }); + const proc = spawn('bash', ['-lc', command], { + cwd: workspace, + env: { ...process.env, ...(ENGINE_BIN ? { IMPECCABLE_BIN: ENGINE_BIN } : {}), ...extraEnv }, + }); let stdout = ''; let stderr = ''; const truncatedFlag = { val: false }; @@ -231,7 +243,7 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}) { const tools = { bash: tool({ description: - 'Run a bash command in the workspace root. Use this to invoke skill scripts (e.g. `node .claude/skills/impeccable/scripts/context.mjs`).', + 'Run a bash command in the workspace root. Use this to invoke skill commands (e.g. `.claude/skills/impeccable/scripts/impeccable context`).', inputSchema: z.object({ command: z.string().describe('The bash command to execute.'), }), diff --git a/tests/skill-behavior/scenarios.test.mjs b/tests/skill-behavior/scenarios.test.mjs index e1304c2b8..5b22eb883 100644 --- a/tests/skill-behavior/scenarios.test.mjs +++ b/tests/skill-behavior/scenarios.test.mjs @@ -23,6 +23,8 @@ import { readsMatching, fileLoaded, summarizeTrace, + ENGINE_BIN, + ENGINE_MISSING_MESSAGE, } from './harness.mjs'; import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs'; import { @@ -86,6 +88,10 @@ for (const modelId of resolveModelList()) { it(`skipped — ${PROVIDERS[provider].envKey} is unset`, { skip: true }, () => {}); return; } + if (!ENGINE_BIN) { + it(`skipped — ${ENGINE_MISSING_MESSAGE}`, { skip: true }, () => {}); + return; + } const model = getModel(modelId); // Gemini Flash tends to inspect one file at a time, while the production // Anthropic/OpenAI models batch setup reads and then begin implementation. @@ -104,10 +110,10 @@ for (const modelId of resolveModelList()) { maxSteps: setupMaxSteps, }); logTrace('S1', 'no-context', modelId, trace, { textSample: text.slice(0, 400) }); - const loadCalls = bashCommandsMatching(trace, 'context.mjs'); + const loadCalls = bashCommandsMatching(trace, 'impeccable context'); assert.ok( loadCalls.length >= 1, - `expected agent to run context.mjs at least once; got ${loadCalls.length}.\n` + + `expected agent to run impeccable context at least once; got ${loadCalls.length}.\n` + `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, ); assert.ok( @@ -137,10 +143,10 @@ for (const modelId of resolveModelList()) { maxSteps: setupMaxSteps, }); logTrace('S2', 'product-only', modelId, trace, { textSample: text.slice(0, 400) }); - const loadCalls = bashCommandsMatching(trace, 'context.mjs'); + const loadCalls = bashCommandsMatching(trace, 'impeccable context'); assert.ok( loadCalls.length >= 1 && loadCalls.length <= 3, - `expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` + + `expected 1-3 impeccable context invocations; got ${loadCalls.length}.\n` + `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`, ); assert.ok( @@ -165,10 +171,10 @@ for (const modelId of resolveModelList()) { maxSteps: setupMaxSteps, }); logTrace('S3', 'product-and-design', modelId, trace, { textSample: text.slice(0, 400) }); - const loadCalls = bashCommandsMatching(trace, 'context.mjs'); + const loadCalls = bashCommandsMatching(trace, 'impeccable context'); assert.ok( loadCalls.length >= 1 && loadCalls.length <= 3, - `expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` + + `expected 1-3 impeccable context invocations; got ${loadCalls.length}.\n` + `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`, ); assert.ok( @@ -177,7 +183,7 @@ for (const modelId of resolveModelList()) { `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, ); // The skill tells the agent to also familiarize with the existing - // design system. DESIGN.md is bundled in context.mjs output, but + // design system. DESIGN.md is bundled in impeccable context output, but // exploring CSS / tokens / theme files or a directory listing // also counts. const designSignal = @@ -200,7 +206,7 @@ for (const modelId of resolveModelList()) { files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE, 'DESIGN.md': DESIGN_MD_SAMPLE }, }); try { - // Turn 1: prime the conversation so context.mjs gets run and its + // Turn 1: prime the conversation so impeccable context gets run and its // output enters the message history. const turn1 = await runTurn({ workspace, @@ -209,10 +215,10 @@ for (const modelId of resolveModelList()) { maxSteps: setupMaxSteps, }); logTrace('S4-T1', 'primer', modelId, turn1.trace, { textSample: turn1.text.slice(0, 200) }); - const turn1Loads = bashCommandsMatching(turn1.trace, 'context.mjs'); + const turn1Loads = bashCommandsMatching(turn1.trace, 'impeccable context'); assert.ok( turn1Loads.length >= 1, - `primer turn should have run context.mjs. bash: ${JSON.stringify(turn1.trace.bashCommands, null, 2)}`, + `primer turn should have run impeccable context. bash: ${JSON.stringify(turn1.trace.bashCommands, null, 2)}`, ); // Turn 2: the real ask. The skill says "skip if you've already @@ -225,11 +231,11 @@ for (const modelId of resolveModelList()) { maxSteps: setupMaxSteps, }); logTrace('S4-T2', 'follow-up', modelId, turn2.trace, { textSample: turn2.text.slice(0, 400) }); - const turn2Loads = bashCommandsMatching(turn2.trace, 'context.mjs'); + const turn2Loads = bashCommandsMatching(turn2.trace, 'impeccable context'); assert.equal( turn2Loads.length, 0, - `agent re-ran context.mjs on turn 2 despite it being in prior conversation. ` + + `agent re-ran impeccable context on turn 2 despite it being in prior conversation. ` + `bashCommands: ${JSON.stringify(turn2.trace.bashCommands, null, 2)}`, ); } finally { @@ -249,10 +255,10 @@ for (const modelId of resolveModelList()) { maxSteps: setupMaxSteps, }); logTrace('S5', 'legacy-product', modelId, trace, { textSample: text.slice(0, 400) }); - const loadCalls = bashCommandsMatching(trace, 'context.mjs'); + const loadCalls = bashCommandsMatching(trace, 'impeccable context'); assert.ok( loadCalls.length >= 1, - `expected context.mjs invocation; got ${loadCalls.length}.\n` + + `expected impeccable context invocation; got ${loadCalls.length}.\n` + `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`, ); assert.ok(fileLoaded(trace, 'new-work.md'), @@ -350,13 +356,13 @@ for (const modelId of resolveModelList()) { }); it('scenario 9: update-available directive is surfaced, never auto-run', async () => { - // context.mjs reads a newer version from its (seeded) cache and appends + // impeccable context reads a newer version from its (seeded) cache and appends // an UPDATE_AVAILABLE directive to the boot output. The agent must // surface it and keep working, but must NOT run `npx impeccable update` // on its own — modifying installed files mid-session without // consent is the exact failure this guards against. // - // `skillVersion` forces copy-mode so context.mjs has a SKILL.md sibling + // `skillVersion` forces copy-mode so impeccable context has a SKILL.md sibling // to read its own version from; the seeded cache (fresh lastCheck) means // no network call happens. const workspace = prepareWorkspace({ @@ -379,14 +385,14 @@ for (const modelId of resolveModelList()) { // Boot ran, so the directive entered the agent's view. assert.ok( - bashCommandsMatching(trace, 'context.mjs').length >= 1, - `expected agent to run context.mjs. bash: ${JSON.stringify(trace.bashCommands, null, 2)}`, + bashCommandsMatching(trace, 'impeccable context').length >= 1, + `expected agent to run impeccable context. bash: ${JSON.stringify(trace.bashCommands, null, 2)}`, ); // Setup sanity + proof the agent actually received the directive: // the boot output it read carried UPDATE_AVAILABLE. assert.ok( trace.bashOutputs.some((o) => o.includes('UPDATE_AVAILABLE')), - `context.mjs should have emitted UPDATE_AVAILABLE (a newer version is cached).\n` + + `impeccable context should have emitted UPDATE_AVAILABLE (a newer version is cached).\n` + `bashOutputs: ${JSON.stringify(trace.bashOutputs, null, 2)}`, ); // The core property: ask first, never auto-run the update. @@ -421,8 +427,8 @@ for (const modelId of resolveModelList()) { logTrace('S10', 'scoped-no-product', modelId, trace, { textSample: text.slice(0, 400) }); // Boot still runs. assert.ok( - bashCommandsMatching(trace, 'context.mjs').length >= 1, - `expected agent to run context.mjs at least once.\n` + + bashCommandsMatching(trace, 'impeccable context').length >= 1, + `expected agent to run impeccable context at least once.\n` + `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, ); // It must load the scoped command's own reference and get on with it. @@ -458,8 +464,8 @@ for (const modelId of resolveModelList()) { }); logTrace('S11', 'shape-no-context', modelId, trace, { textSample: text.slice(0, 400) }); assert.ok( - bashCommandsMatching(trace, 'context.mjs').length >= 1, - `expected agent to run context.mjs at least once.\n` + + bashCommandsMatching(trace, 'impeccable context').length >= 1, + `expected agent to run impeccable context at least once.\n` + `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, ); assert.ok( @@ -483,8 +489,8 @@ for (const modelId of resolveModelList()) { }); logTrace('S12', 'natural-build-no-context', modelId, trace, { textSample: text.slice(0, 400) }); assert.ok( - bashCommandsMatching(trace, 'context.mjs').length >= 1, - `expected agent to run context.mjs at least once.\n` + + bashCommandsMatching(trace, 'impeccable context').length >= 1, + `expected agent to run impeccable context at least once.\n` + `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, ); assert.ok( @@ -510,8 +516,8 @@ for (const modelId of resolveModelList()) { }); logTrace('S13', 'teach-no-context', modelId, trace, { textSample: text.slice(0, 400) }); assert.ok( - bashCommandsMatching(trace, 'context.mjs').length >= 1, - `expected agent to run context.mjs at least once.\n` + + bashCommandsMatching(trace, 'impeccable context').length >= 1, + `expected agent to run impeccable context at least once.\n` + `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`, ); const initLoaded = @@ -528,7 +534,7 @@ for (const modelId of resolveModelList()) { }); it('scenario 14: native iOS project (context loads ios.md)', async () => { - // PRODUCT.md sets `## Platform` to `ios`. context.mjs now reads and emits + // PRODUCT.md sets `## Platform` to `ios`. impeccable context now reads and emits // reference/ios.md itself, so native guidance enters the conversation // without relying on a second model-directed file read. const workspace = prepareWorkspace({ @@ -542,16 +548,16 @@ for (const modelId of resolveModelList()) { maxSteps: provider === 'google' ? 8 : 6, }); logTrace('S14', 'native-ios', modelId, trace, { textSample: text.slice(0, 400) }); - const loadCalls = bashCommandsMatching(trace, 'context.mjs'); + const loadCalls = bashCommandsMatching(trace, 'impeccable context'); assert.ok( loadCalls.length >= 1, - `expected agent to run context.mjs at least once; got ${loadCalls.length}.\n` + + `expected agent to run impeccable context at least once; got ${loadCalls.length}.\n` + `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`, ); // Proof the native reference itself entered the agent's view. assert.ok( trace.bashOutputs.some((o) => /# NATIVE PLATFORM REFERENCE: IOS \(reference\/ios\.md\)/.test(o)), - `context.mjs should have emitted reference/ios.md content (platform is ios).\n` + + `impeccable context should have emitted reference/ios.md content (platform is ios).\n` + `bashOutputs: ${JSON.stringify(trace.bashOutputs, null, 2)}`, ); } finally { @@ -578,8 +584,8 @@ for (const modelId of resolveModelList()) { }); logTrace('S15', 'native-audit-variant', modelId, trace, { textSample: text.slice(0, 400) }); assert.ok( - bashCommandsMatching(trace, 'context.mjs').length >= 1, - `expected agent to run context.mjs at least once.\n` + + bashCommandsMatching(trace, 'impeccable context').length >= 1, + `expected agent to run impeccable context at least once.\n` + `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`, ); assert.ok( diff --git a/tests/skill-behavior/workflow-contract.test.mjs b/tests/skill-behavior/workflow-contract.test.mjs index 13c04d2b4..93dc38716 100644 --- a/tests/skill-behavior/workflow-contract.test.mjs +++ b/tests/skill-behavior/workflow-contract.test.mjs @@ -13,6 +13,8 @@ import { runTurn, fileLoaded, summarizeTrace, + ENGINE_BIN, + ENGINE_MISSING_MESSAGE, } from './harness.mjs'; import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs'; import { PRODUCT_MD_SAMPLE, DESIGN_MD_SAMPLE } from './fixtures.mjs'; @@ -107,6 +109,10 @@ for (const modelId of resolveModelList()) { it(`skipped — ${PROVIDERS[provider].envKey} is unset`, { skip: true }, () => {}); return; } + if (!ENGINE_BIN) { + it(`skipped — ${ENGINE_MISSING_MESSAGE}`, { skip: true }, () => {}); + return; + } const model = getModel(modelId); it('fresh init asks and writes PRODUCT without inventing a visual system', async () => {