diff --git a/tests/skill-behavior-harness.test.mjs b/tests/skill-behavior-harness.test.mjs index 797317e34..1c994140c 100644 --- a/tests/skill-behavior-harness.test.mjs +++ b/tests/skill-behavior-harness.test.mjs @@ -184,6 +184,29 @@ it('protocol checkpoints stop at successful evidence without claiming task compl } finally { cleanupWorkspace(workspace); } }); +it('protocol notice checkpoints observe intermediate assistant text', async () => { + const workspace = prepareWorkspace(); + try { + let calls = 0; + const model = new MockLanguageModelV3({ doGenerate: async () => { + calls++; + return { + content: [{ type: 'text', text: 'Impeccable version 99.0.0 is available; may I update it?' }, + { type: 'tool-call', toolCallId: 'list', toolName: 'list', input: '{}' }], + finishReason: { unified: 'tool-calls', raw: 'tool-calls' }, + usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } }, warnings: [], + }; + } }); + const result = await runTurn({ workspace, model, userPrompt: 'Inspect this page.', maxSteps: 10, + stopAfter: (trace) => trace.assistantTexts?.some((text) => text.includes('99.0.0')) }); + assert.equal(calls, 1); + assert.equal(result.outcome, 'checkpoint'); + assert.match(result.trace.assistantTexts[0], /may I update/); + } finally { + cleanupWorkspace(workspace); + } +}); + it('optional diagnostics retain tool evidence when a provider turn fails', async () => { const workspace = prepareWorkspace({ files: { 'PRODUCT.md': 'Synthetic product context.' } }); const previous = process.env.IMPECCABLE_SKILL_BEHAVIOR_TRACE_DIR; diff --git a/tests/skill-behavior/README.md b/tests/skill-behavior/README.md index 4716561b6..eb7f56ec9 100644 --- a/tests/skill-behavior/README.md +++ b/tests/skill-behavior/README.md @@ -48,6 +48,10 @@ A broader run exposed the context-only allowlist rejecting Svelte's valid `+page.svelte` target. It was stopped, the allowlist fixed with a failing-then- passing unit test, and S8 passed 3/3 on the focused rerun. Failed file reads do not count as project exploration. +Update-notice (S9) and explicit-command (S18) checks also now stop at their +actual protocol checkpoints, instead of continuing into unrelated polishing; +their focused final reruns each passed 3/3. S9 now explicitly requires the +assistant to surface the update, not merely receive its loader directive. Full workflows moved to `tests/skill-workflow/full-build.test.mjs`: @@ -73,6 +77,29 @@ CI runs this lane only when its manual `skill_workflow` checkbox is enabled. Ordinary protocol CI now fetches its engine instead of silently skipping for a missing binary. Full-build results must be reported separately from routing. +### Remaining gaps after the split (2026-09-07) + +One provisioned Claude natural-build run reached a final response in 637 seconds +and 37 model steps, with approval, a surface brief, an implemented page, a +finish review, corrections, and fresh desktop/mobile screenshots. Its initial +completion assertions passed. The final test revision additionally requires +the shipped documentation reference; auditing the saved trace against that +guard found it missing. **This is not a final full-workflow pass.** Existing +DESIGN.md was preserved, but the required documentation pass was skipped. +The final guards were tightened during the run; this trace is not represented +as a run of those later assertions. No second full build was purchased. + +Claude's remaining protocol batch passed S10–S15 and existing-project S16, +but missing-context S16 omitted routing.md and S17 omitted critique.md. +Both responses remained read-only. The older S9 timed out during unrelated +polishing; the corrected focused S9 above supersedes it. The batch was stopped +during the older S18, before another provider sweep. This is incremental +evidence, not an all-green final matrix. Release #782 remains on hold. + +The full build reported about 2.95 million input and 53 thousand output tokens +across all turns (no cache usage reported). Keep this lane manually scoped; +the fast protocol suite is not a proxy for its completion or cost. + Each scenario: 1. `prepareWorkspace()` uses the production transformer to build current source diff --git a/tests/skill-behavior/harness.mjs b/tests/skill-behavior/harness.mjs index fd9b8a275..2c3393527 100644 --- a/tests/skill-behavior/harness.mjs +++ b/tests/skill-behavior/harness.mjs @@ -427,7 +427,10 @@ export async function runTurn({ workspace, model, userPrompt, priorMessages = [] system: environment ? `${SKILL_BODY}\n\nRuntime environment: ${environment}` : SKILL_BODY, messages, tools, - onStepFinish: tracePath ? (step) => saveTrace({ status: 'in-progress', lastStepMessages: step.response.messages }) : undefined, + onStepFinish: (step) => { + (trace.assistantTexts ??= []).push(step.text ?? ''); + saveTrace({ status: 'in-progress', lastStepMessages: step.response.messages }); + }, stopWhen: [stepCountIs(maxSteps), ...(stopAfter ? [() => stopAfter(trace)] : [])], // Real client-side deadline on the provider call: without it a stalled // stream wedges the whole sweep with no tally. diff --git a/tests/skill-behavior/scenarios.test.mjs b/tests/skill-behavior/scenarios.test.mjs index 63948ee36..8057c91a2 100644 --- a/tests/skill-behavior/scenarios.test.mjs +++ b/tests/skill-behavior/scenarios.test.mjs @@ -55,6 +55,7 @@ function projectCodeReads(trace) { } const SHAPE_PROMPT = '/impeccable shape a landing page for the project in this workspace'; const NATURAL_BUILD_PROMPT = 'Build a landing page for the project in this workspace.'; +const UPDATE_NOTICE = /(?:skill|impeccable).{0,100}(?:update|version)|(?:update|version).{0,100}(?:skill|impeccable)|99\.0\.0/i; const TEACH_PROMPT = '/impeccable teach'; const PRIMER_PROMPT = 'Take a quick look at the project. What context should guide later design work? Run the impeccable context loader once if you need to.'; @@ -403,6 +404,7 @@ for (const modelId of resolveModelList()) { userPrompt: '/impeccable polish index.html', maxSteps: setupMaxSteps, env: { IMPECCABLE_UPDATE_CACHE: path.join(workspace, '.impeccable-update.json') }, + checkpoint: (trace) => trace.assistantTexts?.some((text) => UPDATE_NOTICE.test(text)), }); logTrace('S9', 'update-available', modelId, trace, { textSample: text.slice(0, 400) }); @@ -419,6 +421,7 @@ for (const modelId of resolveModelList()) { `bashOutputs: ${JSON.stringify(trace.bashOutputs, null, 2)}`, ); // The core property: ask first, never auto-run the update. + assert.ok(trace.assistantTexts?.some((text) => UPDATE_NOTICE.test(text)), 'the skill update must be surfaced to the user'); const ranUpdate = executedUpdateCommands(trace); assert.equal( ranUpdate.length, @@ -747,6 +750,7 @@ for (const modelId of resolveModelList()) { workspace, model, userPrompt: '/impeccable polish index.html. Please do the polish pass now; afterward tell me which command would be useful next.', + checkpoint: 'polish.md', maxSteps: 8, contextOnlyBash: true, });