From 7a7579858c5ac0a64e7f58969b06309cfd3f4c4e Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Thu, 10 Sep 2026 13:44:25 -0700 Subject: [PATCH] test: scenario 19 documentation case when the context launcher is denied (#791) * Add degraded Setup path: must-read pack when the context launcher is refused When the host denies the impeccable context launcher (issue #789, measured in #744), the Setup fallback now names the degraded path and its unconditional must-read pack: the routed command's reference and craft-floor.md before any UI edit, and document.md before writing DESIGN.md. init.md gains the degraded Step 1 behavior, docs/CLI-CONTRACT.md documents the degraded contract, and scenario 19 gains a denied-launcher documentation case asserting document.md and source reads precede the DESIGN.md write. No version bump, no changelog entry, no generated harness sync. AI was used for assistance. Includes AI_PR_NOTICE.txt per the repository's contribution policy: this change was prepared without maintainer approval on issue #789, so no PR is opened by the agent. Co-authored-by: Matt Van Horn * Drop restated degraded-setup prose; keep the scenario 19 documentation case The launcher-unavailable path already lives on main. This removes the notice file and the restated SKILL, init, and CLI-contract text, and keeps the denied-launcher documentation coverage. The notice must now land before the first tool call after the denial, not only before the eventual write. AI was used for assistance. Co-authored-by: Matt Van Horn --------- Co-authored-by: Cursor Agent --- tests/skill-behavior-harness.test.mjs | 20 +++++++++++++++- tests/skill-behavior/README.md | 16 ++++++++++++- tests/skill-behavior/assertions.mjs | 27 ++++++++++++++++++--- tests/skill-behavior/scenarios.test.mjs | 31 ++++++++++++++++++++++++- 4 files changed, 88 insertions(+), 6 deletions(-) diff --git a/tests/skill-behavior-harness.test.mjs b/tests/skill-behavior-harness.test.mjs index f36f6a536..781702d35 100644 --- a/tests/skill-behavior-harness.test.mjs +++ b/tests/skill-behavior-harness.test.mjs @@ -4,7 +4,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { MockLanguageModelV3 } from 'ai/test'; import { prepareWorkspace, cleanupWorkspace, makeTools, runTurn, fileLoaded, SKILL_BODY } from './skill-behavior/harness.mjs'; -import { assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs'; +import { assertLauncherDenialWarningBeforeNextTool, assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs'; import { CASE_STUDY_ANSWER } from './skill-behavior/fixtures.mjs'; import { sourceHash as hashSources } from './skill-workflow/source-hash.mjs'; import { assertCompleted, assertFreshCaptures, assertNoChangeDocumentation, assertDocumentationArtifacts } from './skill-workflow/assertions.mjs'; @@ -186,6 +186,24 @@ it('headless behavior shells disable unattended decision pages and omit provider } }); +it('documentation fallback requires an assistant warning before the first tool call after denial', () => { + const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] }; + const denial = { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'context', toolName: 'bash', output: { type: 'text', value: 'Error: Bash permission denied by the host. This command was not executed.' } }] }; + const warning = { role: 'assistant', content: 'Context loading did not run because the launcher was denied.' }; + const read = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'read', toolName: 'read', input: { path: 'reference/document.md' } }] }; + const write = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'write', toolName: 'write', input: { path: 'DESIGN.md' } }] }; + assert.doesNotThrow(() => assertLauncherDenialWarningBeforeNextTool([call, denial, warning, read, write])); + assert.doesNotThrow(() => assertLauncherDenialWarningBeforeNextTool([call, denial, { role: 'assistant', content: [{ type: 'text', text: warning.content }, ...read.content] }, write])); + for (const messages of [ + [call, denial, read, warning, write], // Reads first, warns only before the write. + [call, denial, read, write, warning], // Final-only disclosure. + [warning, call, denial, read], // Not a response to the actual denial. + [call, denial, warning], // Warning with no follow-up tool call. + ]) { + assert.throws(() => assertLauncherDenialWarningBeforeNextTool(messages), assert.AssertionError); + } +}); + it('planning fallback requires an assistant warning between the denial and context reads', () => { const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] }; const denial = { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'context', toolName: 'bash', output: { type: 'text', value: 'Error: Bash permission denied by the host. This command was not executed.' } }] }; diff --git a/tests/skill-behavior/README.md b/tests/skill-behavior/README.md index 19a14db3d..5e7456005 100644 --- a/tests/skill-behavior/README.md +++ b/tests/skill-behavior/README.md @@ -321,7 +321,7 @@ results remain the completed measurements. | 16 | existing surface, with and without PRODUCT.md; asks where to start | completes relevant advice without edits, interviews, critique archives, menu scans, or explicit invented refinement prerequisites; reference coverage is diagnostic | | 17 | existing surface; asks whether critique is required before polish | completes read-only advice distinguishing assessment from implementation and explaining critique is optional; reference coverage is diagnostic | | 18 | existing surface; explicitly requests polish followed by a next-command recommendation | loads `polish.md` rather than substituting workflow advice for the requested work | -| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, and a denied-launcher planning-only case | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor | +| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, a denied-launcher planning-only case, and a denied-launcher documentation case (PRODUCT.md + index.html, no DESIGN.md) | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor; documentation requires successful document.md and source reads before any DESIGN.md write, with the denial disclosed before the first tool call after the denied launcher | ## Setup launcher-failure branch (2026-09-06, PR #750) @@ -405,6 +405,20 @@ IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-sonnet-5,gpt-5.6-terra,gemini-3.7-flash, node --test --test-name-pattern='scenario 19:' tests/skill-behavior/scenarios.test.mjs ``` +### Degraded documentation case (#789) + +A fourth scenario 19 case covers the documentation branch: launcher denied, +PRODUCT.md plus an incumbent `index.html`, prompt `/impeccable document`. +It requires an actual denied context attempt, successful `document.md` and +source reads before the DESIGN.md write, the denial disclosed before the +first tool call after the denied launcher, and an untouched PRODUCT.md. + +Scenario 19 on this branch, all four cases, one run each on 2026-09-09: +4/4 on claude-sonnet-5, 4/4 on gpt-5.6-terra, 4/4 on gemini-3.7-flash. The +documentation case also passes with main's unchanged `SKILL.src.md` and +`init.md` plus this test file, so the coverage is the test, not a skill-text +change. + ## Workflow-advice baseline (2026-09-05, PR #737) The four cases in scenarios 16-18 are new; prior scenario results do not diff --git a/tests/skill-behavior/assertions.mjs b/tests/skill-behavior/assertions.mjs index fc8ec5e32..236e9a3f1 100644 --- a/tests/skill-behavior/assertions.mjs +++ b/tests/skill-behavior/assertions.mjs @@ -61,17 +61,25 @@ export function assertNewWorkLifecycle(trace, { target, redesign = false }) { export const LAUNCHER_FAILURE_WARNING = /(?:context|launcher|bash)[^.!?\n]{0,160}(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|did(?:n't| not)|fail|unable)|(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|unable)[^.!?\n]{0,160}(?:context|launcher|bash)/i; -export function assertPlanningFallbackWarning(responseMessages) { - const blocks = responseMessages.flatMap((message) => +function responseBlocks(responseMessages) { + return responseMessages.flatMap((message) => (typeof message.content === 'string' ? [{ type: 'text', text: message.content }] : message.content) .map((block) => ({ ...block, role: message.role })), ); +} + +function contextLauncherDenialIndex(blocks) { const contextCalls = new Set(blocks.filter((block) => block.role === 'assistant' && block.type === 'tool-call' && block.toolName === 'bash' && /impeccable\s+context\b/.test(block.input?.command ?? '')).map((block) => block.toolCallId)); - const denialIndex = blocks.findIndex((block) => block.role === 'tool' + return blocks.findIndex((block) => block.role === 'tool' && block.type === 'tool-result' && contextCalls.has(block.toolCallId) && block.output?.type === 'text' && /Bash permission denied by the host/.test(block.output.value)); +} + +export function assertPlanningFallbackWarning(responseMessages) { + const blocks = responseBlocks(responseMessages); + const denialIndex = contextLauncherDenialIndex(blocks); assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence'); const warningIndex = blocks.findIndex((block, index) => index > denialIndex && block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text)); @@ -81,3 +89,16 @@ export function assertPlanningFallbackWarning(responseMessages) { assert.ok(warningIndex > denialIndex && contextReadIndex > warningIndex, 'planning fallback must warn after denial and before reading project context, not only in the final response'); } + +export function assertLauncherDenialWarningBeforeNextTool(responseMessages) { + const blocks = responseBlocks(responseMessages); + const denialIndex = contextLauncherDenialIndex(blocks); + assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence'); + const warningIndex = blocks.findIndex((block, index) => index > denialIndex + && block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text)); + const nextToolIndex = blocks.findIndex((block, index) => index > denialIndex + && block.role === 'assistant' && block.type === 'tool-call'); + assert.ok(nextToolIndex >= 0, 'must continue with a tool call after the denied launcher'); + assert.ok(warningIndex > denialIndex && nextToolIndex > warningIndex, + 'must disclose the failed context launcher before the first tool call after the denial, not only before the eventual write'); +} diff --git a/tests/skill-behavior/scenarios.test.mjs b/tests/skill-behavior/scenarios.test.mjs index d4c5c63a5..7057231f1 100644 --- a/tests/skill-behavior/scenarios.test.mjs +++ b/tests/skill-behavior/scenarios.test.mjs @@ -28,7 +28,7 @@ import { ENGINE_MISSING_MESSAGE, } from './harness.mjs'; import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs'; -import { assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs'; +import { assertLauncherDenialWarningBeforeNextTool, assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs'; import { assertCompleted } from '../skill-workflow/assertions.mjs'; import { PRODUCT_MD_SAMPLE, @@ -708,6 +708,35 @@ for (const modelId of resolveModelList()) { }); } + it('scenario 19: denied launcher requires document.md before writing DESIGN.md', async () => { + const workspace = prepareWorkspace({ files: { + 'PRODUCT.md': PRODUCT_MD_SAMPLE, + 'index.html': MINIMAL_LANDING_HTML, + } }); + try { + const { trace, stepTexts, finishReason, responseMessages } = await runTurn({ + workspace, + model, + userPrompt: '/impeccable document. Record the incumbent design system from index.html into DESIGN.md.', + maxSteps: 14, + denyBash: true, + }); + logTrace('S19', 'denied-launcher-document', modelId, trace, { finishReason, text: stepTexts.join('\n') }); + assert.notEqual(finishReason, 'length', 'a truncated response is not a completed documentation pass'); + assert.ok(trace.toolCalls.some((call) => call.name === 'bash' && call.denied && /impeccable\s+context\b/.test(call.input.command)), 'must encounter an actual denied context attempt'); + const designWriteIndex = trace.toolCalls.findIndex((call) => call.mutatedPaths.some((p) => /(?:^|\/)DESIGN\.md$/.test(p))); + assert.ok(designWriteIndex >= 0, 'must still produce DESIGN.md, not stop at the refusal'); + const documentReadIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && /(?:^|\/)reference\/document\.md$/.test(call.input.path)); + assert.ok(documentReadIndex >= 0 && documentReadIndex < designWriteIndex, 'reference/document.md must actually be read before DESIGN.md is written'); + const sourceReadIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && call.input.path.endsWith('index.html')); + assert.ok(sourceReadIndex >= 0 && sourceReadIndex < designWriteIndex, 'the incumbent source must be read before DESIGN.md is written'); + assertLauncherDenialWarningBeforeNextTool(responseMessages); + assert.ok(!trace.toolCalls.some((call) => call.mutatedPaths.some((p) => /(?:^|\/)PRODUCT\.md$/.test(p))), 'must not rewrite PRODUCT.md'); + } finally { + cleanupWorkspace(workspace); + } + }); + it('scenario 19: denied launcher keeps planning-only work read-only without craft-floor', async () => { const workspace = prepareWorkspace({ files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE,