test: scenario 19 documentation case when the context launcher is denied (#791)

* Add degraded Setup path: must-read pack when the context launcher is refused

When the host denies the impeccable context launcher (issue #789, measured
in #744), the Setup fallback now names the degraded path and its
unconditional must-read pack: the routed command's reference and
craft-floor.md before any UI edit, and document.md before writing DESIGN.md.
init.md gains the degraded Step 1 behavior, docs/CLI-CONTRACT.md documents
the degraded contract, and scenario 19 gains a denied-launcher documentation
case asserting document.md and source reads precede the DESIGN.md write.

No version bump, no changelog entry, no generated harness sync.

AI was used for assistance.
Includes AI_PR_NOTICE.txt per the repository's contribution policy: this
change was prepared without maintainer approval on issue #789, so no PR is
opened by the agent.

Co-authored-by: Matt Van Horn <mvanhorn@users.noreply.github.com>

* Drop restated degraded-setup prose; keep the scenario 19 documentation case

The launcher-unavailable path already lives on main. This removes the
notice file and the restated SKILL, init, and CLI-contract text, and keeps
the denied-launcher documentation coverage. The notice must now land before
the first tool call after the denial, not only before the eventual write.

AI was used for assistance.

Co-authored-by: Matt Van Horn <mvanhorn@users.noreply.github.com>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
Matt Van Horn
2026-09-10 13:44:25 -07:00
committed by GitHub
co-authored by Cursor Agent
parent 67d018fe05
commit 7a7579858c
4 changed files with 88 additions and 6 deletions
+19 -1
View File
@@ -4,7 +4,7 @@ import fs from 'node:fs';
import path from 'node:path';
import { MockLanguageModelV3 } from 'ai/test';
import { prepareWorkspace, cleanupWorkspace, makeTools, runTurn, fileLoaded, SKILL_BODY } from './skill-behavior/harness.mjs';
import { assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs';
import { assertLauncherDenialWarningBeforeNextTool, assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs';
import { CASE_STUDY_ANSWER } from './skill-behavior/fixtures.mjs';
import { sourceHash as hashSources } from './skill-workflow/source-hash.mjs';
import { assertCompleted, assertFreshCaptures, assertNoChangeDocumentation, assertDocumentationArtifacts } from './skill-workflow/assertions.mjs';
@@ -186,6 +186,24 @@ it('headless behavior shells disable unattended decision pages and omit provider
}
});
it('documentation fallback requires an assistant warning before the first tool call after denial', () => {
const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };
const denial = { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'context', toolName: 'bash', output: { type: 'text', value: 'Error: Bash permission denied by the host. This command was not executed.' } }] };
const warning = { role: 'assistant', content: 'Context loading did not run because the launcher was denied.' };
const read = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'read', toolName: 'read', input: { path: 'reference/document.md' } }] };
const write = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'write', toolName: 'write', input: { path: 'DESIGN.md' } }] };
assert.doesNotThrow(() => assertLauncherDenialWarningBeforeNextTool([call, denial, warning, read, write]));
assert.doesNotThrow(() => assertLauncherDenialWarningBeforeNextTool([call, denial, { role: 'assistant', content: [{ type: 'text', text: warning.content }, ...read.content] }, write]));
for (const messages of [
[call, denial, read, warning, write], // Reads first, warns only before the write.
[call, denial, read, write, warning], // Final-only disclosure.
[warning, call, denial, read], // Not a response to the actual denial.
[call, denial, warning], // Warning with no follow-up tool call.
]) {
assert.throws(() => assertLauncherDenialWarningBeforeNextTool(messages), assert.AssertionError);
}
});
it('planning fallback requires an assistant warning between the denial and context reads', () => {
const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };
const denial = { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'context', toolName: 'bash', output: { type: 'text', value: 'Error: Bash permission denied by the host. This command was not executed.' } }] };
+15 -1
View File
@@ -321,7 +321,7 @@ results remain the completed measurements.
| 16 | existing surface, with and without PRODUCT.md; asks where to start | completes relevant advice without edits, interviews, critique archives, menu scans, or explicit invented refinement prerequisites; reference coverage is diagnostic |
| 17 | existing surface; asks whether critique is required before polish | completes read-only advice distinguishing assessment from implementation and explaining critique is optional; reference coverage is diagnostic |
| 18 | existing surface; explicitly requests polish followed by a next-command recommendation | loads `polish.md` rather than substituting workflow advice for the requested work |
| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, and a denied-launcher planning-only case | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor |
| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, a denied-launcher planning-only case, and a denied-launcher documentation case (PRODUCT.md + index.html, no DESIGN.md) | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor; documentation requires successful document.md and source reads before any DESIGN.md write, with the denial disclosed before the first tool call after the denied launcher |
## Setup launcher-failure branch (2026-09-06, PR #750)
@@ -405,6 +405,20 @@ IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-sonnet-5,gpt-5.6-terra,gemini-3.7-flash,
node --test --test-name-pattern='scenario 19:' tests/skill-behavior/scenarios.test.mjs
```
### Degraded documentation case (#789)
A fourth scenario 19 case covers the documentation branch: launcher denied,
PRODUCT.md plus an incumbent `index.html`, prompt `/impeccable document`.
It requires an actual denied context attempt, successful `document.md` and
source reads before the DESIGN.md write, the denial disclosed before the
first tool call after the denied launcher, and an untouched PRODUCT.md.
Scenario 19 on this branch, all four cases, one run each on 2026-09-09:
4/4 on claude-sonnet-5, 4/4 on gpt-5.6-terra, 4/4 on gemini-3.7-flash. The
documentation case also passes with main's unchanged `SKILL.src.md` and
`init.md` plus this test file, so the coverage is the test, not a skill-text
change.
## Workflow-advice baseline (2026-09-05, PR #737)
The four cases in scenarios 16-18 are new; prior scenario results do not
+24 -3
View File
@@ -61,17 +61,25 @@ export function assertNewWorkLifecycle(trace, { target, redesign = false }) {
export const LAUNCHER_FAILURE_WARNING = /(?:context|launcher|bash)[^.!?\n]{0,160}(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|did(?:n't| not)|fail|unable)|(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|unable)[^.!?\n]{0,160}(?:context|launcher|bash)/i;
export function assertPlanningFallbackWarning(responseMessages) {
const blocks = responseMessages.flatMap((message) =>
function responseBlocks(responseMessages) {
return responseMessages.flatMap((message) =>
(typeof message.content === 'string' ? [{ type: 'text', text: message.content }] : message.content)
.map((block) => ({ ...block, role: message.role })),
);
}
function contextLauncherDenialIndex(blocks) {
const contextCalls = new Set(blocks.filter((block) => block.role === 'assistant'
&& block.type === 'tool-call' && block.toolName === 'bash'
&& /impeccable\s+context\b/.test(block.input?.command ?? '')).map((block) => block.toolCallId));
const denialIndex = blocks.findIndex((block) => block.role === 'tool'
return blocks.findIndex((block) => block.role === 'tool'
&& block.type === 'tool-result' && contextCalls.has(block.toolCallId)
&& block.output?.type === 'text' && /Bash permission denied by the host/.test(block.output.value));
}
export function assertPlanningFallbackWarning(responseMessages) {
const blocks = responseBlocks(responseMessages);
const denialIndex = contextLauncherDenialIndex(blocks);
assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence');
const warningIndex = blocks.findIndex((block, index) => index > denialIndex
&& block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text));
@@ -81,3 +89,16 @@ export function assertPlanningFallbackWarning(responseMessages) {
assert.ok(warningIndex > denialIndex && contextReadIndex > warningIndex,
'planning fallback must warn after denial and before reading project context, not only in the final response');
}
export function assertLauncherDenialWarningBeforeNextTool(responseMessages) {
const blocks = responseBlocks(responseMessages);
const denialIndex = contextLauncherDenialIndex(blocks);
assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence');
const warningIndex = blocks.findIndex((block, index) => index > denialIndex
&& block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text));
const nextToolIndex = blocks.findIndex((block, index) => index > denialIndex
&& block.role === 'assistant' && block.type === 'tool-call');
assert.ok(nextToolIndex >= 0, 'must continue with a tool call after the denied launcher');
assert.ok(warningIndex > denialIndex && nextToolIndex > warningIndex,
'must disclose the failed context launcher before the first tool call after the denial, not only before the eventual write');
}
+30 -1
View File
@@ -28,7 +28,7 @@ import {
ENGINE_MISSING_MESSAGE,
} from './harness.mjs';
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
import { assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs';
import { assertLauncherDenialWarningBeforeNextTool, assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs';
import { assertCompleted } from '../skill-workflow/assertions.mjs';
import {
PRODUCT_MD_SAMPLE,
@@ -708,6 +708,35 @@ for (const modelId of resolveModelList()) {
});
}
it('scenario 19: denied launcher requires document.md before writing DESIGN.md', async () => {
const workspace = prepareWorkspace({ files: {
'PRODUCT.md': PRODUCT_MD_SAMPLE,
'index.html': MINIMAL_LANDING_HTML,
} });
try {
const { trace, stepTexts, finishReason, responseMessages } = await runTurn({
workspace,
model,
userPrompt: '/impeccable document. Record the incumbent design system from index.html into DESIGN.md.',
maxSteps: 14,
denyBash: true,
});
logTrace('S19', 'denied-launcher-document', modelId, trace, { finishReason, text: stepTexts.join('\n') });
assert.notEqual(finishReason, 'length', 'a truncated response is not a completed documentation pass');
assert.ok(trace.toolCalls.some((call) => call.name === 'bash' && call.denied && /impeccable\s+context\b/.test(call.input.command)), 'must encounter an actual denied context attempt');
const designWriteIndex = trace.toolCalls.findIndex((call) => call.mutatedPaths.some((p) => /(?:^|\/)DESIGN\.md$/.test(p)));
assert.ok(designWriteIndex >= 0, 'must still produce DESIGN.md, not stop at the refusal');
const documentReadIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && /(?:^|\/)reference\/document\.md$/.test(call.input.path));
assert.ok(documentReadIndex >= 0 && documentReadIndex < designWriteIndex, 'reference/document.md must actually be read before DESIGN.md is written');
const sourceReadIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && call.input.path.endsWith('index.html'));
assert.ok(sourceReadIndex >= 0 && sourceReadIndex < designWriteIndex, 'the incumbent source must be read before DESIGN.md is written');
assertLauncherDenialWarningBeforeNextTool(responseMessages);
assert.ok(!trace.toolCalls.some((call) => call.mutatedPaths.some((p) => /(?:^|\/)PRODUCT\.md$/.test(p))), 'must not rewrite PRODUCT.md');
} finally {
cleanupWorkspace(workspace);
}
});
it('scenario 19: denied launcher keeps planning-only work read-only without craft-floor', async () => {
const workspace = prepareWorkspace({ files: {
'PRODUCT.md': PRODUCT_MD_SAMPLE,