mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-11 13:46:32 +03:00
test: scenario 19 documentation case when the context launcher is denied (#791)
* Add degraded Setup path: must-read pack when the context launcher is refused When the host denies the impeccable context launcher (issue #789, measured in #744), the Setup fallback now names the degraded path and its unconditional must-read pack: the routed command's reference and craft-floor.md before any UI edit, and document.md before writing DESIGN.md. init.md gains the degraded Step 1 behavior, docs/CLI-CONTRACT.md documents the degraded contract, and scenario 19 gains a denied-launcher documentation case asserting document.md and source reads precede the DESIGN.md write. No version bump, no changelog entry, no generated harness sync. AI was used for assistance. Includes AI_PR_NOTICE.txt per the repository's contribution policy: this change was prepared without maintainer approval on issue #789, so no PR is opened by the agent. Co-authored-by: Matt Van Horn <mvanhorn@users.noreply.github.com> * Drop restated degraded-setup prose; keep the scenario 19 documentation case The launcher-unavailable path already lives on main. This removes the notice file and the restated SKILL, init, and CLI-contract text, and keeps the denied-launcher documentation coverage. The notice must now land before the first tool call after the denial, not only before the eventual write. AI was used for assistance. Co-authored-by: Matt Van Horn <mvanhorn@users.noreply.github.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor Agent
parent
67d018fe05
commit
7a7579858c
@@ -4,7 +4,7 @@ import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { MockLanguageModelV3 } from 'ai/test';
|
||||
import { prepareWorkspace, cleanupWorkspace, makeTools, runTurn, fileLoaded, SKILL_BODY } from './skill-behavior/harness.mjs';
|
||||
import { assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs';
|
||||
import { assertLauncherDenialWarningBeforeNextTool, assertPlanningFallbackWarning, assertNewWorkLifecycle, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './skill-behavior/assertions.mjs';
|
||||
import { CASE_STUDY_ANSWER } from './skill-behavior/fixtures.mjs';
|
||||
import { sourceHash as hashSources } from './skill-workflow/source-hash.mjs';
|
||||
import { assertCompleted, assertFreshCaptures, assertNoChangeDocumentation, assertDocumentationArtifacts } from './skill-workflow/assertions.mjs';
|
||||
@@ -186,6 +186,24 @@ it('headless behavior shells disable unattended decision pages and omit provider
|
||||
}
|
||||
});
|
||||
|
||||
it('documentation fallback requires an assistant warning before the first tool call after denial', () => {
|
||||
const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };
|
||||
const denial = { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'context', toolName: 'bash', output: { type: 'text', value: 'Error: Bash permission denied by the host. This command was not executed.' } }] };
|
||||
const warning = { role: 'assistant', content: 'Context loading did not run because the launcher was denied.' };
|
||||
const read = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'read', toolName: 'read', input: { path: 'reference/document.md' } }] };
|
||||
const write = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'write', toolName: 'write', input: { path: 'DESIGN.md' } }] };
|
||||
assert.doesNotThrow(() => assertLauncherDenialWarningBeforeNextTool([call, denial, warning, read, write]));
|
||||
assert.doesNotThrow(() => assertLauncherDenialWarningBeforeNextTool([call, denial, { role: 'assistant', content: [{ type: 'text', text: warning.content }, ...read.content] }, write]));
|
||||
for (const messages of [
|
||||
[call, denial, read, warning, write], // Reads first, warns only before the write.
|
||||
[call, denial, read, write, warning], // Final-only disclosure.
|
||||
[warning, call, denial, read], // Not a response to the actual denial.
|
||||
[call, denial, warning], // Warning with no follow-up tool call.
|
||||
]) {
|
||||
assert.throws(() => assertLauncherDenialWarningBeforeNextTool(messages), assert.AssertionError);
|
||||
}
|
||||
});
|
||||
|
||||
it('planning fallback requires an assistant warning between the denial and context reads', () => {
|
||||
const call = { role: 'assistant', content: [{ type: 'tool-call', toolCallId: 'context', toolName: 'bash', input: { command: '.claude/skills/impeccable/scripts/impeccable context' } }] };
|
||||
const denial = { role: 'tool', content: [{ type: 'tool-result', toolCallId: 'context', toolName: 'bash', output: { type: 'text', value: 'Error: Bash permission denied by the host. This command was not executed.' } }] };
|
||||
|
||||
@@ -321,7 +321,7 @@ results remain the completed measurements.
|
||||
| 16 | existing surface, with and without PRODUCT.md; asks where to start | completes relevant advice without edits, interviews, critique archives, menu scans, or explicit invented refinement prerequisites; reference coverage is diagnostic |
|
||||
| 17 | existing surface; asks whether critique is required before polish | completes read-only advice distinguishing assessment from implementation and explaining critique is optional; reference coverage is diagnostic |
|
||||
| 18 | existing surface; explicitly requests polish followed by a next-command recommendation | loads `polish.md` rather than substituting workflow advice for the requested work |
|
||||
| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, and a denied-launcher planning-only case | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor |
|
||||
| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, a real-loader success control, a denied-launcher planning-only case, and a denied-launcher documentation case (PRODUCT.md + index.html, no DESIGN.md) | edits require successful playbook/craft-floor reads and a pre-edit denial warning; planning stays read-only and skips craft-floor; documentation requires successful document.md and source reads before any DESIGN.md write, with the denial disclosed before the first tool call after the denied launcher |
|
||||
|
||||
## Setup launcher-failure branch (2026-09-06, PR #750)
|
||||
|
||||
@@ -405,6 +405,20 @@ IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-sonnet-5,gpt-5.6-terra,gemini-3.7-flash,
|
||||
node --test --test-name-pattern='scenario 19:' tests/skill-behavior/scenarios.test.mjs
|
||||
```
|
||||
|
||||
### Degraded documentation case (#789)
|
||||
|
||||
A fourth scenario 19 case covers the documentation branch: launcher denied,
|
||||
PRODUCT.md plus an incumbent `index.html`, prompt `/impeccable document`.
|
||||
It requires an actual denied context attempt, successful `document.md` and
|
||||
source reads before the DESIGN.md write, the denial disclosed before the
|
||||
first tool call after the denied launcher, and an untouched PRODUCT.md.
|
||||
|
||||
Scenario 19 on this branch, all four cases, one run each on 2026-09-09:
|
||||
4/4 on claude-sonnet-5, 4/4 on gpt-5.6-terra, 4/4 on gemini-3.7-flash. The
|
||||
documentation case also passes with main's unchanged `SKILL.src.md` and
|
||||
`init.md` plus this test file, so the coverage is the test, not a skill-text
|
||||
change.
|
||||
|
||||
## Workflow-advice baseline (2026-09-05, PR #737)
|
||||
|
||||
The four cases in scenarios 16-18 are new; prior scenario results do not
|
||||
|
||||
@@ -61,17 +61,25 @@ export function assertNewWorkLifecycle(trace, { target, redesign = false }) {
|
||||
|
||||
export const LAUNCHER_FAILURE_WARNING = /(?:context|launcher|bash)[^.!?\n]{0,160}(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|did(?:n't| not)|fail|unable)|(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|unable)[^.!?\n]{0,160}(?:context|launcher|bash)/i;
|
||||
|
||||
export function assertPlanningFallbackWarning(responseMessages) {
|
||||
const blocks = responseMessages.flatMap((message) =>
|
||||
function responseBlocks(responseMessages) {
|
||||
return responseMessages.flatMap((message) =>
|
||||
(typeof message.content === 'string' ? [{ type: 'text', text: message.content }] : message.content)
|
||||
.map((block) => ({ ...block, role: message.role })),
|
||||
);
|
||||
}
|
||||
|
||||
function contextLauncherDenialIndex(blocks) {
|
||||
const contextCalls = new Set(blocks.filter((block) => block.role === 'assistant'
|
||||
&& block.type === 'tool-call' && block.toolName === 'bash'
|
||||
&& /impeccable\s+context\b/.test(block.input?.command ?? '')).map((block) => block.toolCallId));
|
||||
const denialIndex = blocks.findIndex((block) => block.role === 'tool'
|
||||
return blocks.findIndex((block) => block.role === 'tool'
|
||||
&& block.type === 'tool-result' && contextCalls.has(block.toolCallId)
|
||||
&& block.output?.type === 'text' && /Bash permission denied by the host/.test(block.output.value));
|
||||
}
|
||||
|
||||
export function assertPlanningFallbackWarning(responseMessages) {
|
||||
const blocks = responseBlocks(responseMessages);
|
||||
const denialIndex = contextLauncherDenialIndex(blocks);
|
||||
assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence');
|
||||
const warningIndex = blocks.findIndex((block, index) => index > denialIndex
|
||||
&& block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text));
|
||||
@@ -81,3 +89,16 @@ export function assertPlanningFallbackWarning(responseMessages) {
|
||||
assert.ok(warningIndex > denialIndex && contextReadIndex > warningIndex,
|
||||
'planning fallback must warn after denial and before reading project context, not only in the final response');
|
||||
}
|
||||
|
||||
export function assertLauncherDenialWarningBeforeNextTool(responseMessages) {
|
||||
const blocks = responseBlocks(responseMessages);
|
||||
const denialIndex = contextLauncherDenialIndex(blocks);
|
||||
assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence');
|
||||
const warningIndex = blocks.findIndex((block, index) => index > denialIndex
|
||||
&& block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text));
|
||||
const nextToolIndex = blocks.findIndex((block, index) => index > denialIndex
|
||||
&& block.role === 'assistant' && block.type === 'tool-call');
|
||||
assert.ok(nextToolIndex >= 0, 'must continue with a tool call after the denied launcher');
|
||||
assert.ok(warningIndex > denialIndex && nextToolIndex > warningIndex,
|
||||
'must disclose the failed context launcher before the first tool call after the denial, not only before the eventual write');
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
ENGINE_MISSING_MESSAGE,
|
||||
} from './harness.mjs';
|
||||
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
|
||||
import { assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs';
|
||||
import { assertLauncherDenialWarningBeforeNextTool, assertPlanningFallbackWarning, LAUNCHER_FAILURE_WARNING, assertAdviceOnly, assertWorkflowAdvice, assertCommandComparison, missingReferences } from './assertions.mjs';
|
||||
import { assertCompleted } from '../skill-workflow/assertions.mjs';
|
||||
import {
|
||||
PRODUCT_MD_SAMPLE,
|
||||
@@ -708,6 +708,35 @@ for (const modelId of resolveModelList()) {
|
||||
});
|
||||
}
|
||||
|
||||
it('scenario 19: denied launcher requires document.md before writing DESIGN.md', async () => {
|
||||
const workspace = prepareWorkspace({ files: {
|
||||
'PRODUCT.md': PRODUCT_MD_SAMPLE,
|
||||
'index.html': MINIMAL_LANDING_HTML,
|
||||
} });
|
||||
try {
|
||||
const { trace, stepTexts, finishReason, responseMessages } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable document. Record the incumbent design system from index.html into DESIGN.md.',
|
||||
maxSteps: 14,
|
||||
denyBash: true,
|
||||
});
|
||||
logTrace('S19', 'denied-launcher-document', modelId, trace, { finishReason, text: stepTexts.join('\n') });
|
||||
assert.notEqual(finishReason, 'length', 'a truncated response is not a completed documentation pass');
|
||||
assert.ok(trace.toolCalls.some((call) => call.name === 'bash' && call.denied && /impeccable\s+context\b/.test(call.input.command)), 'must encounter an actual denied context attempt');
|
||||
const designWriteIndex = trace.toolCalls.findIndex((call) => call.mutatedPaths.some((p) => /(?:^|\/)DESIGN\.md$/.test(p)));
|
||||
assert.ok(designWriteIndex >= 0, 'must still produce DESIGN.md, not stop at the refusal');
|
||||
const documentReadIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && /(?:^|\/)reference\/document\.md$/.test(call.input.path));
|
||||
assert.ok(documentReadIndex >= 0 && documentReadIndex < designWriteIndex, 'reference/document.md must actually be read before DESIGN.md is written');
|
||||
const sourceReadIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && call.input.path.endsWith('index.html'));
|
||||
assert.ok(sourceReadIndex >= 0 && sourceReadIndex < designWriteIndex, 'the incumbent source must be read before DESIGN.md is written');
|
||||
assertLauncherDenialWarningBeforeNextTool(responseMessages);
|
||||
assert.ok(!trace.toolCalls.some((call) => call.mutatedPaths.some((p) => /(?:^|\/)PRODUCT\.md$/.test(p))), 'must not rewrite PRODUCT.md');
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
|
||||
it('scenario 19: denied launcher keeps planning-only work read-only without craft-floor', async () => {
|
||||
const workspace = prepareWorkspace({ files: {
|
||||
'PRODUCT.md': PRODUCT_MD_SAMPLE,
|
||||
|
||||
Reference in New Issue
Block a user