mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-11 21:57:14 +03:00
7a7579858c
* Add degraded Setup path: must-read pack when the context launcher is refused When the host denies the impeccable context launcher (issue #789, measured in #744), the Setup fallback now names the degraded path and its unconditional must-read pack: the routed command's reference and craft-floor.md before any UI edit, and document.md before writing DESIGN.md. init.md gains the degraded Step 1 behavior, docs/CLI-CONTRACT.md documents the degraded contract, and scenario 19 gains a denied-launcher documentation case asserting document.md and source reads precede the DESIGN.md write. No version bump, no changelog entry, no generated harness sync. AI was used for assistance. Includes AI_PR_NOTICE.txt per the repository's contribution policy: this change was prepared without maintainer approval on issue #789, so no PR is opened by the agent. Co-authored-by: Matt Van Horn <mvanhorn@users.noreply.github.com> * Drop restated degraded-setup prose; keep the scenario 19 documentation case The launcher-unavailable path already lives on main. This removes the notice file and the restated SKILL, init, and CLI-contract text, and keeps the denied-launcher documentation coverage. The notice must now land before the first tool call after the denial, not only before the eventual write. AI was used for assistance. Co-authored-by: Matt Van Horn <mvanhorn@users.noreply.github.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com>
105 lines
7.1 KiB
JavaScript
105 lines
7.1 KiB
JavaScript
import assert from 'node:assert/strict';
|
||
|
||
// These are bounded English-fixture checks, not a general semantic grader.
|
||
// Reference coverage is reported separately: opening a file proves neither
|
||
// useful advice nor permission to execute it.
|
||
export function missingReferences(trace, filenames) {
|
||
return filenames.filter((filename) => !trace.toolCalls.some((call) =>
|
||
(call.loadedFiles || []).some((file) => file === filename || file.endsWith(`/${filename}`))));
|
||
}
|
||
|
||
export function assertAdviceOnly(trace, text) {
|
||
assert.ok(text.trim(), 'advice must reach the user, not stop at reference loading');
|
||
assert.deepEqual(trace.writePaths, [], 'advice must not use the write tool');
|
||
const mutations = trace.toolCalls.flatMap((call) => call.mutatedPaths ?? [])
|
||
.filter((file) => !file.startsWith('.impeccable/') || file.startsWith('.impeccable/critique/'));
|
||
assert.deepEqual(mutations, [], 'advice must not edit project files or archive an unsolicited critique');
|
||
assert.deepEqual(trace.questionCalls, [], 'advice must not start an init or design interview');
|
||
assert.ok(!trace.bashCommands.some((command) => command.includes('impeccable detect')), 'workflow advice does not run menu scans');
|
||
}
|
||
|
||
function normalizedAdvice(text) {
|
||
return text.replace(/[`*_]/g, '').replace(/[’]/g, "'").toLowerCase();
|
||
}
|
||
|
||
export function assertWorkflowAdvice(trace, text, { missingContext = false } = {}) {
|
||
assertAdviceOnly(trace, text);
|
||
const advice = normalizedAdvice(text);
|
||
assert.match(advice, /index\.html/, 'advice should address the existing surface');
|
||
assert.match(advice, missingContext ? /\binit\b/ : /\b(?:critique|audit|polish)\b/, 'advice must recommend a relevant starting point');
|
||
if (missingContext) assert.match(advice, /\bdocument\b/, 'advice should explain how to record the existing identity');
|
||
assert.doesNotMatch(advice, /(?:must|need to|have to|required to)\s+(?:run\s+)?(?:\/impeccable\s+)?(?:init|document)\b[^.!?\n]{0,100}\bbefore\s+(?:you\s+can\s+)?(?:run(?:ning)?\s+)?(?:polish(?:ing)?|refin(?:e|ing|ement))\b|(?:polish|refinement)\s+(?:requires|is blocked by|cannot run without)\s+(?:init|document|product\.md|design\.md)/,
|
||
'setup is not a mandatory prerequisite for narrow refinement');
|
||
}
|
||
|
||
export function assertCommandComparison(trace, text) {
|
||
assertAdviceOnly(trace, text);
|
||
const advice = normalizedAdvice(text);
|
||
assert.match(advice, /critique[^.!?\n]{0,120}(?:review|assess|evaluat|report|findings)/, 'comparison must explain critique as assessment');
|
||
assert.match(advice, /polish[^.!?\n]{0,120}(?:fix|refin|implement|edit)/, 'comparison must explain polish as implementation');
|
||
assert.match(advice, /critique\s+(?:is\s+)?(?:isn't|is not|not)\s+(?:required|necessary)|critique[^.!?\n]{0,50}\boptional\b|polish[^.!?\n]{0,100}(?:directly|without\s+(?:a\s+)?critique|independent)/,
|
||
'comparison must explain that critique is optional before polish');
|
||
assert.doesNotMatch(advice, /(?:must|need to|have to)\s+(?:run\s+)?critique[^.!?\n]{0,80}before\s+(?:run(?:ning)?\s+)?polish|critique\s+(?:is\s+)?(?:required|mandatory|necessary)\s+before\s+polish|polish\s+(?:requires|cannot run without)\s+(?:a\s+)?critique/,
|
||
'comparison must not invent a critique prerequisite');
|
||
}
|
||
|
||
export function assertNewWorkLifecycle(trace, { target, redesign = false }) {
|
||
const calls = trace.toolCalls;
|
||
const writes = (call, file) => (call.mutatedPaths || []).includes(file);
|
||
const implementation = calls.findIndex((call) => writes(call, target));
|
||
const question = calls.findIndex((call) => call.name === 'ask_user_question');
|
||
const brief = calls.findIndex((call) => (call.mutatedPaths || []).some((file) => file.startsWith('.impeccable/surfaces/')));
|
||
assert.ok(implementation >= 0, `new-work did not produce the requested artifact: ${target}`);
|
||
assert.ok(question >= 0 && question < implementation, 'implementation must follow a user answer');
|
||
assert.ok(brief >= 0 && brief < implementation, 'the direction contract must be recorded in a surface brief before implementation');
|
||
if (redesign) {
|
||
const lastImplementation = calls.findLastIndex((call) => writes(call, target));
|
||
const documentation = calls.findLastIndex((call) => writes(call, 'DESIGN.md'));
|
||
assert.ok(documentation > lastImplementation, 'redesign must record DESIGN.md from the finished build, after the last page edit');
|
||
}
|
||
}
|
||
|
||
export const LAUNCHER_FAILURE_WARNING = /(?:context|launcher|bash)[^.!?\n]{0,160}(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|did(?:n't| not)|fail|unable)|(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|unable)[^.!?\n]{0,160}(?:context|launcher|bash)/i;
|
||
|
||
function responseBlocks(responseMessages) {
|
||
return responseMessages.flatMap((message) =>
|
||
(typeof message.content === 'string' ? [{ type: 'text', text: message.content }] : message.content)
|
||
.map((block) => ({ ...block, role: message.role })),
|
||
);
|
||
}
|
||
|
||
function contextLauncherDenialIndex(blocks) {
|
||
const contextCalls = new Set(blocks.filter((block) => block.role === 'assistant'
|
||
&& block.type === 'tool-call' && block.toolName === 'bash'
|
||
&& /impeccable\s+context\b/.test(block.input?.command ?? '')).map((block) => block.toolCallId));
|
||
return blocks.findIndex((block) => block.role === 'tool'
|
||
&& block.type === 'tool-result' && contextCalls.has(block.toolCallId)
|
||
&& block.output?.type === 'text' && /Bash permission denied by the host/.test(block.output.value));
|
||
}
|
||
|
||
export function assertPlanningFallbackWarning(responseMessages) {
|
||
const blocks = responseBlocks(responseMessages);
|
||
const denialIndex = contextLauncherDenialIndex(blocks);
|
||
assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence');
|
||
const warningIndex = blocks.findIndex((block, index) => index > denialIndex
|
||
&& block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text));
|
||
const contextReadIndex = blocks.findIndex((block, index) => index > denialIndex
|
||
&& block.role === 'assistant' && block.type === 'tool-call' && block.toolName === 'read'
|
||
&& /(?:^|\/)(?:PRODUCT|DESIGN)\.md$/.test(block.input?.path ?? ''));
|
||
assert.ok(warningIndex > denialIndex && contextReadIndex > warningIndex,
|
||
'planning fallback must warn after denial and before reading project context, not only in the final response');
|
||
}
|
||
|
||
export function assertLauncherDenialWarningBeforeNextTool(responseMessages) {
|
||
const blocks = responseBlocks(responseMessages);
|
||
const denialIndex = contextLauncherDenialIndex(blocks);
|
||
assert.ok(denialIndex >= 0, 'must observe the context launcher denial in the response sequence');
|
||
const warningIndex = blocks.findIndex((block, index) => index > denialIndex
|
||
&& block.role === 'assistant' && block.type === 'text' && LAUNCHER_FAILURE_WARNING.test(block.text));
|
||
const nextToolIndex = blocks.findIndex((block, index) => index > denialIndex
|
||
&& block.role === 'assistant' && block.type === 'tool-call');
|
||
assert.ok(nextToolIndex >= 0, 'must continue with a tool call after the denied launcher');
|
||
assert.ok(warningIndex > denialIndex && nextToolIndex > warningIndex,
|
||
'must disclose the failed context launcher before the first tool call after the denial, not only before the eventual write');
|
||
}
|