mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-18 17:16:46 +03:00
Fix: Setup survives a refused launcher (#750)
Preserve Setup context and reference loading after launcher refusal, disclose the failure before editing, and limit Claude skill-directory substitution to SKILL.md. Add plugin-path and denied-launcher behavior regressions. Addresses part of #744 without closing its remaining scope. AI assistance: Cursor on the original contribution; Codex on maintainer-directed follow-up fixes and validation.
This commit is contained in:
Abdul Wahab
@@ -75,6 +75,28 @@ The trace is the source of truth, not the model's free-form reply.
|
||||
| 16 | existing surface, with and without PRODUCT.md; asks where to start | loads `routing.md`, delivers advice, and does not edit project files, start an interview, archive a critique, or run menu scans |
|
||||
| 17 | existing surface; asks whether critique is required before polish | loads `routing.md` and both command references, then delivers advice without executing the playbooks |
|
||||
| 18 | existing surface; explicitly requests polish followed by a next-command recommendation | loads `polish.md` rather than substituting workflow advice for the requested work |
|
||||
| 19 | tiny spacing edit with PRODUCT.md + DESIGN.md; Bash denied, plus a real-loader success control | actually reads playbook and craft floor before editing; denial also requires direct context-file reads and a user-visible warning before the edit |
|
||||
|
||||
## Setup launcher-failure branch (2026-09-06, PR #750)
|
||||
|
||||
Scenario 19 injects a host permission error before any shell command executes,
|
||||
including retries and compound commands. File tools remain available; the
|
||||
staged skill is read-only. Assertions require successful reads, an actual UI
|
||||
edit, unchanged context files, and disclosure before the first write in the
|
||||
assistant message sequence. Failed read attempts and shell commands merely
|
||||
mentioning a reference do not count as loading it. The success control allows
|
||||
only the real context-loader command through Bash and requires exit 0.
|
||||
|
||||
The suite measures continuation after a tool refusal, not Claude's skill
|
||||
activation or plugin substitution. Those are separate loader/path checks.
|
||||
It also does not establish behavior for every missing-binary or runtime error.
|
||||
|
||||
Focused baseline (2026-09-06): both scenario 19 cases passed on
|
||||
`claude-sonnet-5`, one run each. The original wording also continued after
|
||||
denial in the comparison run, but disclosed the failure only in its final
|
||||
summary. This does not reproduce the reporter's complete reference-loading
|
||||
failure or establish a multi-provider pass. An earlier scenario 6 result used
|
||||
attempt-based reference assertions and is not counted as a success control.
|
||||
|
||||
## Workflow-advice baseline (2026-09-05, PR #737)
|
||||
|
||||
|
||||
@@ -219,7 +219,7 @@ function defaultSimulatedAnswer(question) {
|
||||
return 'Use the brief, preserve real operational content, and make the primary decision obvious.';
|
||||
}
|
||||
|
||||
export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contextOnlyBash = false } = {}) {
|
||||
export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contextOnlyBash = false, denyBash = false } = {}) {
|
||||
const trace = {
|
||||
toolCalls: [],
|
||||
bashCommands: [],
|
||||
@@ -250,6 +250,14 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
|
||||
}),
|
||||
execute: async ({ command }) => {
|
||||
const call = record('bash', { command });
|
||||
// Simulate a host refusal, not a process failure. Nothing reaches a
|
||||
// shell, including retries, alternate launchers, and compound commands.
|
||||
if (denyBash) {
|
||||
call.denied = true;
|
||||
const out = 'Error: Bash permission denied by the host. This command was not executed.';
|
||||
trace.bashOutputs.push(out);
|
||||
return out;
|
||||
}
|
||||
// Routing tests need the real context loader, not a general-purpose
|
||||
// shell on the host. Reject before execution (still record attempts).
|
||||
if (contextOnlyBash && command.trim() !== '.claude/skills/impeccable/scripts/impeccable context') {
|
||||
@@ -273,13 +281,16 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
|
||||
path: z.string().describe('Workspace-relative file path.'),
|
||||
}),
|
||||
execute: async ({ path: p }) => {
|
||||
record('read', { path: p });
|
||||
const call = record('read', { path: p });
|
||||
call.succeeded = false;
|
||||
const resolved = safeResolve(workspace, p);
|
||||
if (typeof resolved !== 'string') return `Error: ${resolved.error}`;
|
||||
if (!fs.existsSync(resolved)) return `File not found: ${p}`;
|
||||
const stat = fs.statSync(resolved);
|
||||
if (stat.isDirectory()) return `Path is a directory: ${p}. Use list instead.`;
|
||||
return fs.readFileSync(resolved, 'utf8');
|
||||
const contents = fs.readFileSync(resolved, 'utf8');
|
||||
call.succeeded = true;
|
||||
return contents;
|
||||
},
|
||||
}),
|
||||
write: tool({
|
||||
@@ -292,7 +303,7 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
|
||||
const call = record('write', { path: p, contents });
|
||||
const resolved = safeResolve(workspace, p);
|
||||
if (typeof resolved !== 'string') return `Error: ${resolved.error}`;
|
||||
if (contextOnlyBash && path.relative(workspace, resolved).split(path.sep)[0] === '.claude') {
|
||||
if ((contextOnlyBash || denyBash) && path.relative(workspace, resolved).split(path.sep)[0] === '.claude') {
|
||||
return 'Error: the staged skill is read-only; edits must target project files.';
|
||||
}
|
||||
fs.mkdirSync(path.dirname(resolved), { recursive: true });
|
||||
@@ -370,8 +381,8 @@ export function makeTools(workspace, extraEnv = {}, simulatedUser = {}, { contex
|
||||
// run is never killed. The timer is unref'd (it must not keep the loop alive
|
||||
// after a healthy turn) and cleared on completion.
|
||||
const TURN_TIMEOUT_MS = Number(process.env.IMPECCABLE_SKILL_BEHAVIOR_TURN_TIMEOUT_MS) || 840_000;
|
||||
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8, env = {}, simulatedUser = {}, timeoutMs = TURN_TIMEOUT_MS, contextOnlyBash = false }) {
|
||||
const { tools, trace } = makeTools(workspace, env, simulatedUser, { contextOnlyBash });
|
||||
export async function runTurn({ workspace, model, userPrompt, priorMessages = [], maxSteps = 8, env = {}, simulatedUser = {}, timeoutMs = TURN_TIMEOUT_MS, contextOnlyBash = false, denyBash = false }) {
|
||||
const { tools, trace } = makeTools(workspace, env, simulatedUser, { contextOnlyBash, denyBash });
|
||||
const messages = [
|
||||
...priorMessages,
|
||||
{ role: 'user', content: userPrompt },
|
||||
@@ -409,6 +420,7 @@ export async function runTurn({ workspace, model, userPrompt, priorMessages = []
|
||||
return {
|
||||
trace,
|
||||
text: result.text ?? '',
|
||||
stepTexts: result.steps.map((step) => step.text ?? ''),
|
||||
finishReason: result.finishReason,
|
||||
usage: result.usage,
|
||||
responseMessages,
|
||||
|
||||
@@ -652,6 +652,49 @@ for (const modelId of resolveModelList()) {
|
||||
}
|
||||
});
|
||||
|
||||
for (const denyBash of [true, false]) {
|
||||
it(`scenario 19: ${denyBash ? 'denied launcher' : 'successful launcher control'} loads context and references before editing`, async () => {
|
||||
const workspace = prepareWorkspace({ files: {
|
||||
'PRODUCT.md': PRODUCT_MD_SAMPLE,
|
||||
'DESIGN.md': DESIGN_MD_SAMPLE,
|
||||
'index.html': '<!doctype html><html lang="en"><head><title>Fieldnotes</title><style>body{font:16px system-ui;margin:32px}button{padding:2px 4px}</style></head><body><main><h1>Fieldnotes</h1><p>A calmer place for your notes.</p><button>New note</button></main></body></html>',
|
||||
} });
|
||||
try {
|
||||
const { trace, stepTexts, finishReason, responseMessages } = await runTurn({
|
||||
workspace,
|
||||
model,
|
||||
userPrompt: '/impeccable polish index.html. Keep this pass small: improve the button spacing only, preserving the page content and structure.',
|
||||
maxSteps: 12,
|
||||
denyBash,
|
||||
contextOnlyBash: !denyBash,
|
||||
});
|
||||
const allText = stepTexts.join('\n');
|
||||
logTrace('S19', denyBash ? 'denied-launcher' : 'successful-launcher', modelId, trace, { finishReason, text: allText });
|
||||
assert.notEqual(finishReason, 'length', 'a truncated response is not a completed fallback');
|
||||
if (denyBash) {
|
||||
assert.ok(trace.toolCalls.some((call) => call.name === 'bash' && call.denied && /impeccable\s+context\b/.test(call.input.command)), 'must encounter an actual denied context attempt');
|
||||
} else {
|
||||
assert.ok(trace.bashOutputs.some((out) => out.startsWith('exit=0\n')), 'the control must execute the real context loader successfully');
|
||||
}
|
||||
const writeIndex = trace.toolCalls.findIndex((call) => call.mutatedPaths.includes('index.html'));
|
||||
assert.ok(writeIndex >= 0, 'must continue to the requested edit, not just load references');
|
||||
for (const filename of [...(denyBash ? ['PRODUCT.md', 'DESIGN.md'] : []), 'reference/polish.md', 'reference/craft-floor.md']) {
|
||||
const readIndex = trace.toolCalls.findIndex((call) => call.name === 'read' && call.succeeded && (call.input.path === filename || call.input.path.endsWith(`/${filename}`)));
|
||||
assert.ok(readIndex >= 0 && readIndex < writeIndex, `${filename} must actually be read before editing`);
|
||||
}
|
||||
const warning = /(?:context|launcher|bash)[^.!?\n]{0,160}(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|did(?:n't| not)|fail|unable)|(?:denied|refused|unavailable|blocked|could(?:n't| not)|cannot|can't|unable)[^.!?\n]{0,160}(?:context|launcher|bash)/i;
|
||||
const assistantBlocks = responseMessages.filter((message) => message.role === 'assistant')
|
||||
.flatMap((message) => typeof message.content === 'string' ? [{ type: 'text', text: message.content }] : message.content);
|
||||
const warningIndex = assistantBlocks.findIndex((block) => block.type === 'text' && warning.test(block.text));
|
||||
const writeBlockIndex = assistantBlocks.findIndex((block) => block.type === 'tool-call' && block.toolName === 'write');
|
||||
if (denyBash) assert.ok(warningIndex >= 0 && writeBlockIndex > warningIndex, 'must disclose the failed context launcher before editing, not only in the final summary');
|
||||
assert.ok(!trace.toolCalls.some((call) => call.mutatedPaths.some((p) => /(?:^|\/)(?:PRODUCT|DESIGN)\.md$/.test(p))), 'must not fabricate or replace project context');
|
||||
} finally {
|
||||
cleanupWorkspace(workspace);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
it('scenario 18: explicit command request takes precedence over workflow advice', async () => {
|
||||
const workspace = prepareWorkspace({ files: WORKFLOW_ADVICE_FILES });
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user