mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-17 08:36:25 +03:00
Test advice and documentation outcomes rather than reference coverage
Keep completion, consent, documentation evidence, and new-world artifact requirements strict. Record reference-only omissions as diagnostics in the scoped advice and no-change fixtures. AI assistance: Codex, under maintainer direction.
This commit is contained in:
@@ -1,5 +1,48 @@
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
// These are bounded English-fixture checks, not a general semantic grader.
|
||||
// Reference coverage is reported separately: opening a file proves neither
|
||||
// useful advice nor permission to execute it.
|
||||
export function missingReferences(trace, filenames) {
|
||||
return filenames.filter((filename) => !trace.toolCalls.some((call) =>
|
||||
(call.loadedFiles || []).some((file) => file === filename || file.endsWith(`/${filename}`))));
|
||||
}
|
||||
|
||||
export function assertAdviceOnly(trace, text) {
|
||||
assert.ok(text.trim(), 'advice must reach the user, not stop at reference loading');
|
||||
assert.deepEqual(trace.writePaths, [], 'advice must not use the write tool');
|
||||
const mutations = trace.toolCalls.flatMap((call) => call.mutatedPaths ?? [])
|
||||
.filter((file) => !file.startsWith('.impeccable/') || file.startsWith('.impeccable/critique/'));
|
||||
assert.deepEqual(mutations, [], 'advice must not edit project files or archive an unsolicited critique');
|
||||
assert.deepEqual(trace.questionCalls, [], 'advice must not start an init or design interview');
|
||||
assert.ok(!trace.bashCommands.some((command) => command.includes('impeccable detect')), 'workflow advice does not run menu scans');
|
||||
}
|
||||
|
||||
function normalizedAdvice(text) {
|
||||
return text.replace(/[`*_]/g, '').replace(/[’]/g, "'").toLowerCase();
|
||||
}
|
||||
|
||||
export function assertWorkflowAdvice(trace, text, { missingContext = false } = {}) {
|
||||
assertAdviceOnly(trace, text);
|
||||
const advice = normalizedAdvice(text);
|
||||
assert.match(advice, /index\.html/, 'advice should address the existing surface');
|
||||
assert.match(advice, missingContext ? /\binit\b/ : /\b(?:critique|audit|polish)\b/, 'advice must recommend a relevant starting point');
|
||||
if (missingContext) assert.match(advice, /\bdocument\b/, 'advice should explain how to record the existing identity');
|
||||
assert.doesNotMatch(advice, /(?:must|need to|have to|required to)\s+(?:run\s+)?(?:\/impeccable\s+)?(?:init|document)\b[^.!?\n]{0,100}\bbefore\s+(?:you\s+can\s+)?(?:run(?:ning)?\s+)?(?:polish(?:ing)?|refin(?:e|ing|ement))\b|(?:polish|refinement)\s+(?:requires|is blocked by|cannot run without)\s+(?:init|document|product\.md|design\.md)/,
|
||||
'setup is not a mandatory prerequisite for narrow refinement');
|
||||
}
|
||||
|
||||
export function assertCommandComparison(trace, text) {
|
||||
assertAdviceOnly(trace, text);
|
||||
const advice = normalizedAdvice(text);
|
||||
assert.match(advice, /critique[^.!?\n]{0,120}(?:review|assess|evaluat|report|findings)/, 'comparison must explain critique as assessment');
|
||||
assert.match(advice, /polish[^.!?\n]{0,120}(?:fix|refin|implement|edit)/, 'comparison must explain polish as implementation');
|
||||
assert.match(advice, /critique\s+(?:is\s+)?(?:isn't|is not|not)\s+(?:required|necessary)|critique[^.!?\n]{0,50}\boptional\b|polish[^.!?\n]{0,100}(?:directly|without\s+(?:a\s+)?critique|independent)/,
|
||||
'comparison must explain that critique is optional before polish');
|
||||
assert.doesNotMatch(advice, /(?:must|need to|have to)\s+(?:run\s+)?critique[^.!?\n]{0,80}before\s+(?:run(?:ning)?\s+)?polish|critique\s+(?:is\s+)?(?:required|mandatory|necessary)\s+before\s+polish|polish\s+(?:requires|cannot run without)\s+(?:a\s+)?critique/,
|
||||
'comparison must not invent a critique prerequisite');
|
||||
}
|
||||
|
||||
export function assertNewWorkLifecycle(trace, { target, redesign = false }) {
|
||||
const calls = trace.toolCalls;
|
||||
const writes = (call, file) => (call.mutatedPaths || []).includes(file);
|
||||
|
||||
Reference in New Issue
Block a user