Compare commits

...
Author SHA1 Message Date
Paul Bakaus cf8f295dc3 Simplify provider hook smoke orchestration
Centralize provider fixture, hook, and agent-launch contracts while preserving provider-specific verification behavior. Reuse the shared CLI argument parser and characterize the public usage contract.

AI-assisted: prepared by Codex under pbakaus's scheduled architecture-refactor authorization.
2026-08-17 08:45:12 -07:00
2 changed files with 155 additions and 151 deletions
+107 -151
View File
@@ -12,6 +12,8 @@ import { homedir, tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseArgs } from './lib/cli-args.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const prRoot = resolve(__dirname, '..');
const defaultBundle = join(prRoot, 'dist', 'universal.zip');
@@ -32,31 +34,60 @@ if (args.help || args.h || !args.repo) {
process.exit(1);
}
const targetRepo = resolve(args.repo);
const bundlePath = resolve(args.bundle || defaultBundle);
const selectedProviders = (args.providers || defaultProviders.join(','))
const targetRepo = resolve(legacySmokeArg(args.repo));
const bundlePath = resolve(legacySmokeArg(args.bundle) || defaultBundle);
const selectedProviders = (legacySmokeArg(args.providers) || defaultProviders.join(','))
.split(',')
.map((provider) => provider.trim().toLowerCase())
.filter(Boolean);
const smokeDir = join(targetRepo, '.impeccable', 'provider-smoke');
const summaryPath = join(smokeDir, 'summary.json');
const smokeFiles = {
direct: 'src/__impeccable_provider_smoke_direct.html',
claude: 'src/__impeccable_provider_smoke_claude.html',
codex: 'src/__impeccable_provider_smoke_codex.html',
cursor: 'src/__impeccable_provider_smoke_cursor.html',
confirmedClaude: 'src/__impeccable_provider_smoke_confirmed_claude.html',
confirmedCodex: 'src/__impeccable_provider_smoke_confirmed_codex.html',
confirmedCursor: 'src/__impeccable_provider_smoke_confirmed_cursor.html',
agentChoiceClaude: 'src/__impeccable_provider_smoke_font_choice_claude.html',
agentChoiceCodex: 'src/__impeccable_provider_smoke_font_choice_codex.html',
agentChoiceCursor: 'src/__impeccable_provider_smoke_font_choice_cursor.html',
const directSmokeFile = 'src/__impeccable_provider_smoke_direct.html';
const providerSmoke = {
claude: {
fixture: 'src/__impeccable_provider_smoke_claude.html',
confirmedFixture: 'src/__impeccable_provider_smoke_confirmed_claude.html',
agentChoiceFixture: 'src/__impeccable_provider_smoke_font_choice_claude.html',
admin: '.claude/skills/impeccable/scripts/hook-admin.mjs',
hook: '.claude/skills/impeccable/scripts/hook.mjs',
event: (file) => postToolUseEvent('confirmed-claude', file, 'Edit'),
},
codex: {
fixture: 'src/__impeccable_provider_smoke_codex.html',
confirmedFixture: 'src/__impeccable_provider_smoke_confirmed_codex.html',
agentChoiceFixture: 'src/__impeccable_provider_smoke_font_choice_codex.html',
admin: '.agents/skills/impeccable/scripts/hook-admin.mjs',
hook: '.agents/skills/impeccable/scripts/hook.mjs',
event: (file) => postToolUseEvent('confirmed-codex', file, 'apply_patch'),
},
cursor: {
fixture: 'src/__impeccable_provider_smoke_cursor.html',
confirmedFixture: 'src/__impeccable_provider_smoke_confirmed_cursor.html',
agentChoiceFixture: 'src/__impeccable_provider_smoke_font_choice_cursor.html',
admin: '.cursor/skills/impeccable/scripts/hook-admin.mjs',
hook: '.cursor/skills/impeccable/scripts/hook-before-edit.mjs',
event: (file) => ({
hook_event_name: 'preToolUse',
cwd: targetRepo,
tool_name: 'Write',
tool_input: {
file_path: file,
content: readFileSync(file, 'utf8'),
},
}),
},
};
const results = [];
const hookConfigFiles = ['.impeccable/config.json', '.impeccable/config.local.json'];
const originalHookConfigFiles = new Map();
function legacySmokeArg(value) {
// The retired local parser represented a bare flag as the string "true".
// Preserve that CLI/error behavior while sharing the repository parser.
return value === true ? 'true' : value;
}
main().catch((error) => {
restoreHookConfigFiles();
if (!results.some((result) => !result.pass)) {
@@ -109,21 +140,6 @@ async function checked(name, classification, fn) {
}
}
function parseArgs(argv) {
const out = {};
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
if (!arg.startsWith('--')) continue;
const eq = arg.indexOf('=');
if (eq !== -1) {
out[arg.slice(2, eq)] = arg.slice(eq + 1);
} else {
out[arg.slice(2)] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : 'true';
}
}
return out;
}
function assertPath(path, label) {
if (!existsSync(path)) throw new Error(`${label} does not exist: ${path}`);
}
@@ -466,7 +482,7 @@ function assertNoPluginInstall() {
function runDirectContractChecks() {
clearRuntimeState();
const file = writeBadFixture(smokeFiles.direct);
const file = writeBadFixture(directSmokeFile);
const env = { IMPECCABLE_HOOK_LOG: join(smokeDir, 'direct.ndjson') };
const claude = run('node', ['.claude/skills/impeccable/scripts/hook.mjs'], {
cwd: targetRepo,
@@ -495,7 +511,7 @@ function runDirectContractChecks() {
cwd: targetRepo,
tool_name: 'Write',
tool_input: {
file_path: join(targetRepo, smokeFiles.direct),
file_path: join(targetRepo, directSmokeFile),
content: badFixtureContent(),
},
}),
@@ -515,7 +531,7 @@ function runConfirmedExceptionPersistenceChecks() {
function runConfirmedExceptionForProvider(provider) {
clearRuntimeState();
const rel = confirmedSmokeFile(provider);
const rel = providerSmoke[provider].confirmedFixture;
const file = writeConfirmedFixture(rel);
const beforeLog = `${provider}-confirmed-before.ndjson`;
const afterLog = `${provider}-confirmed-after.ndjson`;
@@ -528,7 +544,7 @@ function runConfirmedExceptionForProvider(provider) {
assertNoSpecificFontIgnoreConfig(provider);
run('node', [
providerAdminScript(provider),
providerSmoke[provider].admin,
'ignore-value',
'overused-font',
'Roboto',
@@ -580,7 +596,7 @@ function runAgentChosenFontExceptionChecks() {
function runAgentChosenFontExceptionForProvider(provider) {
clearRuntimeState();
const rel = agentChoiceSmokeFile(provider);
const rel = providerSmoke[provider].agentChoiceFixture;
const file = writeConfirmedFixture(rel);
const beforeLog = `${provider}-agent-choice-before.ndjson`;
const afterLog = `${provider}-agent-choice-after.ndjson`;
@@ -612,27 +628,40 @@ function runAgentChosenFontExceptionForProvider(provider) {
}
function runProviderAgentFontException(provider, rel) {
const prompt = fontExceptionPrompt(provider, rel);
runProviderAgent(provider, fontExceptionPrompt(provider, rel), {
logName: `${provider}-agent-choice.log`,
claudeDebugLog: 'claude-agent-choice-debug.log',
});
}
function runProviderAgent(provider, prompt, {
logName,
env = {},
claudeDebugLog,
claudeTools = 'Read,Bash',
claudeAllowedTools = 'Read Bash',
cursorReady = false,
} = {}) {
if (provider === 'claude') {
run('claude', [
return run('claude', [
'-p',
'--setting-sources', 'project',
'--permission-mode', 'acceptEdits',
'--tools', 'Read,Bash',
'--allowedTools', 'Read Bash',
'--tools', claudeTools,
'--allowedTools', claudeAllowedTools,
'--debug', 'hooks',
'--debug-file', join(smokeDir, 'claude-agent-choice-debug.log'),
'--debug-file', join(smokeDir, claudeDebugLog),
prompt,
], {
cwd: targetRepo,
logName: 'claude-agent-choice.log',
env,
logName,
timeoutMs: 10 * 60 * 1000,
});
return;
}
if (provider === 'codex') {
run('codex', [
return run('codex', [
'exec',
'-C', targetRepo,
'--dangerously-bypass-hook-trust',
@@ -641,14 +670,14 @@ function runProviderAgentFontException(provider, rel) {
prompt,
], {
cwd: targetRepo,
logName: 'codex-agent-choice.log',
env,
logName,
timeoutMs: 10 * 60 * 1000,
});
return;
}
if (provider === 'cursor') {
ensureCursorAgent();
if (!cursorReady) ensureCursorAgent();
const res = run('agent', [
'-p',
'--force',
@@ -658,7 +687,8 @@ function runProviderAgentFontException(provider, rel) {
prompt,
], {
cwd: targetRepo,
logName: 'cursor-agent-choice.log',
env,
logName,
timeoutMs: 10 * 60 * 1000,
allowFailure: true,
});
@@ -671,10 +701,10 @@ function runProviderAgentFontException(provider, rel) {
}
throw new Error(res.error ? `agent failed: ${res.error.message}` : `agent exited ${res.status}`);
}
return;
return res;
}
throw new Error(`Unsupported agent-choice provider: ${provider}`);
throw new Error(`Unsupported provider agent: ${provider}`);
}
function assertSpecificFontIgnoreConfig(provider, config) {
@@ -713,84 +743,29 @@ function readSharedHookConfig() {
}
function runInstalledProviderHook(provider, file, logName) {
const smoke = providerSmoke[provider];
const env = { IMPECCABLE_HOOK_LOG: join(smokeDir, logName) };
if (provider === 'claude') {
return run('node', ['.claude/skills/impeccable/scripts/hook.mjs'], {
cwd: targetRepo,
env,
logName: `direct-${provider}-confirmed-${logName.replace(/\.ndjson$/, '.log')}`,
input: JSON.stringify(postToolUseEvent(`confirmed-${provider}`, file, 'Edit')),
});
}
if (provider === 'codex') {
return run('node', ['.agents/skills/impeccable/scripts/hook.mjs'], {
cwd: targetRepo,
env,
logName: `direct-${provider}-confirmed-${logName.replace(/\.ndjson$/, '.log')}`,
input: JSON.stringify(postToolUseEvent(`confirmed-${provider}`, file, 'apply_patch')),
});
}
if (provider === 'cursor') {
return run('node', ['.cursor/skills/impeccable/scripts/hook-before-edit.mjs'], {
cwd: targetRepo,
env,
logName: `direct-${provider}-confirmed-${logName.replace(/\.ndjson$/, '.log')}`,
input: JSON.stringify({
hook_event_name: 'preToolUse',
cwd: targetRepo,
tool_name: 'Write',
tool_input: {
file_path: file,
content: readFileSync(file, 'utf8'),
},
}),
});
}
throw new Error(`Unsupported confirmed exception provider: ${provider}`);
}
function confirmedSmokeFile(provider) {
if (provider === 'claude') return smokeFiles.confirmedClaude;
if (provider === 'codex') return smokeFiles.confirmedCodex;
if (provider === 'cursor') return smokeFiles.confirmedCursor;
throw new Error(`Unsupported confirmed exception provider: ${provider}`);
}
function agentChoiceSmokeFile(provider) {
if (provider === 'claude') return smokeFiles.agentChoiceClaude;
if (provider === 'codex') return smokeFiles.agentChoiceCodex;
if (provider === 'cursor') return smokeFiles.agentChoiceCursor;
throw new Error(`Unsupported agent-choice provider: ${provider}`);
}
function providerAdminScript(provider) {
if (provider === 'claude') return '.claude/skills/impeccable/scripts/hook-admin.mjs';
if (provider === 'codex') return '.agents/skills/impeccable/scripts/hook-admin.mjs';
if (provider === 'cursor') return '.cursor/skills/impeccable/scripts/hook-admin.mjs';
throw new Error(`Unsupported admin provider: ${provider}`);
return run('node', [smoke.hook], {
cwd: targetRepo,
env,
logName: `direct-${provider}-confirmed-${logName.replace(/\.ndjson$/, '.log')}`,
input: JSON.stringify(smoke.event(file)),
});
}
function runClaudeProviderSmoke() {
clearRuntimeState();
const env = { IMPECCABLE_HOOK_LOG: join(smokeDir, 'claude.ndjson') };
const prompt = providerPrompt(smokeFiles.claude);
const res = run('claude', [
'-p',
'--setting-sources', 'project',
'--permission-mode', 'acceptEdits',
'--tools', 'Read,Write,Edit',
'--allowedTools', 'Read Write Edit',
'--debug', 'hooks',
'--debug-file', join(smokeDir, 'claude-debug.log'),
prompt,
], {
cwd: targetRepo,
const prompt = providerPrompt(providerSmoke.claude.fixture);
const res = runProviderAgent('claude', prompt, {
env,
logName: 'claude-provider.log',
timeoutMs: 10 * 60 * 1000,
claudeDebugLog: 'claude-debug.log',
claudeTools: 'Read,Write,Edit',
claudeAllowedTools: 'Read Write Edit',
});
const evidence = `${res.stdout}\n${res.stderr}\n${readMaybe(join(smokeDir, 'claude.ndjson'))}\n${readMaybe(join(smokeDir, 'claude-debug.log'))}`;
requireFile(smokeFiles.claude, 'Claude provider fixture');
requireFile(providerSmoke.claude.fixture, 'Claude provider fixture');
requireFinding('Claude provider hook', evidence);
if (!/PostToolUse|hook/i.test(evidence)) throw new Error('Claude provider evidence lacks hook/PostToolUse marker');
record('claude provider', true, 'Claude edit triggered PostToolUse hook and side-tab detection');
@@ -799,23 +774,14 @@ function runClaudeProviderSmoke() {
function runCodexProviderSmoke() {
clearRuntimeState();
const env = { IMPECCABLE_HOOK_LOG: join(smokeDir, 'codex.ndjson') };
const prompt = `Use apply_patch to ${providerPrompt(smokeFiles.codex)}`;
const res = run('codex', [
'exec',
'-C', targetRepo,
'--dangerously-bypass-hook-trust',
'--dangerously-bypass-approvals-and-sandbox',
'--json',
prompt,
], {
cwd: targetRepo,
const prompt = `Use apply_patch to ${providerPrompt(providerSmoke.codex.fixture)}`;
const res = runProviderAgent('codex', prompt, {
env,
logName: 'codex-provider.log',
timeoutMs: 10 * 60 * 1000,
});
const evidence = `${res.stdout}\n${res.stderr}\n${readMaybe(join(smokeDir, 'codex.ndjson'))}`;
const cacheEvidence = `${readMaybe(join(targetRepo, '.impeccable', 'hook.cache.json'))}\n${readMaybe(join(targetRepo, '.impeccable', 'hook.pending.json'))}`;
requireFile(smokeFiles.codex, 'Codex provider fixture');
requireFile(providerSmoke.codex.fixture, 'Codex provider fixture');
requireFinding('Codex provider hook', `${evidence}\n${cacheEvidence}`);
record('codex provider', true, 'Codex apply_patch triggered project hook and side-tab detection');
}
@@ -824,37 +790,19 @@ function runCursorProviderSmoke() {
ensureCursorAgent();
clearRuntimeState();
const env = { IMPECCABLE_HOOK_LOG: join(smokeDir, 'cursor.ndjson') };
const prompt = providerPrompt(smokeFiles.cursor);
const res = run('agent', [
'-p',
'--force',
'--trust',
'--workspace', targetRepo,
'--output-format', 'stream-json',
prompt,
], {
cwd: targetRepo,
const prompt = providerPrompt(providerSmoke.cursor.fixture);
const res = runProviderAgent('cursor', prompt, {
env,
logName: 'cursor-provider.log',
timeoutMs: 10 * 60 * 1000,
allowFailure: true,
cursorReady: true,
});
if (res.error || res.status !== 0) {
const output = `${res.stdout}\n${res.stderr}\n${res.error?.message || ''}`;
if (/Authentication required|agent login|CURSOR_API_KEY/i.test(output)) {
const err = new Error('Cursor CLI authentication required. Run `agent login` or set CURSOR_API_KEY, then rerun `bun run smoke:hooks -- --providers=cursor`.');
err.classification = 'cursor auth required';
throw err;
}
throw new Error(res.error ? `agent failed: ${res.error.message}` : `agent exited ${res.status}`);
}
const evidence = `${res.stdout}\n${res.stderr}\n${readMaybe(join(smokeDir, 'cursor.ndjson'))}\n${readMaybe(join(targetRepo, '.impeccable', 'hook.pending.json'))}\n${readMaybe(join(targetRepo, '.impeccable', 'hook.cache.json'))}`;
requireFinding('Cursor provider hook', evidence);
const auditEvents = readAuditEvents(join(smokeDir, 'cursor.ndjson'));
if (!auditEvents.some((event) => event.event === 'preToolUse' && event.blocked === true)) {
throw new Error('Cursor provider evidence lacks a preToolUse audit entry with blocked=true');
}
const fixturePath = join(targetRepo, smokeFiles.cursor);
const fixturePath = join(targetRepo, providerSmoke.cursor.fixture);
const intentionalIgnore = auditEvents.some((event) =>
event.event === 'preToolUse'
&& event.file === fixturePath
@@ -1017,7 +965,15 @@ function cleanSmokeArtifacts() {
}
function cleanSmokeFiles() {
for (const rel of Object.values(smokeFiles)) {
const files = [
directSmokeFile,
...Object.values(providerSmoke).flatMap(({ fixture, confirmedFixture, agentChoiceFixture }) => [
fixture,
confirmedFixture,
agentChoiceFixture,
]),
];
for (const rel of files) {
rmSync(join(targetRepo, rel), { force: true });
}
}
+48
View File
@@ -6,9 +6,16 @@
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { boolFlag, parseArgs, positiveIntFlag, resolveEnum, toCamel } from '../scripts/lib/cli-args.mjs';
const PROVIDER_SMOKE_SCRIPT = fileURLToPath(new URL('../scripts/smoke-provider-hooks.mjs', import.meta.url));
describe('parseArgs', () => {
it('reads space-separated values', () => {
// The regression: without the argv[i+1] lookahead this yielded
@@ -125,3 +132,44 @@ describe('resolveEnum', () => {
);
});
});
describe('provider hook smoke CLI', () => {
it('prints help without requiring a target repository', () => {
const result = spawnSync(process.execPath, [PROVIDER_SMOKE_SCRIPT, '--help'], { encoding: 'utf8' });
assert.equal(result.status, 0);
assert.match(result.stdout, /^Usage: bun run smoke:hooks/);
assert.match(result.stdout, /target repo must be explicit/);
assert.equal(result.stderr, '');
});
it('fails with the same usage guidance when the target repository is omitted', () => {
const result = spawnSync(process.execPath, [PROVIDER_SMOKE_SCRIPT], { encoding: 'utf8' });
assert.equal(result.status, 1);
assert.equal(result.stdout, '');
assert.match(result.stderr, /^Usage: bun run smoke:hooks/);
assert.match(result.stderr, /target repo must be explicit/);
});
it('preserves the legacy string sentinel for value-less options', () => {
const cases = [
{ args: ['--repo'], error: /target repo does not exist: .*\/true/ },
{ args: ['--repo', '.', '--bundle'], error: /universal bundle does not exist: .*\/true/ },
{ args: ['--repo', '.', '--bundle', './missing.zip', '--providers'], error: /universal bundle does not exist: .*\/missing\.zip/ },
];
for (const { args, error } of cases) {
const cwd = mkdtempSync(join(tmpdir(), 'impeccable-provider-smoke-cli-'));
try {
const result = spawnSync(process.execPath, [PROVIDER_SMOKE_SCRIPT, ...args], { cwd, encoding: 'utf8' });
assert.equal(result.status, 1);
assert.doesNotMatch(result.stderr, /TypeError/);
assert.match(result.stderr, error);
} finally {
rmSync(cwd, { recursive: true, force: true });
}
}
});
});