diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6ee4d27c5..f9235ba68 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -29,6 +29,15 @@ assignees: '' +## How did you run impeccable? + + + ## Provider & environment - **Provider** (Cursor / Claude Code / Gemini CLI / Codex / Copilot / Kiro / OpenCode): diff --git a/.github/workflows/issue-gate.yml b/.github/workflows/issue-gate.yml index e3296a3cd..4ce9fc586 100644 --- a/.github/workflows/issue-gate.yml +++ b/.github/workflows/issue-gate.yml @@ -5,12 +5,12 @@ on: types: [opened, edited, reopened] issue_comment: types: [created] - schedule: - # Daily sweep: closes issues that have failed the template checks for - # --close-days days after the warning comment. - - cron: "43 14 * * *" workflow_dispatch: inputs: + issue: + description: "Issue number to evaluate" + type: number + required: true dry_run: description: "Print planned changes without mutating GitHub" type: boolean @@ -21,7 +21,7 @@ permissions: issues: write concurrency: - group: issue-gate-${{ github.event.issue.number || 'sweep' }} + group: issue-gate-${{ github.event.issue.number || inputs.issue }} cancel-in-progress: false jobs: @@ -51,20 +51,20 @@ jobs: fi case "${{ github.event_name }}" in - issues) - node scripts/github/issue-gate.mjs "$mode" \ - --repo "$GITHUB_REPOSITORY" \ - --issue "${{ github.event.issue.number }}" - ;; issue_comment) node scripts/github/issue-gate.mjs "$mode" \ --repo "$GITHUB_REPOSITORY" \ --issue "${{ github.event.issue.number }}" \ --comment-id "${{ github.event.comment.id }}" ;; + workflow_dispatch) + node scripts/github/issue-gate.mjs "$mode" \ + --repo "$GITHUB_REPOSITORY" \ + --issue "${{ inputs.issue }}" + ;; *) node scripts/github/issue-gate.mjs "$mode" \ --repo "$GITHUB_REPOSITORY" \ - --sweep + --issue "${{ github.event.issue.number }}" ;; esac diff --git a/scripts/github/issue-gate.mjs b/scripts/github/issue-gate.mjs index 69db37976..8e141c2b4 100644 --- a/scripts/github/issue-gate.mjs +++ b/scripts/github/issue-gate.mjs @@ -2,23 +2,22 @@ // Issue gate: deterministic slop control for GitHub issues. // // Three gates, all mechanical: -// 1. Template structure: the body must contain the section headings from one -// of the issue templates, with real content under the required ones. +// 1. Template structure: the body must contain the required section headings +// from one of the issue templates. // 2. Prose length: the body may not exceed a word budget once fenced code // blocks and
blocks are excluded, so context dumps fail while // long logs stay legal. // 3. Comment dumps: an oversized comment from the issue author within the // first hour of the issue's life gets minimized as off-topic. // -// Failing issues are labeled and warned once; a daily sweep closes issues that -// still fail after --close-days. Every check re-runs on edit, and an issue the -// gate closed is reopened automatically once it passes. +// Issues with no template structure at all are closed immediately; partial +// failures are labeled and warned once. Every check re-runs on edit, and an +// issue the gate closed is reopened automatically once it passes. import { readFileSync, readdirSync } from 'node:fs'; import { join } from 'node:path'; import { spawnSync } from 'node:child_process'; import { pathToFileURL } from 'node:url'; -const DAY_MS = 24 * 60 * 60 * 1000; const MINUTE_MS = 60 * 1000; export const GATE_LABEL = 'policy: needs template'; @@ -31,7 +30,6 @@ export const LABEL_DEFS = [ export const GATE_MARKER = ''; export const REJECT_MARKER = ''; -export const CLOSE_MARKER = ''; const DEFAULT_MAINTAINERS = ['pbakaus', 'abdulwahabone']; const DEFAULT_TRUSTED_MARKER_AUTHORS = ['github-actions', 'github-actions[bot]']; @@ -52,7 +50,6 @@ const OPTIONAL_SECTIONS = new Set([ export const DEFAULT_MAX_PROSE_WORDS = 600; export const DEFAULT_COMMENT_MAX_PROSE_WORDS = 300; export const DEFAULT_COMMENT_WINDOW_MINUTES = 60; -export const DEFAULT_CLOSE_DAYS = 5; // --- prose measurement ------------------------------------------------------- @@ -96,15 +93,11 @@ export function parseTemplate(fileName, raw) { const body = stripFrontmatter(raw); const nameMatch = raw.match(/^name:\s*(.+)$/m); const headings = extractHeadings(body); - const scaffoldLines = new Set( - body.split(/\r?\n/).map((line) => line.trim()).filter(Boolean), - ); return { file: fileName, name: nameMatch ? nameMatch[1].trim() : fileName, headings, requiredSections: headings.filter((heading) => !OPTIONAL_SECTIONS.has(heading)), - scaffoldLines, }; } @@ -117,43 +110,11 @@ export function loadTemplates(dir) { // --- issue evaluation -------------------------------------------------------- -// Only headings that name a template section start a new section; authors use -// other headings (sub-scenarios, log labels) as content inside a section. -function splitSections(body, knownHeadings) { - const sections = new Map(); - let current = null; - for (const line of String(body || '').split(/\r?\n/)) { - const match = line.match(/^#{2,3}\s+(.+)$/); - if (match && knownHeadings.has(normalizeHeading(match[1]))) { - current = normalizeHeading(match[1]); - if (!sections.has(current)) sections.set(current, []); - continue; - } - if (current) sections.get(current).push(line); - } - return sections; -} - -// A section counts as filled when it contains at least one line the author -// wrote: non-empty, not an HTML comment, and not copied verbatim from the -// template scaffolding (numbered placeholders, field labels, unchecked boxes). -function sectionIsFilled(lines, scaffoldLines) { - const withoutComments = stripNonProse(lines.join('\n')); - for (const rawLine of withoutComments.split(/\r?\n/)) { - const line = rawLine.trim(); - if (!line) continue; - if (scaffoldLines.has(line)) continue; - return true; - } - return false; -} - export function evaluateIssue(issue, options = {}) { const templates = options.templates || []; const maxProseWords = Number.isFinite(options.maxProseWords) ? options.maxProseWords : DEFAULT_MAX_PROSE_WORDS; - const closeDays = Number.isFinite(options.closeDays) ? options.closeDays : DEFAULT_CLOSE_DAYS; const maintainers = loginSet(options.maintainers || DEFAULT_MAINTAINERS); const trustedMarkerAuthors = loginSet(options.trustedMarkerAuthors || DEFAULT_TRUSTED_MARKER_AUTHORS); const repo = options.repo || ''; @@ -201,13 +162,9 @@ export function evaluateIssue(issue, options = {}) { matched: template.requiredSections.filter((section) => bodyHeadings.has(section)).length, })) .sort((a, b) => b.matched - a.matched)[0].template; - const knownHeadings = new Set(templates.flatMap((template) => template.headings)); - const sections = splitSections(body, knownHeadings); for (const section of primary.requiredSections) { if (!bodyHeadings.has(section)) { reasons.push(`missing section "${section}" from the ${primary.name.toLowerCase()} template`); - } else if (!sectionIsFilled(sections.get(section) || [], primary.scaffoldLines)) { - reasons.push(`section "${section}" is empty or still contains only template placeholders`); } } } @@ -227,8 +184,7 @@ export function evaluateIssue(issue, options = {}) { if (verdict === 'pass') { if (labels.has(GATE_LABEL)) plan.labelsToRemove.push(GATE_LABEL); - const closedByGate = hasMarker(comments, REJECT_MARKER, trustedMarkerAuthors) - || hasMarker(comments, CLOSE_MARKER, trustedMarkerAuthors); + const closedByGate = hasMarker(comments, REJECT_MARKER, trustedMarkerAuthors); plan.shouldReopen = issue.state === 'closed' && labels.has(GATE_LABEL) && closedByGate; return plan; } @@ -243,33 +199,10 @@ export function evaluateIssue(issue, options = {}) { } plan.shouldComment = !hasMarker(comments, GATE_MARKER, trustedMarkerAuthors); - plan.comment = needsWorkComment(reasons, { repo, closeDays }); + plan.comment = needsWorkComment(reasons, { repo }); return plan; } -export function evaluateSweep(issue, options = {}) { - const closeDays = Number.isFinite(options.closeDays) ? options.closeDays : DEFAULT_CLOSE_DAYS; - const trustedMarkerAuthors = loginSet(options.trustedMarkerAuthors || DEFAULT_TRUSTED_MARKER_AUTHORS); - const now = toDate(options.now || new Date()); - - const plan = evaluateIssue(issue, options); - if (plan.verdict === 'pass' || plan.verdict === 'reject') { - return { ...plan, shouldSweepClose: false, sweepComment: '' }; - } - - const warnedAt = latestMarkerAt(issue.comments || [], GATE_MARKER, trustedMarkerAuthors); - const failingDays = warnedAt - ? Math.floor((now.getTime() - warnedAt.getTime()) / DAY_MS) - : 0; - const shouldSweepClose = issue.state === 'open' && Boolean(warnedAt) && failingDays >= closeDays; - return { - ...plan, - shouldComment: plan.shouldComment && !shouldSweepClose, - shouldSweepClose, - sweepComment: shouldSweepClose ? sweepCloseComment(failingDays) : '', - }; -} - // --- comment evaluation ------------------------------------------------------ export function evaluateComment(input, options = {}) { @@ -311,14 +244,14 @@ function templatesUrl(repo) { : '.github/ISSUE_TEMPLATE'; } -export function needsWorkComment(reasons, { repo = '', closeDays = DEFAULT_CLOSE_DAYS } = {}) { +export function needsWorkComment(reasons, { repo = '' } = {}) { return [ GATE_MARKER, 'Thanks for filing this. It does not pass the issue template checks yet:', '', ...reasons.map((reason) => `- ${reason}`), '', - `Please edit the issue body itself (not a new comment) to follow one of the [issue templates](${templatesUrl(repo)}). The checks run again on every edit and clear the label once they pass. Issues that still fail after ${closeDays} days are closed automatically.`, + `Please edit the issue body itself (not a new comment) to follow one of the [issue templates](${templatesUrl(repo)}). The checks run again on every edit and clear the label once they pass.`, '', 'If AI helped write this issue, include the line `AI-assisted: yes` in the body.', ].join('\n'); @@ -335,15 +268,6 @@ export function rejectComment(repo = '') { ].join('\n'); } -export function sweepCloseComment(failingDays) { - return [ - CLOSE_MARKER, - `Closing this because the issue template checks have been failing for ${failingDays} days.`, - '', - 'Edit the issue body to pass the checks and it is reopened automatically on the next edit.', - ].join('\n'); -} - // --- driver ------------------------------------------------------------------ export async function main(argv = process.argv.slice(2)) { @@ -362,11 +286,6 @@ export async function main(argv = process.argv.slice(2)) { return; } - if (options.sweep) { - runSweep(repo, options); - return; - } - if (options.issue) { const issue = fetchIssue(repo, options.issue); if (!issue) throw new Error(`Issue #${options.issue} not found.`); @@ -376,7 +295,7 @@ export async function main(argv = process.argv.slice(2)) { return; } - throw new Error('Nothing to do. Pass --issue N, --issue N --comment-id ID, or --sweep.'); + throw new Error('Nothing to do. Pass --issue N or --issue N --comment-id ID.'); } function runCommentGate(repo, options) { @@ -406,22 +325,6 @@ function runCommentGate(repo, options) { if (options.apply) minimizeComment(comment.nodeId); } -function runSweep(repo, options) { - const issues = fetchGateLabeledIssues(repo); - for (const issue of issues) { - issue.comments = fetchIssueComments(repo, issue.number); - const plan = evaluateSweep(issue, options); - printPlan(plan, options); - if (!options.apply) continue; - applyIssuePlan(repo, plan); - if (plan.shouldSweepClose) { - postComment(repo, plan.number, plan.sweepComment); - closeIssue(repo, plan.number); - } - } - console.log(`${options.apply ? 'Applied' : 'Dry run'} issue-gate sweep for ${issues.length} labeled issue(s).`); -} - function applyIssuePlan(repo, plan) { if (plan.exempt) return; if (plan.shouldReopen) reopenIssue(repo, plan.number); @@ -439,7 +342,6 @@ function printPlan(plan, { apply }) { plan.shouldComment ? 'comment' : '', plan.shouldClose ? 'close' : '', plan.shouldReopen ? 'reopen' : '', - plan.shouldSweepClose ? 'sweep-close' : '', ].filter(Boolean); console.log(`${apply ? 'apply' : 'dry-run'} #${plan.number} ${plan.title || ''}: ${changes.join(' ')}`); for (const reason of plan.reasons) console.log(` - ${reason}`); @@ -449,14 +351,12 @@ export function parseArgs(argv) { const options = { apply: false, ensureLabels: true, - sweep: false, issue: null, commentId: null, templatesDir: '.github/ISSUE_TEMPLATE', maxProseWords: DEFAULT_MAX_PROSE_WORDS, commentMaxProseWords: DEFAULT_COMMENT_MAX_PROSE_WORDS, windowMinutes: DEFAULT_COMMENT_WINDOW_MINUTES, - closeDays: DEFAULT_CLOSE_DAYS, maintainers: DEFAULT_MAINTAINERS, now: new Date(), }; @@ -465,7 +365,6 @@ export function parseArgs(argv) { const arg = argv[i]; if (arg === '--apply') options.apply = true; else if (arg === '--dry-run') options.apply = false; - else if (arg === '--sweep') options.sweep = true; else if (arg === '--no-label-ensure') options.ensureLabels = false; else if (arg === '--repo') options.repo = requireValue(argv, ++i, arg); else if (arg === '--issue') options.issue = Number(requireValue(argv, ++i, arg)); @@ -474,7 +373,6 @@ export function parseArgs(argv) { else if (arg === '--max-words') options.maxProseWords = Number(requireValue(argv, ++i, arg)); else if (arg === '--comment-max-words') options.commentMaxProseWords = Number(requireValue(argv, ++i, arg)); else if (arg === '--comment-window-minutes') options.windowMinutes = Number(requireValue(argv, ++i, arg)); - else if (arg === '--close-days') options.closeDays = Number(requireValue(argv, ++i, arg)); else if (arg === '--maintainers') options.maintainers = splitList(requireValue(argv, ++i, arg)); else if (arg === '--now') options.now = new Date(requireValue(argv, ++i, arg)); else if (arg === '--help' || arg === '-h') { @@ -491,9 +389,6 @@ export function parseArgs(argv) { if (!Number.isFinite(options.maxProseWords) || options.maxProseWords <= 0) { throw new Error('--max-words must be a positive number.'); } - if (!Number.isFinite(options.closeDays) || options.closeDays < 1) { - throw new Error('--close-days must be at least 1.'); - } if (Number.isNaN(options.now.getTime())) throw new Error('--now must be a valid date.'); return options; @@ -553,17 +448,6 @@ function fetchComment(repo, commentId) { }; } -function fetchGateLabeledIssues(repo) { - const pages = runGhJson([ - 'api', - '--paginate', - '--slurp', - `repos/${repo}/issues?state=open&labels=${encodeURIComponent(GATE_LABEL)}&per_page=100`, - ]); - const flat = Array.isArray(pages) ? pages.flat() : []; - return flat.map(normalizeIssue).filter((issue) => !issue.isPullRequest); -} - function ensureLabels(repo) { for (const label of LABEL_DEFS) { const encoded = encodeURIComponent(label.name); @@ -637,17 +521,6 @@ function hasMarker(comments = [], marker, trustedAuthors) { return comments.some((comment) => isTrustedMarkerComment(comment, marker, trustedAuthors)); } -function latestMarkerAt(comments = [], marker, trustedAuthors) { - let latest = null; - for (const comment of comments) { - if (!isTrustedMarkerComment(comment, marker, trustedAuthors)) continue; - const date = toDate(comment.createdAt); - if (Number.isNaN(date.getTime())) continue; - if (!latest || date > latest) latest = date; - } - return latest; -} - function isTrustedMarkerComment(comment, marker, trustedAuthors) { if (typeof comment?.body !== 'string' || !comment.body.includes(marker)) return false; return trustedAuthors.has(normalizeLogin(comment.authorLogin)); @@ -709,7 +582,6 @@ Default mode is a dry run. Modes: --issue N evaluate one issue body (workflow: issues opened/edited) --issue N --comment-id ID evaluate one comment (workflow: issue_comment created) - --sweep re-check all '${GATE_LABEL}' issues and close long-failing ones Options: --apply mutate labels, comments, and issue state @@ -719,7 +591,6 @@ Options: --max-words n prose word budget for issue bodies (default: ${DEFAULT_MAX_PROSE_WORDS}) --comment-max-words n prose word budget for early self-replies (default: ${DEFAULT_COMMENT_MAX_PROSE_WORDS}) --comment-window-minutes n self-reply window after issue creation (default: ${DEFAULT_COMMENT_WINDOW_MINUTES}) - --close-days n days a warned issue may keep failing before close (default: ${DEFAULT_CLOSE_DAYS}) --maintainers a,b logins exempt from all gates --no-label-ensure skip creating gate labels `); diff --git a/tests/github-issue-gate.test.mjs b/tests/github-issue-gate.test.mjs index e252b4a68..bd9ca5bfb 100644 --- a/tests/github-issue-gate.test.mjs +++ b/tests/github-issue-gate.test.mjs @@ -5,20 +5,17 @@ import { dirname, join } from 'node:path'; import { AI_LABEL, - CLOSE_MARKER, GATE_LABEL, GATE_MARKER, REJECT_MARKER, evaluateComment, evaluateIssue, - evaluateSweep, loadTemplates, parseArgs, proseWordCount, } from '../scripts/github/issue-gate.mjs'; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); -const NOW = '2026-08-05T12:00:00Z'; let templates; before(() => { @@ -38,6 +35,10 @@ Running the detect command against a directory crashes with a TypeError. A findings report. +## How did you run impeccable? + +Via \`npx impeccable detect\` as documented. + ## Provider & environment - **Provider** (Cursor / Claude Code / Gemini CLI / Codex / Copilot / Kiro / OpenCode): Cursor @@ -76,7 +77,12 @@ describe('issue gate templates', () => { it('derives required sections from the real templates', () => { const bug = templates.find((template) => template.file === 'bug_report.md'); const feature = templates.find((template) => template.file === 'feature_request.md'); - assert.deepEqual(bug.requiredSections, ['what happened', 'steps to reproduce', 'provider & environment']); + assert.deepEqual(bug.requiredSections, [ + 'what happened', + 'steps to reproduce', + 'how did you run impeccable', + 'provider & environment', + ]); assert.deepEqual(feature.requiredSections, ['what problem does this solve', 'proposed solution']); }); }); @@ -117,10 +123,7 @@ describe('issue body gate', () => { assert.ok(plan.comment.includes(REJECT_MARKER)); }); - it('flags untouched template placeholders as empty sections', () => { - const bugTemplate = templates.find((template) => template.file === 'bug_report.md'); - const untouched = [...bugTemplate.scaffoldLines].length > 0; - assert.ok(untouched); + it('accepts headings without judging the content under them', () => { const plan = evaluateIssue(issue({ body: [ '## What happened?', @@ -133,6 +136,8 @@ describe('issue body gate', () => { '2. ', '3. ', '', + '## How did you run impeccable?', + '', '## Provider & environment', '', '- **Provider** (Cursor / Claude Code / Gemini CLI / Codex / Copilot / Kiro / OpenCode):', @@ -140,20 +145,20 @@ describe('issue body gate', () => { '- **OS**: ', ].join('\n'), }), { templates }); - assert.equal(plan.verdict, 'needs-work'); - assert.equal(plan.reasons.length, 3); - assert.match(plan.reasons[0], /section "what happened" is empty/); - assert.equal(plan.shouldClose, false); - assert.deepEqual(plan.labelsToAdd, [GATE_LABEL]); + assert.equal(plan.verdict, 'pass'); }); - it('names a missing required section', () => { + it('names missing required sections', () => { const plan = evaluateIssue(issue({ body: [ '## What happened?', '', 'The build fails.', '', + '## How did you run impeccable?', + '', + 'As documented, via /impeccable audit.', + '', '## Provider & environment', '', '- **OS**: macOS 15 and provider Cursor 2.4.0', @@ -179,7 +184,7 @@ describe('issue body gate', () => { assert.equal(plan.verdict, 'pass'); }); - it('treats author sub-headings as section content, not section boundaries', () => { + it('ignores author sub-headings between template sections', () => { const plan = evaluateIssue(issue({ body: [ '## What happened?', @@ -196,6 +201,10 @@ describe('issue body gate', () => { '', '1. Seed a Claude install first, then run the same command.', '', + '## How did you run impeccable?', + '', + 'CLI, as documented.', + '', '## Provider & environment', '', '- **OS**: macOS 15, CLI 4.0.4', @@ -220,6 +229,17 @@ describe('issue body gate', () => { assert.equal(member.exempt, true); }); + it('warns a needs-work issue only once', () => { + const plan = evaluateIssue(issue({ + body: FILLED_BUG_BODY + '\n' + words(700), + labels: [GATE_LABEL], + comments: [comment('github-actions[bot]', '2026-08-01T01:00:00Z', GATE_MARKER)], + }), { templates }); + assert.equal(plan.verdict, 'needs-work'); + assert.equal(plan.shouldComment, false); + assert.deepEqual(plan.labelsToAdd, []); + }); + it('comments only once per issue', () => { const plan = evaluateIssue(issue({ body: 'no structure here', @@ -261,45 +281,6 @@ describe('issue body gate', () => { }); }); -describe('sweep', () => { - it('closes an issue still failing five days after the warning', () => { - const plan = evaluateSweep(issue({ - body: 'still no structure', - labels: [GATE_LABEL], - comments: [comment('github-actions[bot]', '2026-07-30T12:00:00Z', GATE_MARKER)], - }), { templates, now: NOW }); - assert.equal(plan.shouldSweepClose, false); - - const oversized = evaluateSweep(issue({ - body: FILLED_BUG_BODY + '\n' + words(700), - labels: [GATE_LABEL], - comments: [comment('github-actions[bot]', '2026-07-30T12:00:00Z', GATE_MARKER)], - }), { templates, now: NOW }); - assert.equal(oversized.shouldSweepClose, true); - assert.match(oversized.sweepComment, /failing for 6 days/); - assert.ok(oversized.sweepComment.includes(CLOSE_MARKER)); - }); - - it('does not close before the warning has aged past close-days', () => { - const plan = evaluateSweep(issue({ - body: FILLED_BUG_BODY + '\n' + words(700), - labels: [GATE_LABEL], - comments: [comment('github-actions[bot]', '2026-08-03T12:00:00Z', GATE_MARKER)], - }), { templates, now: NOW }); - assert.equal(plan.shouldSweepClose, false); - }); - - it('lets a fixed issue pass instead of closing it', () => { - const plan = evaluateSweep(issue({ - labels: [GATE_LABEL], - comments: [comment('github-actions[bot]', '2026-07-25T12:00:00Z', GATE_MARKER)], - }), { templates, now: NOW }); - assert.equal(plan.verdict, 'pass'); - assert.equal(plan.shouldSweepClose, false); - assert.deepEqual(plan.labelsToRemove, [GATE_LABEL]); - }); -}); - describe('comment gate', () => { const base = { commentAuthorLogin: 'outside-reporter',