From 85119487333af42650f0fdbdc4abd124b305ed80 Mon Sep 17 00:00:00 2001 From: Abdul Wahab Date: Wed, 5 Aug 2026 21:06:44 +0500 Subject: [PATCH] Add issue gate: deterministic slop control for agent-filed issues Three mechanical checks on every issue open/edit: template section structure, a 600-word prose budget (code and
excluded), and minimizing oversized early self-reply comments. Failing issues label themselves, warn once, and a daily sweep closes ones still failing after 5 days; a passing edit clears the label and reopens gate-closed issues. Maintainers and repo members are exempt. AI-assisted: written with an AI agent under maintainer direction. Co-authored-by: Cursor --- .github/ISSUE_TEMPLATE/bug_report.md | 9 + .github/ISSUE_TEMPLATE/feature_request.md | 9 + .github/workflows/issue-gate.yml | 70 +++ scripts/github/issue-gate.mjs | 733 ++++++++++++++++++++++ scripts/test-suites.mjs | 1 + tests/github-issue-gate.test.mjs | 383 +++++++++++ 6 files changed, 1205 insertions(+) create mode 100644 .github/workflows/issue-gate.yml create mode 100644 scripts/github/issue-gate.mjs create mode 100644 tests/github-issue-gate.test.mjs diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index b552969ff..6ee4d27c5 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -6,6 +6,15 @@ labels: bug assignees: '' --- + + ## What happened? diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 4f538a6f8..8936a150c 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -6,6 +6,15 @@ labels: enhancement assignees: '' --- + + ## What problem does this solve? diff --git a/.github/workflows/issue-gate.yml b/.github/workflows/issue-gate.yml new file mode 100644 index 000000000..e3296a3cd --- /dev/null +++ b/.github/workflows/issue-gate.yml @@ -0,0 +1,70 @@ +name: Issue Gate + +on: + issues: + types: [opened, edited, reopened] + issue_comment: + types: [created] + schedule: + # Daily sweep: closes issues that have failed the template checks for + # --close-days days after the warning comment. + - cron: "43 14 * * *" + workflow_dispatch: + inputs: + dry_run: + description: "Print planned changes without mutating GitHub" + type: boolean + default: false + +permissions: + contents: read + issues: write + +concurrency: + group: issue-gate-${{ github.event.issue.number || 'sweep' }} + cancel-in-progress: false + +jobs: + gate: + runs-on: ubuntu-latest + # Skip PR comments (issue_comment fires for those too) and anything the + # gate itself posts. + if: >- + github.event_name != 'issue_comment' || + (github.event.issue.pull_request == null && github.event.comment.user.type != 'Bot') + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Node + uses: actions/setup-node@v7 + with: + node-version: 24 + + - name: Run issue gate + env: + GH_TOKEN: ${{ github.token }} + run: | + mode="--apply" + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.dry_run }}" = "true" ]; then + mode="--dry-run" + fi + + case "${{ github.event_name }}" in + issues) + node scripts/github/issue-gate.mjs "$mode" \ + --repo "$GITHUB_REPOSITORY" \ + --issue "${{ github.event.issue.number }}" + ;; + issue_comment) + node scripts/github/issue-gate.mjs "$mode" \ + --repo "$GITHUB_REPOSITORY" \ + --issue "${{ github.event.issue.number }}" \ + --comment-id "${{ github.event.comment.id }}" + ;; + *) + node scripts/github/issue-gate.mjs "$mode" \ + --repo "$GITHUB_REPOSITORY" \ + --sweep + ;; + esac diff --git a/scripts/github/issue-gate.mjs b/scripts/github/issue-gate.mjs new file mode 100644 index 000000000..69db37976 --- /dev/null +++ b/scripts/github/issue-gate.mjs @@ -0,0 +1,733 @@ +#!/usr/bin/env node +// Issue gate: deterministic slop control for GitHub issues. +// +// Three gates, all mechanical: +// 1. Template structure: the body must contain the section headings from one +// of the issue templates, with real content under the required ones. +// 2. Prose length: the body may not exceed a word budget once fenced code +// blocks and
blocks are excluded, so context dumps fail while +// long logs stay legal. +// 3. Comment dumps: an oversized comment from the issue author within the +// first hour of the issue's life gets minimized as off-topic. +// +// Failing issues are labeled and warned once; a daily sweep closes issues that +// still fail after --close-days. Every check re-runs on edit, and an issue the +// gate closed is reopened automatically once it passes. +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; + +const DAY_MS = 24 * 60 * 60 * 1000; +const MINUTE_MS = 60 * 1000; + +export const GATE_LABEL = 'policy: needs template'; +export const AI_LABEL = 'ai-assisted'; + +export const LABEL_DEFS = [ + { name: GATE_LABEL, color: 'b60205', description: 'Issue body fails the template checks and will be closed if not edited' }, + { name: AI_LABEL, color: 'c5def5', description: 'Author disclosed AI assistance in the issue body' }, +]; + +export const GATE_MARKER = ''; +export const REJECT_MARKER = ''; +export const CLOSE_MARKER = ''; + +const DEFAULT_MAINTAINERS = ['pbakaus', 'abdulwahabone']; +const DEFAULT_TRUSTED_MARKER_AUTHORS = ['github-actions', 'github-actions[bot]']; +const EXEMPT_ASSOCIATIONS = new Set(['OWNER', 'MEMBER', 'COLLABORATOR']); +const AI_DISCLOSURE = /^\s*(?:[-*>\s]*)ai-assisted:\s*yes\b/im; + +// Sections a legitimate report can leave empty. Everything else in a template +// is required. Normalized (lowercase, no trailing punctuation). +const OPTIONAL_SECTIONS = new Set([ + 'expected behavior', + 'additional context', + 'alternatives considered', + 'provider(s) this applies to', + 'willing to work on a fix', + 'willing to work on this', +]); + +export const DEFAULT_MAX_PROSE_WORDS = 600; +export const DEFAULT_COMMENT_MAX_PROSE_WORDS = 300; +export const DEFAULT_COMMENT_WINDOW_MINUTES = 60; +export const DEFAULT_CLOSE_DAYS = 5; + +// --- prose measurement ------------------------------------------------------- + +export function stripNonProse(markdown) { + return String(markdown || '') + .replace(/```[\s\S]*?(?:```|$)/g, ' ') + .replace(/~~~[\s\S]*?(?:~~~|$)/g, ' ') + .replace(//gi, ' ') + .replace(//g, ' '); +} + +export function proseWordCount(markdown) { + return stripNonProse(markdown).split(/\s+/).filter(Boolean).length; +} + +// --- template parsing -------------------------------------------------------- + +export function normalizeHeading(text) { + return String(text || '') + .trim() + .toLowerCase() + .replace(/[?:!.]+$/, '') + .replace(/\s+/g, ' '); +} + +function extractHeadings(markdown) { + const headings = []; + for (const line of String(markdown || '').split(/\r?\n/)) { + const match = line.match(/^#{2,3}\s+(.+)$/); + if (match) headings.push(normalizeHeading(match[1])); + } + return headings; +} + +function stripFrontmatter(markdown) { + const match = String(markdown || '').match(/^---\r?\n[\s\S]*?\r?\n---\r?\n/); + return match ? markdown.slice(match[0].length) : markdown; +} + +export function parseTemplate(fileName, raw) { + const body = stripFrontmatter(raw); + const nameMatch = raw.match(/^name:\s*(.+)$/m); + const headings = extractHeadings(body); + const scaffoldLines = new Set( + body.split(/\r?\n/).map((line) => line.trim()).filter(Boolean), + ); + return { + file: fileName, + name: nameMatch ? nameMatch[1].trim() : fileName, + headings, + requiredSections: headings.filter((heading) => !OPTIONAL_SECTIONS.has(heading)), + scaffoldLines, + }; +} + +export function loadTemplates(dir) { + return readdirSync(dir) + .filter((file) => file.endsWith('.md')) + .sort() + .map((file) => parseTemplate(file, readFileSync(join(dir, file), 'utf-8'))); +} + +// --- issue evaluation -------------------------------------------------------- + +// Only headings that name a template section start a new section; authors use +// other headings (sub-scenarios, log labels) as content inside a section. +function splitSections(body, knownHeadings) { + const sections = new Map(); + let current = null; + for (const line of String(body || '').split(/\r?\n/)) { + const match = line.match(/^#{2,3}\s+(.+)$/); + if (match && knownHeadings.has(normalizeHeading(match[1]))) { + current = normalizeHeading(match[1]); + if (!sections.has(current)) sections.set(current, []); + continue; + } + if (current) sections.get(current).push(line); + } + return sections; +} + +// A section counts as filled when it contains at least one line the author +// wrote: non-empty, not an HTML comment, and not copied verbatim from the +// template scaffolding (numbered placeholders, field labels, unchecked boxes). +function sectionIsFilled(lines, scaffoldLines) { + const withoutComments = stripNonProse(lines.join('\n')); + for (const rawLine of withoutComments.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + if (scaffoldLines.has(line)) continue; + return true; + } + return false; +} + +export function evaluateIssue(issue, options = {}) { + const templates = options.templates || []; + const maxProseWords = Number.isFinite(options.maxProseWords) + ? options.maxProseWords + : DEFAULT_MAX_PROSE_WORDS; + const closeDays = Number.isFinite(options.closeDays) ? options.closeDays : DEFAULT_CLOSE_DAYS; + const maintainers = loginSet(options.maintainers || DEFAULT_MAINTAINERS); + const trustedMarkerAuthors = loginSet(options.trustedMarkerAuthors || DEFAULT_TRUSTED_MARKER_AUTHORS); + const repo = options.repo || ''; + + const author = normalizeLogin(issue.authorLogin); + const labels = new Set(issue.labels || []); + const comments = issue.comments || []; + + const base = { + number: issue.number, + title: issue.title, + author, + exempt: false, + verdict: 'pass', + reasons: [], + aiAssisted: false, + labelsToAdd: [], + labelsToRemove: [], + shouldComment: false, + comment: '', + shouldClose: false, + shouldReopen: false, + }; + + if (maintainers.has(author) || EXEMPT_ASSOCIATIONS.has(issue.authorAssociation)) { + return { ...base, exempt: true }; + } + + const body = String(issue.body || ''); + const bodyHeadings = new Set(extractHeadings(body)); + const anyTemplateHeading = templates.some( + (template) => template.headings.some((heading) => bodyHeadings.has(heading)), + ); + + const reasons = []; + let verdict = 'pass'; + + if (!anyTemplateHeading) { + verdict = 'reject'; + reasons.push('the body contains none of the issue template sections'); + } else { + const primary = templates + .map((template) => ({ + template, + matched: template.requiredSections.filter((section) => bodyHeadings.has(section)).length, + })) + .sort((a, b) => b.matched - a.matched)[0].template; + const knownHeadings = new Set(templates.flatMap((template) => template.headings)); + const sections = splitSections(body, knownHeadings); + for (const section of primary.requiredSections) { + if (!bodyHeadings.has(section)) { + reasons.push(`missing section "${section}" from the ${primary.name.toLowerCase()} template`); + } else if (!sectionIsFilled(sections.get(section) || [], primary.scaffoldLines)) { + reasons.push(`section "${section}" is empty or still contains only template placeholders`); + } + } + } + + const words = proseWordCount(body); + if (words > maxProseWords) { + reasons.push( + `the body has ${words} words of prose outside code blocks and
blocks; the limit is ${maxProseWords}. Trim to the essentials and move logs into a
block or a gist`, + ); + } + + if (verdict !== 'reject' && reasons.length > 0) verdict = 'needs-work'; + + const plan = { ...base, verdict, reasons }; + plan.aiAssisted = AI_DISCLOSURE.test(body); + if (plan.aiAssisted && !labels.has(AI_LABEL)) plan.labelsToAdd.push(AI_LABEL); + + if (verdict === 'pass') { + if (labels.has(GATE_LABEL)) plan.labelsToRemove.push(GATE_LABEL); + const closedByGate = hasMarker(comments, REJECT_MARKER, trustedMarkerAuthors) + || hasMarker(comments, CLOSE_MARKER, trustedMarkerAuthors); + plan.shouldReopen = issue.state === 'closed' && labels.has(GATE_LABEL) && closedByGate; + return plan; + } + + if (!labels.has(GATE_LABEL)) plan.labelsToAdd.push(GATE_LABEL); + + if (verdict === 'reject') { + plan.shouldClose = issue.state === 'open'; + plan.shouldComment = !hasMarker(comments, REJECT_MARKER, trustedMarkerAuthors); + plan.comment = rejectComment(repo); + return plan; + } + + plan.shouldComment = !hasMarker(comments, GATE_MARKER, trustedMarkerAuthors); + plan.comment = needsWorkComment(reasons, { repo, closeDays }); + return plan; +} + +export function evaluateSweep(issue, options = {}) { + const closeDays = Number.isFinite(options.closeDays) ? options.closeDays : DEFAULT_CLOSE_DAYS; + const trustedMarkerAuthors = loginSet(options.trustedMarkerAuthors || DEFAULT_TRUSTED_MARKER_AUTHORS); + const now = toDate(options.now || new Date()); + + const plan = evaluateIssue(issue, options); + if (plan.verdict === 'pass' || plan.verdict === 'reject') { + return { ...plan, shouldSweepClose: false, sweepComment: '' }; + } + + const warnedAt = latestMarkerAt(issue.comments || [], GATE_MARKER, trustedMarkerAuthors); + const failingDays = warnedAt + ? Math.floor((now.getTime() - warnedAt.getTime()) / DAY_MS) + : 0; + const shouldSweepClose = issue.state === 'open' && Boolean(warnedAt) && failingDays >= closeDays; + return { + ...plan, + shouldComment: plan.shouldComment && !shouldSweepClose, + shouldSweepClose, + sweepComment: shouldSweepClose ? sweepCloseComment(failingDays) : '', + }; +} + +// --- comment evaluation ------------------------------------------------------ + +export function evaluateComment(input, options = {}) { + const maxProseWords = Number.isFinite(options.maxProseWords) + ? options.maxProseWords + : DEFAULT_COMMENT_MAX_PROSE_WORDS; + const windowMinutes = Number.isFinite(options.windowMinutes) + ? options.windowMinutes + : DEFAULT_COMMENT_WINDOW_MINUTES; + const maintainers = loginSet(options.maintainers || DEFAULT_MAINTAINERS); + + const commentAuthor = normalizeLogin(input.commentAuthorLogin); + const issueAuthor = normalizeLogin(input.issueAuthorLogin); + const result = { shouldMinimize: false, reasons: [] }; + + if (maintainers.has(commentAuthor)) return result; + if (EXEMPT_ASSOCIATIONS.has(input.commentAuthorAssociation)) return result; + if (commentAuthor !== issueAuthor) return result; + + const minutesSinceIssue = (toDate(input.commentCreatedAt).getTime() + - toDate(input.issueCreatedAt).getTime()) / MINUTE_MS; + if (!(minutesSinceIssue >= 0 && minutesSinceIssue <= windowMinutes)) return result; + + const words = proseWordCount(input.commentBody); + if (words <= maxProseWords) return result; + + result.shouldMinimize = true; + result.reasons.push( + `self-reply with ${words} words of prose within ${Math.round(minutesSinceIssue)} minutes of opening the issue; the limit is ${maxProseWords}. This is the context-dump pattern the gate folds away`, + ); + return result; +} + +// --- comment copy -------------------------------------------------------------- + +function templatesUrl(repo) { + return repo + ? `https://github.com/${repo}/tree/main/.github/ISSUE_TEMPLATE` + : '.github/ISSUE_TEMPLATE'; +} + +export function needsWorkComment(reasons, { repo = '', closeDays = DEFAULT_CLOSE_DAYS } = {}) { + return [ + GATE_MARKER, + 'Thanks for filing this. It does not pass the issue template checks yet:', + '', + ...reasons.map((reason) => `- ${reason}`), + '', + `Please edit the issue body itself (not a new comment) to follow one of the [issue templates](${templatesUrl(repo)}). The checks run again on every edit and clear the label once they pass. Issues that still fail after ${closeDays} days are closed automatically.`, + '', + 'If AI helped write this issue, include the line `AI-assisted: yes` in the body.', + ].join('\n'); +} + +export function rejectComment(repo = '') { + return [ + REJECT_MARKER, + 'Closing this because the body does not use either issue template: none of the template sections were found.', + '', + `To continue, edit the issue body to follow one of the [issue templates](${templatesUrl(repo)}). The checks run again on every edit, and this issue is reopened automatically once they pass.`, + '', + 'If AI helped write this issue, include the line `AI-assisted: yes` in the body.', + ].join('\n'); +} + +export function sweepCloseComment(failingDays) { + return [ + CLOSE_MARKER, + `Closing this because the issue template checks have been failing for ${failingDays} days.`, + '', + 'Edit the issue body to pass the checks and it is reopened automatically on the next edit.', + ].join('\n'); +} + +// --- driver ------------------------------------------------------------------ + +export async function main(argv = process.argv.slice(2)) { + const options = parseArgs(argv); + const repo = options.repo || process.env.GITHUB_REPOSITORY; + if (!repo || !repo.includes('/')) { + throw new Error('Missing repository. Pass --repo owner/name or set GITHUB_REPOSITORY.'); + } + options.repo = repo; + options.templates = loadTemplates(options.templatesDir); + + if (options.apply && options.ensureLabels) ensureLabels(repo); + + if (options.commentId) { + runCommentGate(repo, options); + return; + } + + if (options.sweep) { + runSweep(repo, options); + return; + } + + if (options.issue) { + const issue = fetchIssue(repo, options.issue); + if (!issue) throw new Error(`Issue #${options.issue} not found.`); + const plan = evaluateIssue(issue, options); + printPlan(plan, options); + if (options.apply) applyIssuePlan(repo, plan); + return; + } + + throw new Error('Nothing to do. Pass --issue N, --issue N --comment-id ID, or --sweep.'); +} + +function runCommentGate(repo, options) { + const comment = fetchComment(repo, options.commentId); + const issue = fetchIssue(repo, options.issue); + if (!comment || !issue) throw new Error('Comment or issue not found.'); + + const result = evaluateComment({ + commentBody: comment.body, + commentAuthorLogin: comment.authorLogin, + commentAuthorAssociation: comment.authorAssociation, + commentCreatedAt: comment.createdAt, + issueAuthorLogin: issue.authorLogin, + issueCreatedAt: issue.createdAt, + }, { + maxProseWords: options.commentMaxProseWords, + windowMinutes: options.windowMinutes, + maintainers: options.maintainers, + }); + + if (!result.shouldMinimize) { + console.log(`comment ${options.commentId} on #${options.issue}: pass`); + return; + } + + console.log(`${options.apply ? 'apply' : 'dry-run'} comment ${options.commentId} on #${options.issue}: minimize (${result.reasons.join('; ')})`); + if (options.apply) minimizeComment(comment.nodeId); +} + +function runSweep(repo, options) { + const issues = fetchGateLabeledIssues(repo); + for (const issue of issues) { + issue.comments = fetchIssueComments(repo, issue.number); + const plan = evaluateSweep(issue, options); + printPlan(plan, options); + if (!options.apply) continue; + applyIssuePlan(repo, plan); + if (plan.shouldSweepClose) { + postComment(repo, plan.number, plan.sweepComment); + closeIssue(repo, plan.number); + } + } + console.log(`${options.apply ? 'Applied' : 'Dry run'} issue-gate sweep for ${issues.length} labeled issue(s).`); +} + +function applyIssuePlan(repo, plan) { + if (plan.exempt) return; + if (plan.shouldReopen) reopenIssue(repo, plan.number); + for (const label of plan.labelsToRemove) removeLabel(repo, plan.number, label); + if (plan.labelsToAdd.length > 0) addLabels(repo, plan.number, plan.labelsToAdd); + if (plan.shouldComment) postComment(repo, plan.number, plan.comment); + if (plan.shouldClose) closeIssue(repo, plan.number); +} + +function printPlan(plan, { apply }) { + const changes = [ + plan.exempt ? 'exempt' : `verdict=${plan.verdict}`, + plan.labelsToAdd.length ? `add=${plan.labelsToAdd.join(',')}` : '', + plan.labelsToRemove.length ? `remove=${plan.labelsToRemove.join(',')}` : '', + plan.shouldComment ? 'comment' : '', + plan.shouldClose ? 'close' : '', + plan.shouldReopen ? 'reopen' : '', + plan.shouldSweepClose ? 'sweep-close' : '', + ].filter(Boolean); + console.log(`${apply ? 'apply' : 'dry-run'} #${plan.number} ${plan.title || ''}: ${changes.join(' ')}`); + for (const reason of plan.reasons) console.log(` - ${reason}`); +} + +export function parseArgs(argv) { + const options = { + apply: false, + ensureLabels: true, + sweep: false, + issue: null, + commentId: null, + templatesDir: '.github/ISSUE_TEMPLATE', + maxProseWords: DEFAULT_MAX_PROSE_WORDS, + commentMaxProseWords: DEFAULT_COMMENT_MAX_PROSE_WORDS, + windowMinutes: DEFAULT_COMMENT_WINDOW_MINUTES, + closeDays: DEFAULT_CLOSE_DAYS, + maintainers: DEFAULT_MAINTAINERS, + now: new Date(), + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--apply') options.apply = true; + else if (arg === '--dry-run') options.apply = false; + else if (arg === '--sweep') options.sweep = true; + else if (arg === '--no-label-ensure') options.ensureLabels = false; + else if (arg === '--repo') options.repo = requireValue(argv, ++i, arg); + else if (arg === '--issue') options.issue = Number(requireValue(argv, ++i, arg)); + else if (arg === '--comment-id') options.commentId = requireValue(argv, ++i, arg); + else if (arg === '--templates-dir') options.templatesDir = requireValue(argv, ++i, arg); + else if (arg === '--max-words') options.maxProseWords = Number(requireValue(argv, ++i, arg)); + else if (arg === '--comment-max-words') options.commentMaxProseWords = Number(requireValue(argv, ++i, arg)); + else if (arg === '--comment-window-minutes') options.windowMinutes = Number(requireValue(argv, ++i, arg)); + else if (arg === '--close-days') options.closeDays = Number(requireValue(argv, ++i, arg)); + else if (arg === '--maintainers') options.maintainers = splitList(requireValue(argv, ++i, arg)); + else if (arg === '--now') options.now = new Date(requireValue(argv, ++i, arg)); + else if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + + if (options.issue !== null && !Number.isInteger(options.issue)) { + throw new Error('--issue must be an integer.'); + } + if (!Number.isFinite(options.maxProseWords) || options.maxProseWords <= 0) { + throw new Error('--max-words must be a positive number.'); + } + if (!Number.isFinite(options.closeDays) || options.closeDays < 1) { + throw new Error('--close-days must be at least 1.'); + } + if (Number.isNaN(options.now.getTime())) throw new Error('--now must be a valid date.'); + + return options; +} + +// --- gh plumbing --------------------------------------------------------------- + +function normalizeIssue(raw) { + return { + number: raw.number, + title: raw.title, + body: raw.body || '', + state: raw.state, + createdAt: raw.created_at, + updatedAt: raw.updated_at, + authorLogin: raw.user?.login || '', + authorAssociation: raw.author_association || '', + labels: (raw.labels || []).map((label) => (typeof label === 'string' ? label : label.name)), + isPullRequest: Boolean(raw.pull_request), + comments: [], + }; +} + +function fetchIssue(repo, number) { + const raw = runGhJson(['api', `repos/${repo}/issues/${number}`]); + if (!raw || !raw.number) return null; + const issue = normalizeIssue(raw); + issue.comments = fetchIssueComments(repo, number); + return issue; +} + +function fetchIssueComments(repo, number) { + const pages = runGhJson([ + 'api', + '--paginate', + '--slurp', + `repos/${repo}/issues/${number}/comments?per_page=100`, + ]); + const flat = Array.isArray(pages) ? pages.flat() : []; + return flat.map((comment) => ({ + authorLogin: comment.user?.login || '', + createdAt: comment.created_at, + body: comment.body || '', + })); +} + +function fetchComment(repo, commentId) { + const raw = runGhJson(['api', `repos/${repo}/issues/comments/${commentId}`]); + if (!raw || !raw.id) return null; + return { + id: raw.id, + nodeId: raw.node_id, + body: raw.body || '', + createdAt: raw.created_at, + authorLogin: raw.user?.login || '', + authorAssociation: raw.author_association || '', + }; +} + +function fetchGateLabeledIssues(repo) { + const pages = runGhJson([ + 'api', + '--paginate', + '--slurp', + `repos/${repo}/issues?state=open&labels=${encodeURIComponent(GATE_LABEL)}&per_page=100`, + ]); + const flat = Array.isArray(pages) ? pages.flat() : []; + return flat.map(normalizeIssue).filter((issue) => !issue.isPullRequest); +} + +function ensureLabels(repo) { + for (const label of LABEL_DEFS) { + const encoded = encodeURIComponent(label.name); + const get = runGh(['api', `repos/${repo}/labels/${encoded}`], { allowFailure: true, quiet: true }); + if (get.status === 0) continue; + runGh([ + 'api', + '-X', + 'POST', + `repos/${repo}/labels`, + '-f', + `name=${label.name}`, + '-f', + `color=${label.color}`, + '-f', + `description=${label.description}`, + ], { allowFailure: true }); + } +} + +function addLabels(repo, number, labels) { + const args = ['api', '-X', 'POST', `repos/${repo}/issues/${number}/labels`]; + for (const label of labels) args.push('-f', `labels[]=${label}`); + runGh(args); +} + +function removeLabel(repo, number, label) { + runGh([ + 'api', + '-X', + 'DELETE', + `repos/${repo}/issues/${number}/labels/${encodeURIComponent(label)}`, + ], { allowFailure: true }); +} + +function postComment(repo, number, body) { + runGh(['api', '-X', 'POST', `repos/${repo}/issues/${number}/comments`, '-f', `body=${body}`]); +} + +function closeIssue(repo, number) { + runGh([ + 'api', + '-X', + 'PATCH', + `repos/${repo}/issues/${number}`, + '-f', + 'state=closed', + '-f', + 'state_reason=not_planned', + ]); +} + +function reopenIssue(repo, number) { + runGh(['api', '-X', 'PATCH', `repos/${repo}/issues/${number}`, '-f', 'state=open']); +} + +function minimizeComment(nodeId) { + runGh([ + 'api', + 'graphql', + '-f', + 'query=mutation($id: ID!) { minimizeComment(input: { subjectId: $id, classifier: OFF_TOPIC }) { minimizedComment { isMinimized } } }', + '-f', + `id=${nodeId}`, + ]); +} + +// --- shared helpers ------------------------------------------------------------ + +function hasMarker(comments = [], marker, trustedAuthors) { + return comments.some((comment) => isTrustedMarkerComment(comment, marker, trustedAuthors)); +} + +function latestMarkerAt(comments = [], marker, trustedAuthors) { + let latest = null; + for (const comment of comments) { + if (!isTrustedMarkerComment(comment, marker, trustedAuthors)) continue; + const date = toDate(comment.createdAt); + if (Number.isNaN(date.getTime())) continue; + if (!latest || date > latest) latest = date; + } + return latest; +} + +function isTrustedMarkerComment(comment, marker, trustedAuthors) { + if (typeof comment?.body !== 'string' || !comment.body.includes(marker)) return false; + return trustedAuthors.has(normalizeLogin(comment.authorLogin)); +} + +function toDate(value) { + return value instanceof Date ? value : new Date(value); +} + +function splitList(value) { + return String(value || '') + .split(',') + .map((item) => item.trim()) + .filter(Boolean); +} + +function loginSet(logins) { + return new Set(logins.map(normalizeLogin).filter(Boolean)); +} + +function normalizeLogin(login) { + return String(login || '').toLowerCase(); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (!value || value.startsWith('--')) throw new Error(`${flag} requires a value.`); + return value; +} + +function runGhJson(args) { + const result = runGh(args, { quiet: true }); + try { + return JSON.parse(result.stdout || '{}'); + } catch (err) { + throw new Error(`Failed to parse gh JSON output: ${err.message}`); + } +} + +function runGh(args, options = {}) { + const result = spawnSync('gh', args, { + encoding: 'utf-8', + env: process.env, + }); + if (!options.quiet && result.stdout) process.stdout.write(result.stdout); + if (!options.quiet && result.stderr) process.stderr.write(result.stderr); + if (result.error) throw result.error; + if (result.status !== 0 && !options.allowFailure) { + throw new Error(`gh ${args.join(' ')} failed with exit ${result.status}: ${result.stderr || result.stdout}`); + } + return result; +} + +function printHelp() { + console.log(`Usage: node scripts/github/issue-gate.mjs [--repo owner/name] [--apply] + +Default mode is a dry run. + +Modes: + --issue N evaluate one issue body (workflow: issues opened/edited) + --issue N --comment-id ID evaluate one comment (workflow: issue_comment created) + --sweep re-check all '${GATE_LABEL}' issues and close long-failing ones + +Options: + --apply mutate labels, comments, and issue state + --dry-run print planned changes without mutating GitHub + --repo owner/name repository (defaults to GITHUB_REPOSITORY) + --templates-dir path issue template dir (default: .github/ISSUE_TEMPLATE) + --max-words n prose word budget for issue bodies (default: ${DEFAULT_MAX_PROSE_WORDS}) + --comment-max-words n prose word budget for early self-replies (default: ${DEFAULT_COMMENT_MAX_PROSE_WORDS}) + --comment-window-minutes n self-reply window after issue creation (default: ${DEFAULT_COMMENT_WINDOW_MINUTES}) + --close-days n days a warned issue may keep failing before close (default: ${DEFAULT_CLOSE_DAYS}) + --maintainers a,b logins exempt from all gates + --no-label-ensure skip creating gate labels +`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((err) => { + console.error(err.message); + process.exit(1); + }); +} diff --git a/scripts/test-suites.mjs b/scripts/test-suites.mjs index 9bb675c1f..61f51a9e3 100644 --- a/scripts/test-suites.mjs +++ b/scripts/test-suites.mjs @@ -59,6 +59,7 @@ export const SUITES = { 'tests/context-signals.test.mjs', 'tests/critique-storage.test.mjs', 'tests/design-parser.test.mjs', + 'tests/github-issue-gate.test.mjs', 'tests/github-sheriff.test.mjs', 'tests/hook-build.test.mjs', 'tests/hook.test.mjs', diff --git a/tests/github-issue-gate.test.mjs b/tests/github-issue-gate.test.mjs new file mode 100644 index 000000000..e252b4a68 --- /dev/null +++ b/tests/github-issue-gate.test.mjs @@ -0,0 +1,383 @@ +import { describe, it, before } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +import { + AI_LABEL, + CLOSE_MARKER, + GATE_LABEL, + GATE_MARKER, + REJECT_MARKER, + evaluateComment, + evaluateIssue, + evaluateSweep, + loadTemplates, + parseArgs, + proseWordCount, +} from '../scripts/github/issue-gate.mjs'; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const NOW = '2026-08-05T12:00:00Z'; + +let templates; +before(() => { + templates = loadTemplates(join(REPO_ROOT, '.github', 'ISSUE_TEMPLATE')); +}); + +const FILLED_BUG_BODY = `## What happened? + +Running the detect command against a directory crashes with a TypeError. + +## Steps to reproduce + +1. Run \`npx impeccable detect src/\` +2. Watch it crash + +## Expected behavior + +A findings report. + +## Provider & environment + +- **Provider** (Cursor / Claude Code / Gemini CLI / Codex / Copilot / Kiro / OpenCode): Cursor +- **Provider version**: 2.4.0 +- **OS**: macOS 15 + +## Additional context + +None. +`; + +function issue(overrides = {}) { + return { + number: 42, + title: '[Bug] detect crashes on directories', + body: FILLED_BUG_BODY, + state: 'open', + createdAt: '2026-08-01T00:00:00Z', + authorLogin: 'outside-reporter', + authorAssociation: 'NONE', + labels: [], + comments: [], + ...overrides, + }; +} + +function comment(authorLogin, createdAt, body) { + return { authorLogin, createdAt, body }; +} + +function words(count) { + return Array.from({ length: count }, (_, i) => `word${i}`).join(' '); +} + +describe('issue gate templates', () => { + it('derives required sections from the real templates', () => { + const bug = templates.find((template) => template.file === 'bug_report.md'); + const feature = templates.find((template) => template.file === 'feature_request.md'); + assert.deepEqual(bug.requiredSections, ['what happened', 'steps to reproduce', 'provider & environment']); + assert.deepEqual(feature.requiredSections, ['what problem does this solve', 'proposed solution']); + }); +}); + +describe('issue body gate', () => { + it('passes a properly filled bug report', () => { + const plan = evaluateIssue(issue(), { templates }); + assert.equal(plan.verdict, 'pass'); + assert.deepEqual(plan.reasons, []); + assert.equal(plan.shouldComment, false); + assert.equal(plan.shouldClose, false); + }); + + it('passes a properly filled feature request', () => { + const plan = evaluateIssue(issue({ + body: [ + '## What problem does this solve?', + '', + 'The audit command has no way to scope to one route.', + '', + '## Proposed solution', + '', + 'Accept a path argument that narrows the audit to matching files.', + ].join('\n'), + }), { templates }); + assert.equal(plan.verdict, 'pass'); + }); + + it('rejects and closes a body with no template sections at all', () => { + const plan = evaluateIssue(issue({ + body: 'hey the tool is broken please fix asap. also add dark mode.', + }), { templates }); + assert.equal(plan.verdict, 'reject'); + assert.deepEqual(plan.labelsToAdd, [GATE_LABEL]); + assert.equal(plan.shouldClose, true); + assert.equal(plan.shouldComment, true); + assert.match(plan.comment, /none of the template sections/); + assert.ok(plan.comment.includes(REJECT_MARKER)); + }); + + it('flags untouched template placeholders as empty sections', () => { + const bugTemplate = templates.find((template) => template.file === 'bug_report.md'); + const untouched = [...bugTemplate.scaffoldLines].length > 0; + assert.ok(untouched); + const plan = evaluateIssue(issue({ + body: [ + '## What happened?', + '', + '', + '', + '## Steps to reproduce', + '', + '1. ', + '2. ', + '3. ', + '', + '## Provider & environment', + '', + '- **Provider** (Cursor / Claude Code / Gemini CLI / Codex / Copilot / Kiro / OpenCode):', + '- **Provider version**: ', + '- **OS**: ', + ].join('\n'), + }), { templates }); + assert.equal(plan.verdict, 'needs-work'); + assert.equal(plan.reasons.length, 3); + assert.match(plan.reasons[0], /section "what happened" is empty/); + assert.equal(plan.shouldClose, false); + assert.deepEqual(plan.labelsToAdd, [GATE_LABEL]); + }); + + it('names a missing required section', () => { + const plan = evaluateIssue(issue({ + body: [ + '## What happened?', + '', + 'The build fails.', + '', + '## Provider & environment', + '', + '- **OS**: macOS 15 and provider Cursor 2.4.0', + ].join('\n'), + }), { templates }); + assert.equal(plan.verdict, 'needs-work'); + assert.deepEqual(plan.reasons, ['missing section "steps to reproduce" from the bug report template']); + }); + + it('flags an oversized prose body even when the structure passes', () => { + const plan = evaluateIssue(issue({ + body: FILLED_BUG_BODY + '\n' + words(700), + }), { templates }); + assert.equal(plan.verdict, 'needs-work'); + assert.equal(plan.reasons.length, 1); + assert.match(plan.reasons[0], /the limit is 600/); + }); + + it('does not count fenced code blocks or details blocks against the budget', () => { + const plan = evaluateIssue(issue({ + body: `${FILLED_BUG_BODY}\n\`\`\`\n${words(700)}\n\`\`\`\n
log\n${words(700)}\n
\n`, + }), { templates }); + assert.equal(plan.verdict, 'pass'); + }); + + it('treats author sub-headings as section content, not section boundaries', () => { + const plan = evaluateIssue(issue({ + body: [ + '## What happened?', + '', + 'Install fails for the grok provider.', + '', + '## Steps to reproduce', + '', + '### A. Clean project', + '', + '1. Run `npx impeccable install --providers=grok`', + '', + '### B. Another provider already installed', + '', + '1. Seed a Claude install first, then run the same command.', + '', + '## Provider & environment', + '', + '- **OS**: macOS 15, CLI 4.0.4', + ].join('\n'), + }), { templates }); + assert.equal(plan.verdict, 'pass'); + }); + + it('labels disclosed AI assistance without failing the issue', () => { + const plan = evaluateIssue(issue({ + body: `AI-assisted: yes\n\n${FILLED_BUG_BODY}`, + }), { templates }); + assert.equal(plan.verdict, 'pass'); + assert.equal(plan.aiAssisted, true); + assert.deepEqual(plan.labelsToAdd, [AI_LABEL]); + }); + + it('exempts maintainers and repo members', () => { + const maintainer = evaluateIssue(issue({ authorLogin: 'pbakaus', body: 'quick note' }), { templates }); + assert.equal(maintainer.exempt, true); + const member = evaluateIssue(issue({ authorAssociation: 'COLLABORATOR', body: 'quick note' }), { templates }); + assert.equal(member.exempt, true); + }); + + it('comments only once per issue', () => { + const plan = evaluateIssue(issue({ + body: 'no structure here', + labels: [GATE_LABEL], + comments: [comment('github-actions[bot]', '2026-08-01T01:00:00Z', REJECT_MARKER)], + state: 'closed', + }), { templates }); + assert.equal(plan.verdict, 'reject'); + assert.equal(plan.shouldComment, false); + assert.equal(plan.shouldClose, false); + assert.deepEqual(plan.labelsToAdd, []); + }); + + it('ignores gate markers posted by untrusted commenters', () => { + const plan = evaluateIssue(issue({ + body: 'no structure here', + comments: [comment('drive-by', '2026-08-01T01:00:00Z', REJECT_MARKER)], + }), { templates }); + assert.equal(plan.shouldComment, true); + }); + + it('clears the label and reopens a gate-closed issue once it passes', () => { + const plan = evaluateIssue(issue({ + state: 'closed', + labels: [GATE_LABEL], + comments: [comment('github-actions[bot]', '2026-08-01T01:00:00Z', REJECT_MARKER)], + }), { templates }); + assert.equal(plan.verdict, 'pass'); + assert.deepEqual(plan.labelsToRemove, [GATE_LABEL]); + assert.equal(plan.shouldReopen, true); + }); + + it('does not reopen issues the gate did not close', () => { + const plan = evaluateIssue(issue({ + state: 'closed', + labels: [GATE_LABEL], + }), { templates }); + assert.equal(plan.shouldReopen, false); + }); +}); + +describe('sweep', () => { + it('closes an issue still failing five days after the warning', () => { + const plan = evaluateSweep(issue({ + body: 'still no structure', + labels: [GATE_LABEL], + comments: [comment('github-actions[bot]', '2026-07-30T12:00:00Z', GATE_MARKER)], + }), { templates, now: NOW }); + assert.equal(plan.shouldSweepClose, false); + + const oversized = evaluateSweep(issue({ + body: FILLED_BUG_BODY + '\n' + words(700), + labels: [GATE_LABEL], + comments: [comment('github-actions[bot]', '2026-07-30T12:00:00Z', GATE_MARKER)], + }), { templates, now: NOW }); + assert.equal(oversized.shouldSweepClose, true); + assert.match(oversized.sweepComment, /failing for 6 days/); + assert.ok(oversized.sweepComment.includes(CLOSE_MARKER)); + }); + + it('does not close before the warning has aged past close-days', () => { + const plan = evaluateSweep(issue({ + body: FILLED_BUG_BODY + '\n' + words(700), + labels: [GATE_LABEL], + comments: [comment('github-actions[bot]', '2026-08-03T12:00:00Z', GATE_MARKER)], + }), { templates, now: NOW }); + assert.equal(plan.shouldSweepClose, false); + }); + + it('lets a fixed issue pass instead of closing it', () => { + const plan = evaluateSweep(issue({ + labels: [GATE_LABEL], + comments: [comment('github-actions[bot]', '2026-07-25T12:00:00Z', GATE_MARKER)], + }), { templates, now: NOW }); + assert.equal(plan.verdict, 'pass'); + assert.equal(plan.shouldSweepClose, false); + assert.deepEqual(plan.labelsToRemove, [GATE_LABEL]); + }); +}); + +describe('comment gate', () => { + const base = { + commentAuthorLogin: 'outside-reporter', + commentAuthorAssociation: 'NONE', + issueAuthorLogin: 'outside-reporter', + issueCreatedAt: '2026-08-05T10:00:00Z', + commentCreatedAt: '2026-08-05T10:10:00Z', + }; + + it('minimizes an oversized early self-reply', () => { + const result = evaluateComment({ ...base, commentBody: words(400) }); + assert.equal(result.shouldMinimize, true); + assert.match(result.reasons[0], /400 words of prose/); + }); + + it('leaves short self-replies alone', () => { + const result = evaluateComment({ ...base, commentBody: 'Forgot to say: version 2.4.0.' }); + assert.equal(result.shouldMinimize, false); + }); + + it('leaves long comments from other participants alone', () => { + const result = evaluateComment({ + ...base, + commentAuthorLogin: 'helpful-stranger', + commentBody: words(400), + }); + assert.equal(result.shouldMinimize, false); + }); + + it('leaves late self-replies alone', () => { + const result = evaluateComment({ + ...base, + commentCreatedAt: '2026-08-05T12:30:00Z', + commentBody: words(400), + }); + assert.equal(result.shouldMinimize, false); + }); + + it('does not count pasted logs against the comment budget', () => { + const result = evaluateComment({ + ...base, + commentBody: `Full log below.\n\`\`\`\n${words(900)}\n\`\`\``, + }); + assert.equal(result.shouldMinimize, false); + }); + + it('exempts maintainers and members', () => { + const maintainer = evaluateComment({ + ...base, + commentAuthorLogin: 'abdulwahabone', + issueAuthorLogin: 'abdulwahabone', + commentBody: words(400), + }); + assert.equal(maintainer.shouldMinimize, false); + const member = evaluateComment({ + ...base, + commentAuthorAssociation: 'MEMBER', + commentBody: words(400), + }); + assert.equal(member.shouldMinimize, false); + }); +}); + +describe('helpers', () => { + it('counts prose words outside code and details blocks', () => { + assert.equal(proseWordCount('one two three'), 3); + assert.equal(proseWordCount('one\n```\nskip these words\n```\ntwo'), 2); + assert.equal(proseWordCount('one
skip skip
two'), 2); + assert.equal(proseWordCount(' one'), 1); + }); + + it('parses workflow arguments', () => { + const options = parseArgs(['--apply', '--repo', 'pbakaus/impeccable', '--issue', '42', '--comment-id', '99']); + assert.equal(options.apply, true); + assert.equal(options.issue, 42); + assert.equal(options.commentId, '99'); + + assert.throws(() => parseArgs(['--max-words', '0']), /positive/); + assert.throws(() => parseArgs(['--unknown']), /Unknown argument/); + }); +});