Improve PR sheriff queue flow

Make policy and merge-conflict blockers age as contributor work, keep maintainer-policy decisions out of ready state, auto-close regular contributors unless explicitly exempted, and mark new or reopened issues for triage.

AI-assisted change: implemented and validated by Codex under maintainer direction.
This commit is contained in:
Paul Bakaus
2026-08-31 15:47:32 -07:00
parent 33367e2297
commit ee1442d7af
3 changed files with 130 additions and 8 deletions
+22 -1
View File
@@ -1,6 +1,8 @@
name: PR Sheriff
on:
issues:
types: [opened, reopened]
schedule:
# Daily UTC afternoon pass. The script uses an aggressive 7/14 day window:
# warn contributor-blocked PRs after 7 days open, close after 14.
@@ -24,7 +26,25 @@ concurrency:
cancel-in-progress: false
jobs:
issue-inbox:
if: github.event_name == 'issues'
runs-on: ubuntu-latest
steps:
- name: Mark issue for triage
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
ISSUE_NUMBER: ${{ github.event.issue.number }}
run: |
gh label create "needs triage" \
--repo "$GH_REPO" \
--color "d4c5f9" \
--description "New or reopened issue awaiting maintainer triage" \
--force
gh issue edit "$ISSUE_NUMBER" --repo "$GH_REPO" --add-label "needs triage"
sheriff:
if: github.event_name != 'issues'
runs-on: ubuntu-latest
steps:
- name: Checkout repository
@@ -49,4 +69,5 @@ jobs:
--warning-days 7 \
--close-days 14 \
--maintainers "pbakaus" \
--regular-contributors "pbakaus,abdulwahabone"
--regular-contributors "pbakaus,abdulwahabone" \
--auto-close-regulars
+37 -2
View File
@@ -5,6 +5,7 @@ import { pathToFileURL } from 'node:url';
const DAY_MS = 24 * 60 * 60 * 1000;
export const LABEL_DEFS = [
{ name: 'needs triage', color: 'd4c5f9', description: 'New or reopened issue awaiting maintainer triage' },
{ name: 'waiting on contributor', color: 'fbca04', description: 'Waiting for the PR author to respond or make changes' },
{ name: 'needs maintainer review', color: '5319e7', description: 'Ready for a maintainer to review or decide' },
{ name: 'ready to merge', color: '0e8a16', description: 'Passing, resolved, and ready for a maintainer merge decision' },
@@ -42,6 +43,11 @@ const DEFAULT_TRUSTED_MARKER_AUTHORS = ['github-actions', 'github-actions[bot]']
const REVIEW_BLOCKING_STATES = new Set(['CHANGES_REQUESTED']);
const FAILING_STATUS_STATES = new Set(['ERROR', 'FAILURE']);
const CONTRIBUTOR_POLICY_LABELS = new Set([
'policy: needs issue',
'policy: needs ai disclosure',
'policy: generated output',
]);
const SHERIFF_WAIT_COMMAND = /^\/sheriff\s+wait\s*$/i;
const PR_QUERY = `
@@ -144,6 +150,7 @@ export function evaluatePullRequest(pr, options = {}) {
const autoCloseRegulars = options.autoCloseRegulars === true;
const labels = new Set(pr.labels || []);
const policyBlockers = [...labels].filter(isBlockingPolicyLabel);
const author = normalizeLogin(pr.authorLogin || pr.author?.login || '');
const daysOpen = Math.floor((now.getTime() - toDate(pr.createdAt).getTime()) / DAY_MS);
const latestContributorCommitAt = latestCommitBelongsToAuthor(pr, author) ? pr.latestCommitAt : null;
@@ -173,7 +180,30 @@ export function evaluatePullRequest(pr, options = {}) {
}
if (pr.mergeable === 'CONFLICTING') {
addBlocker({ kind: 'merge-conflicts', label: 'blocked: merge conflicts', at: pr.updatedAt });
const firstSeenAt = latestLabelEventAt(
pr.labelEvents || [],
'blocked: merge conflicts',
'LabeledEvent',
) || pr.updatedAt || pr.latestCommitAt || pr.createdAt;
addContributorBlocker({
kind: 'merge-conflicts',
label: 'blocked: merge conflicts',
at: latestDate([firstSeenAt, latestContributorAt]),
});
}
for (const label of policyBlockers) {
const firstSeenAt = latestLabelEventAt(
pr.labelEvents || [],
label,
'LabeledEvent',
) || pr.updatedAt || pr.createdAt;
const blocker = {
kind: label,
at: latestDate([firstSeenAt, latestContributorAt]),
};
if (CONTRIBUTOR_POLICY_LABELS.has(label)) addContributorBlocker(blocker);
else addBlocker(blocker);
}
if (latestBlockingReviewAt && !isAfter(latestContributorAt, latestBlockingReviewAt)) {
@@ -216,6 +246,7 @@ export function evaluatePullRequest(pr, options = {}) {
&& !contributorActionRequired
&& unresolvedThreadCount === 0
&& pr.reviewDecision !== 'CHANGES_REQUESTED'
&& policyBlockers.length === 0
&& statusIsReady
&& mergeableIsReady;
@@ -373,7 +404,7 @@ export function staleWarningComment(pr, {
WARNING_MARKER,
`Thanks for the PR. Impeccable is moving quickly, and this PR is currently waiting on contributor action.`,
'',
`It has been waiting for contributor action for ${waitingDays} days. Please address the outstanding review feedback, draft state, or explicit maintainer wait request. PRs that are still waiting on contributor action after ${closeDays} days are closed automatically.`,
`It has been waiting for contributor action for ${waitingDays} days. Please address the outstanding review feedback, draft state, merge conflict, policy requirement, or explicit maintainer wait request. PRs that are still waiting on contributor action after ${closeDays} days are closed automatically.`,
'',
`If nothing changes, this PR may be closed on or after ${closeDate}. Happy to reopen when it is ready to continue.`,
].join('\n');
@@ -760,6 +791,10 @@ function normalizeLogin(login) {
return String(login || '').toLowerCase();
}
function isBlockingPolicyLabel(label) {
return String(label || '').startsWith('policy:') && label !== 'policy: approved';
}
function requireValue(argv, index, flag) {
const value = argv[index];
if (!value || value.startsWith('--')) throw new Error(`${flag} requires a value.`);
+71 -5
View File
@@ -360,12 +360,15 @@ describe('github sheriff', () => {
assert.equal(plan.shouldClose, false);
});
it('does not warn or close PRs blocked only by merge conflicts', () => {
it('treats merge conflicts as contributor action and ages from the first conflict label', () => {
const plan = evaluatePullRequest(pr({
createdAt: '2026-06-20T00:00:00Z',
updatedAt: '2026-07-07T00:00:00Z',
latestCommitAt: '2026-06-20T01:00:00Z',
mergeable: 'CONFLICTING',
labels: ['waiting on contributor', 'stale'],
labelEvents: [
labelEvent('LabeledEvent', 'blocked: merge conflicts', 'github-actions[bot]', '2026-06-21T00:00:00Z'),
labelEvent('LabeledEvent', 'waiting on contributor', 'github-actions[bot]', '2026-07-01T00:00:00Z'),
],
comments: [
@@ -373,11 +376,54 @@ describe('github sheriff', () => {
],
}), { now: NOW });
assert.equal(plan.contributorActionRequired, false);
assert.deepEqual(plan.labelsToAdd, ['blocked: merge conflicts', 'needs maintainer review']);
assert.deepEqual(plan.labelsToRemove, ['stale', 'waiting on contributor']);
assert.equal(plan.contributorActionRequired, true);
assert.deepEqual(plan.labelsToAdd, ['blocked: merge conflicts']);
assert.deepEqual(plan.labelsToRemove, []);
assert.equal(plan.shouldWarn, false);
assert.equal(plan.shouldClose, false);
assert.equal(plan.shouldClose, true);
});
it('blocks ready state and starts the stale clock for contributor-resolvable policy labels', () => {
const plan = evaluatePullRequest(pr({
createdAt: '2026-06-20T00:00:00Z',
latestCommitAt: '2026-06-20T01:00:00Z',
statusState: 'SUCCESS',
mergeable: 'MERGEABLE',
labels: ['ready to merge', 'policy: generated output'],
labelEvents: [
labelEvent('LabeledEvent', 'policy: generated output', 'pbakaus', '2026-06-21T00:00:00Z'),
],
}), { now: NOW });
assert.equal(plan.contributorActionRequired, true);
assert.equal(plan.readyToMerge, false);
assert.deepEqual(plan.labelsToAdd, ['stale', 'waiting on contributor']);
assert.deepEqual(plan.labelsToRemove, ['ready to merge']);
assert.equal(plan.shouldWarn, true);
});
it('keeps maintainer policy decisions out of ready state without blaming the contributor', () => {
const plan = evaluatePullRequest(pr({
statusState: 'SUCCESS',
mergeable: 'MERGEABLE',
labels: ['ready to merge', 'policy: needs approval'],
}), { now: NOW });
assert.equal(plan.contributorActionRequired, false);
assert.equal(plan.readyToMerge, false);
assert.deepEqual(plan.labelsToAdd, ['needs maintainer review']);
assert.deepEqual(plan.labelsToRemove, ['ready to merge']);
});
it('does not treat policy approval as a blocker', () => {
const plan = evaluatePullRequest(pr({
statusState: 'SUCCESS',
mergeable: 'MERGEABLE',
labels: ['policy: approved'],
}), { now: NOW });
assert.equal(plan.readyToMerge, true);
assert.deepEqual(plan.labelsToAdd, ['ready to merge']);
});
it('uses PR creation, not unrelated updates, for drafts opened as draft', () => {
@@ -634,6 +680,26 @@ describe('github sheriff', () => {
assert.equal(plan.shouldClose, false);
});
it('auto-closes regular contributors in the scheduled mode unless explicitly exempted', () => {
const source = {
authorLogin: 'abdulwahabone',
createdAt: '2026-06-20T00:00:00Z',
latestCommitAt: '2026-06-20T01:00:00Z',
comments: [
comment('pbakaus', '2026-06-21T00:00:00Z', '/sheriff wait'),
comment('github-actions[bot]', '2026-06-27T00:00:00Z', WARNING_MARKER),
],
};
const closePlan = evaluatePullRequest(pr(source), { now: NOW, autoCloseRegulars: true });
const exemptPlan = evaluatePullRequest(pr({ ...source, labels: ['do not close'] }), {
now: NOW,
autoCloseRegulars: true,
});
assert.equal(closePlan.shouldClose, true);
assert.equal(exemptPlan.shouldClose, false);
});
it('keeps stale comments idempotent', () => {
const plan = evaluatePullRequest(pr({
createdAt: '2026-06-20T00:00:00Z',