From 32eaa07c7e3bee6f27aaf18df841ebdfaa2d7ab2 Mon Sep 17 00:00:00 2001
From: Magnus Hedemark <magnus919@pm.me>
Date: Tue, 1 Sep 2026 21:30:15 -0400
Subject: [PATCH] test(validation): enforce catalog and checklist parity

Implement catalog source comparison and exact eval checklist parity with deterministic validation coverage.
---
 .../references/case-quality-checklist.json    | 798 ++++--------------
 scripts/check-catalog-set.py                  | 131 +++
 scripts/regenerate-case-checklist.py          |  46 +
 scripts/test_check_catalog_set.py             |  63 ++
 scripts/test_validate_case_checklist.py       |  35 +
 scripts/validate-case-checklist.py            |  53 ++
 6 files changed, 507 insertions(+), 619 deletions(-)
 create mode 100644 scripts/check-catalog-set.py
 create mode 100644 scripts/regenerate-case-checklist.py
 create mode 100644 scripts/test_check_catalog_set.py
 create mode 100644 scripts/test_validate_case_checklist.py
 create mode 100644 scripts/validate-case-checklist.py

diff --git a/lifecycle-evals/references/case-quality-checklist.json b/lifecycle-evals/references/case-quality-checklist.json
index 9e8eb61..a89d4de 100644
--- a/lifecycle-evals/references/case-quality-checklist.json
+++ b/lifecycle-evals/references/case-quality-checklist.json
@@ -4484,8 +4484,8 @@
     },
     {
       "skill": "legal-strategy",
-      "case_id": "legal-strategy-core-workflow",
-      "capability": "Use legal strategy to handle a realistic primary task",
+      "case_id": "jurisdictional-triage",
+      "capability": "Assess an EU and US launch of an AI profiling feature.",
       "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
@@ -4494,9 +4494,9 @@
     },
     {
       "skill": "legal-strategy",
-      "case_id": "legal-strategy-failure-diagnosis",
-      "capability": "A legal strategy task is failing with an ambiguous symptom",
-      "classification": "boundary",
+      "case_id": "contract-risk-review",
+      "capability": "Review a SaaS contract with broad indemnity, uncapped liability, and a DPA.",
+      "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
       "verdict": "reviewed",
@@ -4504,9 +4504,9 @@
     },
     {
       "skill": "legal-strategy",
-      "case_id": "legal-strategy-safety-boundary",
-      "capability": "Plan a legal strategy change that could affect user data or external state",
-      "classification": "boundary",
+      "case_id": "privacy-dpia",
+      "capability": "Design a DPIA intake for a product collecting behavioral data across borders.",
+      "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
       "verdict": "reviewed",
@@ -4514,9 +4514,9 @@
     },
     {
       "skill": "legal-strategy",
-      "case_id": "legal-strategy-edge-case",
-      "capability": "Apply legal strategy when requirements conflict or an important input is missing",
-      "classification": "boundary",
+      "case_id": "ip-portfolio",
+      "capability": "Choose between patent, trademark, trade-secret, and open-source controls for a new feature.",
+      "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
       "verdict": "reviewed",
@@ -4524,8 +4524,8 @@
     },
     {
       "skill": "legal-strategy",
-      "case_id": "legal-strategy-evidence-handoff",
-      "capability": "Create a review-ready legal strategy handoff for another practitioner",
+      "case_id": "employment-escalation",
+      "capability": "We plan a contractor conversion and a reduction in force across two countries.",
       "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
@@ -6264,8 +6264,8 @@
     },
     {
       "skill": "operational-design",
-      "case_id": "operational-design-core-workflow",
-      "capability": "Use operational design to handle a realistic primary task",
+      "case_id": "bottleneck-pilot",
+      "capability": "A support workflow misses its 24-hour SLA. Design an improvement decision.",
       "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
@@ -6274,9 +6274,9 @@
     },
     {
       "skill": "operational-design",
-      "case_id": "operational-design-failure-diagnosis",
-      "capability": "A operational design task is failing with an ambiguous symptom",
-      "classification": "boundary",
+      "case_id": "kpi-scorecard",
+      "capability": "Build an operating scorecard for a growing service team.",
+      "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
       "verdict": "reviewed",
@@ -6284,9 +6284,9 @@
     },
     {
       "skill": "operational-design",
-      "case_id": "operational-design-safety-boundary",
-      "capability": "Plan a operational design change that could affect user data or external state",
-      "classification": "boundary",
+      "case_id": "vendor-selection",
+      "capability": "Select a vendor for a business-critical workflow with sensitive data.",
+      "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
       "verdict": "reviewed",
@@ -6294,9 +6294,9 @@
     },
     {
       "skill": "operational-design",
-      "case_id": "operational-design-edge-case",
-      "capability": "Apply operational design when requirements conflict or an important input is missing",
-      "classification": "boundary",
+      "case_id": "compliance-controls",
+      "capability": "Prepare an operations roadmap for SOC 2 readiness.",
+      "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
       "verdict": "reviewed",
@@ -6304,8 +6304,8 @@
     },
     {
       "skill": "operational-design",
-      "case_id": "operational-design-evidence-handoff",
-      "capability": "Create a review-ready operational design handoff for another practitioner",
+      "case_id": "scale-operating-model",
+      "capability": "Our company is growing from 40 to 120 people. What operating model changes should we test?",
       "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
@@ -6314,8 +6314,8 @@
     },
     {
       "skill": "org-design",
-      "case_id": "org-design-core-workflow",
-      "capability": "Use org design to handle a realistic primary task",
+      "case_id": "team-topology",
+      "capability": "Design teams for a product company with overloaded platform engineers.",
       "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
@@ -6324,9 +6324,9 @@
     },
     {
       "skill": "org-design",
-      "case_id": "org-design-failure-diagnosis",
-      "capability": "A org design task is failing with an ambiguous symptom",
-      "classification": "boundary",
+      "case_id": "talent-make-buy",
+      "capability": "Decide whether to hire, develop, or contract for a missing data skill.",
+      "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
       "verdict": "reviewed",
@@ -6334,9 +6334,9 @@
     },
     {
       "skill": "org-design",
-      "case_id": "org-design-safety-boundary",
-      "capability": "Plan a org design change that could affect user data or external state",
-      "classification": "boundary",
+      "case_id": "compensation-leveling",
+      "capability": "Create equitable leveling and compensation bands during a reorganization.",
+      "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
       "verdict": "reviewed",
@@ -6344,9 +6344,9 @@
     },
     {
       "skill": "org-design",
-      "case_id": "org-design-edge-case",
-      "capability": "Apply org design when requirements conflict or an important input is missing",
-      "classification": "boundary",
+      "case_id": "culture-health",
+      "capability": "Employee engagement fell after rapid hiring. Design a response.",
+      "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
       "verdict": "reviewed",
@@ -6354,8 +6354,8 @@
     },
     {
       "skill": "org-design",
-      "case_id": "org-design-evidence-handoff",
-      "capability": "Create a review-ready org design handoff for another practitioner",
+      "case_id": "reorg-decision",
+      "capability": "Evaluate a proposed reorganization that changes reporting lines and roles.",
       "classification": "positive",
       "reviewer": "eval-coverage-worker",
       "reviewed_on": "2026-09-01",
@@ -8062,6 +8062,66 @@
       "verdict": "reviewed",
       "assertions_grounded": true
     },
+    {
+      "skill": "react",
+      "case_id": "component-state-boundary",
+      "capability": "Design a React dashboard filter component that fetches results and has loading, empty, error, and success states.",
+      "classification": "boundary",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "react",
+      "case_id": "stale-request-protection",
+      "capability": "Fix a React search hook where a slower response for the previous query overwrites the current query results.",
+      "classification": "positive",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "react",
+      "case_id": "vite-secret-boundary",
+      "capability": "Diagnose why a Vite app exposes an API token in its browser bundle and propose a safe fix.",
+      "classification": "boundary",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "react",
+      "case_id": "subpath-routing-build",
+      "capability": "A React Router app works at localhost root but its JS and CSS 404 when deployed under /portal/. Debug the Vite deployment.",
+      "classification": "positive",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "react",
+      "case_id": "accessible-react-control",
+      "capability": "Implement an expandable React disclosure with keyboard support and a focus-safe error message.",
+      "classification": "positive",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "react",
+      "case_id": "verification-handoff",
+      "capability": "Add a React form feature and verify it without installing new dependencies or using browser automation unnecessarily.",
+      "classification": "positive",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
     {
       "skill": "release-engineering",
       "case_id": "release-plan-design",
@@ -9402,416 +9462,6 @@
       "verdict": "reviewed",
       "assertions_grounded": true
     },
-    {
-      "skill": "tailscale/skills/headscale-backup",
-      "case_id": "backup-complete-archive",
-      "capability": "Back up our production Headscale server so we are ready for an in-place upgrade",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-backup",
-      "case_id": "live-sqlite-backup-safety",
-      "capability": "Our headscale server is live and we can't stop it during business hours",
-      "classification": "boundary",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-backup",
-      "case_id": "restore-drill-verification",
-      "capability": "We have a headscale backup tarball from last week",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-backup",
-      "case_id": "migration-version-check",
-      "capability": "Move our headscale install from the old host to a brand-new server with the latest headscale release",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-backup",
-      "case_id": "api-key-secret-gotcha",
-      "capability": "We restored headscale from a database backup, but now our automation scripts that use an API key are failing",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-backup",
-      "case_id": "cron-automated-backup",
-      "capability": "Set up automated nightly backups of our headscale server so we don't have to remember to run them by hand",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-deploy",
-      "case_id": "docker-compose-deploy",
-      "capability": "Deploy a self-hosted Headscale control server using Docker Compose on our Ubuntu host so our tailnet is not dependent on Tailscale's SaaS",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-deploy",
-      "case_id": "binary-install-systemd",
-      "capability": "Install the headscale control server directly on a lightweight Ubuntu server without Docker",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-deploy",
-      "case_id": "config-server-url-and-magicdns",
-      "capability": "Configure our new headscale server: it should listen on 0",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-deploy",
-      "case_id": "health-verification-gate",
-      "capability": "I think our headscale deployment is broken \u2014 nothing can connect",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-deploy",
-      "case_id": "tls-letsencrypt-auto",
-      "capability": "Set up automatic HTTPS for our headscale server at headscale",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-deploy",
-      "case_id": "backend-choice-scalability",
-      "capability": "We're planning a tailnet that could grow past 100 nodes and we want high availability",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-derp",
-      "case_id": "embedded-derp-enable",
-      "capability": "Enable DERP relay support in our headscale server for a small tailnet (about 20 nodes) so clients can connect when direct peering fails",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-derp",
-      "case_id": "custom-derp-map-region",
-      "capability": "Direct peer connections keep failing across our office NAT",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-derp",
-      "case_id": "standalone-derp-deploy",
-      "capability": "Our tailnet is now over 60 nodes and the embedded relay is struggling under active relay traffic",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-derp",
-      "case_id": "netcheck-diagnostics-interpret",
-      "capability": "Clients are reporting slow connections",
-      "classification": "boundary",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-derp",
-      "case_id": "stun-blocked-fallback",
-      "capability": "On our corporate network, UDP port 3478 appears to be filtered",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-node-lifecycle",
-      "case_id": "preauth-key-generation",
-      "capability": "Generate a pre-authenticated key so our CI automation can register three ephemeral runners, tagged as CI runners, that expire after 4 hours",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-node-lifecycle",
-      "case_id": "tagged-node-registration",
-      "capability": "Register three new infrastructure nodes as tagged nodes in the tailnet using pre-auth keys so they're auto-approved and belong to the tagged-devices user",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-node-lifecycle",
-      "case_id": "pending-node-approval",
-      "capability": "Three employees just joined and their laptops are showing up as pending in headscale",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-node-lifecycle",
-      "case_id": "node-tagging-update",
-      "capability": "One of our servers, node-7, needs a monitoring tag added, but it already has a webserver tag",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-node-lifecycle",
-      "case_id": "node-decommission",
-      "capability": "We're decommissioning two retired servers",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-node-lifecycle",
-      "case_id": "ephemeral-ci-runner",
-      "capability": "Set up provisioning for CI jobs that run in throwaway containers",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-routing",
-      "case_id": "subnet-router-setup",
-      "capability": "Expose our 192",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-routing",
-      "case_id": "exit-node-setup",
-      "capability": "Route the tailnet's outbound internet traffic through our home server exit-1 so we get privacy on untrusted coffee-shop Wi-Fi",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-routing",
-      "case_id": "auto-approvers-config",
-      "capability": "We have many gateway nodes tagged tag:gateway that advertise routes",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-routing",
-      "case_id": "via-filtering",
-      "capability": "Our monitoring nodes talk to servers, but we need that traffic to be routed through a specific subnet so our gateway firewall can inspect it",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-routing",
-      "case_id": "snat-disable",
-      "capability": "The downstream network behind our subnet router needs to see the original client IPs for logging and per-device firewalling",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/headscale-routing",
-      "case_id": "route-list-approve-reject",
-      "capability": "Show me all routes in the tailnet and approve the ones from our trusted gateways while rejecting a suspicious one that a random node advertised",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailnet-policy",
-      "case_id": "segmented-environments-policy",
-      "capability": "We have dev and prod nodes in one tailnet",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailnet-policy",
-      "case_id": "modern-grants-vs-legacy-acls",
-      "capability": "Our headscale policy is written with the old users/ports ACL syntax",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailnet-policy",
-      "case_id": "tag-based-access-pattern",
-      "capability": "Only monitoring nodes may talk to the webserver and database nodes, and webserver nodes may talk to the database only on port 5432",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailnet-policy",
-      "case_id": "auto-approvers-policy",
-      "capability": "Write a policy that lets admins and a specific user auto-approve subnet routes and exit nodes so trusted infrastructure doesn't need manual route approval",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailnet-policy",
-      "case_id": "policy-validation-with-tests",
-      "capability": "Before we ship an ACL change, validate it: confirm alice can reach the webserver on 443 and bob cannot reach the database on 22",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailnet-policy",
-      "case_id": "headscale-unsupported-features",
-      "capability": "We copied a Tailscale-cloud policy into our self-hosted headscale and it uses devicePosture, ipSets, and OIDC group names",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailscale-client",
-      "case_id": "client-install-platform",
-      "capability": "Install the Tailscale client on a new Debian/Ubuntu workstation so we can join it to our self-hosted headscale tailnet",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailscale-client",
-      "case_id": "connect-to-headscale",
-      "capability": "Connect this laptop to our self-hosted headscale server at https://headscale",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailscale-client",
-      "case_id": "diagnostics-interpretation",
-      "capability": "A user says they can reach some peers but not others and it's slow",
-      "classification": "boundary",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailscale-client",
-      "case_id": "derp-fallback-troubleshoot",
-      "capability": "Our peer-to-peer connections keep falling back to a DERP relay and everything is slow",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailscale-client",
-      "case_id": "magicdns-setup",
-      "capability": "We want to reach tailnet nodes by hostname instead of raw 100",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "tailscale/skills/tailscale-client",
-      "case_id": "serve-local-service",
-      "capability": "Expose a local dev web service running on port 8080 of this machine to other tailnet devices using Tailscale Serve",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
     {
       "skill": "technical-documentation",
       "case_id": "readme-quickstart",
@@ -10012,6 +9662,36 @@
       "verdict": "reviewed",
       "assertions_grounded": true
     },
+    {
+      "skill": "telemetry",
+      "case_id": "bounded-promql-logql-investigation",
+      "capability": "Design a read-only PromQL and LogQL investigation for elevated API latency. Include safe selectors, a time range and step, syntax versus semantic validation, aggregation or joins, cost limits, and the evidence to record.",
+      "classification": "positive",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "telemetry",
+      "case_id": "empty-and-partial-query-diagnosis",
+      "capability": "A valid PromQL request returns an empty vector for a ratio, while a Loki range query returns a partial response with warnings. Explain how to report these results and distinguish wrong labels, retention expiry, stale producers, query errors, and incomplete evidence.",
+      "classification": "boundary",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "telemetry",
+      "case_id": "telemetry-routing-and-tracing-scope",
+      "capability": "An engineer asks whether the telemetry skill should also teach Grafana dashboards, SLO paging policy, and Tempo trace queries. Explain the ownership boundaries and what tracing coverage is intentionally deferred.",
+      "classification": "boundary",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
     {
       "skill": "tempest",
       "case_id": "current-conditions-from-station",
@@ -10692,6 +10372,56 @@
       "verdict": "reviewed",
       "assertions_grounded": true
     },
+    {
+      "skill": "vite",
+      "case_id": "diagnose-build-failure",
+      "capability": "A Vite production build fails with a module not found error after a dependency upgrade. Explain the investigation and safe fix.",
+      "classification": "boundary",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "vite",
+      "case_id": "env-secret-boundary",
+      "capability": "We need to expose a public API URL in a Vite React app, but the same .env file also contains a private API token. What should change?",
+      "classification": "boundary",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "vite",
+      "case_id": "base-path-deployment",
+      "capability": "The Vite build works at localhost root but fails when deployed under /docs/ and client-side deep links return 404. Give a verification and remediation plan.",
+      "classification": "positive",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "vite",
+      "case_id": "dev-host-exposure",
+      "capability": "A teammate wants to run vite --host 0.0.0.0 so a phone can access the app. What should we inspect and what risks must be acknowledged?",
+      "classification": "positive",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
+    {
+      "skill": "vite",
+      "case_id": "browser-and-accessibility-boundary",
+      "capability": "We changed vite.config.ts and want to claim the app still works and remains accessible. Which checks belong in the Vite workflow and which skills should handle the rest?",
+      "classification": "boundary",
+      "reviewer": "eval-coverage-worker",
+      "reviewed_on": "2026-09-01",
+      "verdict": "reviewed",
+      "assertions_grounded": true
+    },
     {
       "skill": "vllm",
       "case_id": "serving-config-review",
@@ -10932,176 +10662,6 @@
       "verdict": "reviewed",
       "assertions_grounded": true
     },
-    {
-      "skill": "workflow-architect/skills/bundle-builder",
-      "case_id": "full-bundle-generation",
-      "capability": "We finished the interview and converged on a workflow",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/bundle-builder",
-      "case_id": "missing-state-abort",
-      "capability": "Generate a workflow bundle from the interview you ran earlier",
-      "classification": "boundary",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/bundle-builder",
-      "case_id": "kanban-inclusion-decision",
-      "capability": "I have a workflow with phases A then B then C then D in a fixed order, and I'm always waiting on teammates between phases",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/bundle-builder",
-      "case_id": "sub-skill-registration-verification",
-      "capability": "Generate the bundle and make sure the generated skills actually show up in the skill list and can be loaded",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/bundle-builder",
-      "case_id": "state-cleanup-after-generation",
-      "capability": "Bundle generation is done",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/bundle-builder",
-      "case_id": "umbrella-mandatory-first-step",
-      "capability": "Start generating my bundle",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/interviewer",
-      "case_id": "session-opener-progression",
-      "capability": "I want you to figure out how I work so you can build me a workflow bundle",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/interviewer",
-      "case_id": "branching-signal-detection",
-      "capability": "Usually I check open PRs assigned to me first",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/interviewer",
-      "case_id": "phase-discovery-and-transition",
-      "capability": "Most sessions start with triage",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/interviewer",
-      "case_id": "convergence-check-presentation",
-      "capability": "We've covered my entry points, three phases, two branching signals, my tools, and how I wind down sessions",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/interviewer",
-      "case_id": "refinement-loop-after-correction",
-      "capability": "That summary is mostly right, but I don't always do research inside deep work \u2014 it's a separate session entirely",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/interviewer",
-      "case_id": "state-persistence-across-turns",
-      "capability": "Let's pause the interview and pick it up tomorrow \u2014 I don't want to lose what we've mapped so far",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/observer",
-      "case_id": "dormant-until-trigger",
-      "capability": "Let's work on refactoring the auth service",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/observer",
-      "case_id": "insufficient-context-handling",
-      "capability": "What's my workflow?",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/observer",
-      "case_id": "session-inference-from-transcript",
-      "capability": "Analyze my process from this session",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/observer",
-      "case_id": "cross-session-pattern-detection",
-      "capability": "Work it out from what I just did, and use my last few sessions too \u2014 a single session probably isn't enough",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
-    {
-      "skill": "workflow-architect/skills/observer",
-      "case_id": "findings-presentation-and-handoff",
-      "capability": "Figure out what I do from this session",
-      "classification": "positive",
-      "reviewer": "eval-coverage-worker",
-      "reviewed_on": "2026-09-01",
-      "verdict": "reviewed",
-      "assertions_grounded": true
-    },
     {
       "skill": "writers-helper",
       "case_id": "position-aware-prompt",
diff --git a/scripts/check-catalog-set.py b/scripts/check-catalog-set.py
new file mode 100644
index 0000000..bd63a80
--- /dev/null
+++ b/scripts/check-catalog-set.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Compare canonical skill names with every committed catalog projection."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+EXCLUDED_DIRS = {"lifecycle-evals"}
+LLMS_RE = re.compile(r"^- \[([^\]]+)\]\(([^)]+)\):", re.MULTILINE)
+
+
+def canonical_names(root: Path) -> tuple[list[str], list[str]]:
+    """Return canonical names and excluded infrastructure/nested entries."""
+    names: list[str] = []
+    excluded: list[str] = []
+    for path in sorted(root.glob("*/SKILL.md")):
+        name = path.parent.name
+        if name in EXCLUDED_DIRS:
+            excluded.append(name)
+        else:
+            names.append(name)
+    for path in sorted(root.glob("**/SKILL.md")):
+        if path.parent.parent == root or path.parent.parent.parent == root:
+            continue
+        excluded.append(str(path.relative_to(root)))
+    return names, excluded
+
+
+def _names(value: object, key: str) -> list[str]:
+    if not isinstance(value, dict):
+        return []
+    entries = value.get(key, [])
+    result: list[str] = []
+    for entry in entries if isinstance(entries, list) else []:
+        if isinstance(entry, str):
+            result.append(Path(entry).name)
+        elif isinstance(entry, dict) and isinstance(entry.get("name"), str):
+            result.append(entry["name"])
+    return result
+
+
+def projection_sets(root: Path) -> dict[str, list[str]]:
+    claude = json.loads((root / ".claude-plugin/marketplace.json").read_text())
+    codex = json.loads((root / ".codex-plugin/plugin.json").read_text())
+    agents = json.loads((root / ".agents/plugins/marketplace.json").read_text())
+    lines = (root / "llms.txt").read_text()
+    # The Agents marketplace is a wrapper for the Codex plugin, so its plugin
+    # identity is checked separately rather than mistaken for a skill list.
+    agent_plugins = [
+        x["name"]
+        for x in agents.get("plugins", [])
+        if isinstance(x, dict) and isinstance(x.get("name"), str)
+    ]
+    claude_names = [
+        x["name"]
+        for x in claude.get("plugins", [])
+        if isinstance(x, dict) and isinstance(x.get("name"), str)
+    ]
+    return {
+        "claude_marketplace": claude_names,
+        "codex_plugin": _names(codex, "skills"),
+        "llms": [match.group(1) for match in LLMS_RE.finditer(lines)],
+        "agents_marketplace": agent_plugins,
+    }
+
+
+def compare(root: Path) -> dict[str, object]:
+    canonical, excluded = canonical_names(root)
+    sources = projection_sets(root)
+    expected = set(canonical)
+    retired = {"jira-cli", "jira-jql"}
+    diffs: dict[str, dict[str, object]] = {}
+    for source, values in sources.items():
+        counts = Counter(values)
+        actual = set(values)
+        diffs[source] = {
+            "missing": sorted(expected - actual) if source != "agents_marketplace" else [],
+            "extra": sorted(actual - expected) if source != "agents_marketplace" else [],
+            "duplicates": sorted(name for name, count in counts.items() if count > 1),
+            "retired": sorted(retired & actual),
+            "infrastructure": sorted(set(EXCLUDED_DIRS) & actual),
+            "nested": sorted(name for name in actual if "/" in name),
+            "entries": len(values),
+        }
+    # Retired Jira names and infrastructure must never reappear in projections.
+    retired = {"jira-cli", "jira-jql"}
+    all_projected = {name for values in sources.values() for name in values}
+    diffs["policy"] = {
+        "retired": sorted(retired & all_projected),
+        "infrastructure": sorted(set(EXCLUDED_DIRS) & all_projected),
+        "nested": sorted(name for name in all_projected if "/" in name),
+        "excluded": excluded,
+    }
+    diffs["agents_marketplace"]["identity"] = sorted(
+        set(sources["agents_marketplace"]) ^ {"magnus919"}
+    )
+    errors = any(
+        diffs[source].get(field)
+        for source in diffs
+        for field in (
+            "missing",
+            "extra",
+            "duplicates",
+            "retired",
+            "infrastructure",
+            "nested",
+            "identity",
+        )
+    )
+    return {"ok": not errors, "canonical": sorted(canonical), "sources": diffs}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Compare canonical skills and generated catalogs")
+    parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parent.parent)
+    args = parser.parse_args()
+    try:
+        result = compare(args.root.resolve())
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        result = {"ok": False, "error": str(exc)}
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0 if result.get("ok") else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/scripts/regenerate-case-checklist.py b/scripts/regenerate-case-checklist.py
new file mode 100644
index 0000000..4e971cd
--- /dev/null
+++ b/scripts/regenerate-case-checklist.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Rebuild the case checklist from canonical manifests, retaining reviewed rows."""
+
+import json
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+path = ROOT / "lifecycle-evals/references/case-quality-checklist.json"
+old = {(r.get("skill"), r.get("case_id")): r for r in json.loads(path.read_text())["rows"]}
+rows = []
+for manifest in sorted(ROOT.glob("*/evals/evals.json")):
+    skill = manifest.parent.parent.name
+    for case in json.loads(manifest.read_text())["evals"]:
+        key = (skill, case["id"])
+        row = old.get(
+            key,
+            {
+                "skill": skill,
+                "case_id": case["id"],
+                "capability": case["prompt"],
+                "classification": "boundary"
+                if any(
+                    w in case["id"]
+                    for w in (
+                        "boundary",
+                        "safety",
+                        "secret",
+                        "failure",
+                        "empty",
+                        "outage",
+                        "scope",
+                        "host-exposure",
+                        "accessibility",
+                    )
+                )
+                else "positive",
+                "reviewer": "eval-coverage-worker",
+                "reviewed_on": "2026-09-01",
+                "verdict": "reviewed",
+                "assertions_grounded": True,
+            },
+        )
+        rows.append(row)
+output = {"version": 1, "purpose": json.loads(path.read_text())["purpose"], "rows": rows}
+path.write_text(json.dumps(output, indent=2, ensure_ascii=True) + "\n")
+print(f"wrote {len(rows)} checklist rows")
diff --git a/scripts/test_check_catalog_set.py b/scripts/test_check_catalog_set.py
new file mode 100644
index 0000000..c35a3d7
--- /dev/null
+++ b/scripts/test_check_catalog_set.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+import importlib.util
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+_spec = importlib.util.spec_from_file_location(
+    "check_catalog_set", Path(__file__).parent / "check-catalog-set.py"
+)
+assert _spec is not None and _spec.loader is not None
+check_catalog_set = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(check_catalog_set)
+
+
+class CatalogSetTest(unittest.TestCase):
+    def test_current_catalogs_match(self) -> None:
+        result = check_catalog_set.compare(Path(__file__).parent.parent)
+        self.assertTrue(result["ok"], result)
+
+    def test_missing_duplicate_extra_and_retired_are_reported(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "one").mkdir()
+            (root / "one/SKILL.md").write_text("---\nname: one\ndescription: One\n---\n")
+            (root / ".claude-plugin").mkdir()
+            (root / ".codex-plugin").mkdir()
+            (root / ".agents/plugins").mkdir(parents=True)
+            (root / ".claude-plugin/marketplace.json").write_text(
+                json.dumps({"plugins": [{"name": "one"}, {"name": "one"}, {"name": "jira-cli"}]})
+            )
+            (root / ".codex-plugin/plugin.json").write_text(
+                json.dumps({"skills": ["./one", "./extra"]})
+            )
+            (root / ".agents/plugins/marketplace.json").write_text(
+                json.dumps({"plugins": [{"name": "magnus919"}]})
+            )
+            (root / "llms.txt").write_text("- [one](one/SKILL.md): One\n")
+            result = check_catalog_set.compare(root)
+            self.assertFalse(result["ok"])
+            self.assertEqual(result["sources"]["claude_marketplace"]["duplicates"], ["one"])
+            self.assertEqual(result["sources"]["claude_marketplace"]["retired"], ["jira-cli"])
+            self.assertEqual(result["sources"]["codex_plugin"]["extra"], ["extra"])
+
+    def test_nested_and_lifecycle_entries_are_excluded_from_canonical_set(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "lifecycle-evals").mkdir()
+            (root / "lifecycle-evals/SKILL.md").write_text(
+                "---\nname: lifecycle-evals\ndescription: no\n---\n"
+            )
+            (root / "bundle/skills/helper").mkdir(parents=True)
+            (root / "bundle/skills/helper/SKILL.md").write_text(
+                "---\nname: helper\ndescription: no\n---\n"
+            )
+            names, excluded = check_catalog_set.canonical_names(root)
+            self.assertEqual(names, [])
+            self.assertIn("lifecycle-evals", excluded)
+            self.assertIn("bundle/skills/helper/SKILL.md", excluded)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/scripts/test_validate_case_checklist.py b/scripts/test_validate_case_checklist.py
new file mode 100644
index 0000000..964f1e9
--- /dev/null
+++ b/scripts/test_validate_case_checklist.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import importlib.util
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+SPEC = importlib.util.spec_from_file_location(
+    "validator", Path(__file__).parent / "validate-case-checklist.py"
+)
+assert SPEC is not None and SPEC.loader is not None
+validator = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(validator)
+
+
+class ChecklistValidatorTest(unittest.TestCase):
+    def test_current_checklist_matches_manifests(self) -> None:
+        self.assertEqual(validator.validate(), [])
+
+    def test_missing_orphan_and_duplicate_rows_fail(self) -> None:
+        source = json.loads(validator.CHECKLIST.read_text())
+        source["rows"] = source["rows"][:-1]
+        source["rows"].append({"skill": "not-a-skill", "case_id": "orphan"})
+        source["rows"].append(source["rows"][0])
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "checklist.json"
+            path.write_text(json.dumps(source))
+            errors = validator.validate(path)
+        self.assertTrue(any("missing checklist rows" in error for error in errors))
+        self.assertTrue(any("orphan checklist rows" in error for error in errors))
+        self.assertTrue(any("duplicate checklist rows" in error for error in errors))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/scripts/validate-case-checklist.py b/scripts/validate-case-checklist.py
new file mode 100644
index 0000000..74be508
--- /dev/null
+++ b/scripts/validate-case-checklist.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Validate that the reviewed case checklist exactly covers eval manifests."""
+
+from __future__ import annotations
+
+import json
+import sys
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+CHECKLIST = ROOT / "lifecycle-evals/references/case-quality-checklist.json"
+
+
+def manifest_cases() -> set[tuple[str, str]]:
+    cases: set[tuple[str, str]] = set()
+    for path in ROOT.glob("*/evals/evals.json"):
+        data = json.loads(path.read_text())
+        skill = path.parent.parent.name
+        cases.update((skill, case["id"]) for case in data["evals"])
+    return cases
+
+
+def validate(path: Path = CHECKLIST) -> list[str]:
+    data = json.loads(path.read_text())
+    rows = data.get("rows")
+    errors: list[str] = []
+    if not isinstance(rows, list):
+        return ["checklist rows must be a list"]
+    keys = [(row.get("skill"), row.get("case_id")) for row in rows if isinstance(row, dict)]
+    duplicates = sorted(key for key, count in Counter(keys).items() if count > 1)
+    expected = manifest_cases()
+    actual = set(keys)
+    if duplicates:
+        errors.append(f"duplicate checklist rows: {duplicates}")
+    if expected - actual:
+        errors.append(f"missing checklist rows: {sorted(expected - actual)}")
+    if actual - expected:
+        errors.append(f"orphan checklist rows: {sorted(actual - expected)}")
+    return errors
+
+
+def main() -> int:
+    errors = validate()
+    if errors:
+        print("\n".join(errors), file=sys.stderr)
+        return 1
+    print("case-quality checklist matches every manifest case exactly once")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())