test(validation): enforce catalog and checklist parity

Implement catalog source comparison and exact eval checklist parity with deterministic validation coverage.

scripts/validate-case-checklist.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Validate that the reviewed case checklist exactly covers eval manifests."""
+
+from __future__ import annotations
+
+import json
+import sys
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+CHECKLIST = ROOT / "lifecycle-evals/references/case-quality-checklist.json"
+
+
+def manifest_cases() -> set[tuple[str, str]]:
+    cases: set[tuple[str, str]] = set()
+    for path in ROOT.glob("*/evals/evals.json"):
+        data = json.loads(path.read_text())
+        skill = path.parent.parent.name
+        cases.update((skill, case["id"]) for case in data["evals"])
+    return cases
+
+
+def validate(path: Path = CHECKLIST) -> list[str]:
+    data = json.loads(path.read_text())
+    rows = data.get("rows")
+    errors: list[str] = []
+    if not isinstance(rows, list):
+        return ["checklist rows must be a list"]
+    keys = [(row.get("skill"), row.get("case_id")) for row in rows if isinstance(row, dict)]
+    duplicates = sorted(key for key, count in Counter(keys).items() if count > 1)
+    expected = manifest_cases()
+    actual = set(keys)
+    if duplicates:
+        errors.append(f"duplicate checklist rows: {duplicates}")
+    if expected - actual:
+        errors.append(f"missing checklist rows: {sorted(expected - actual)}")
+    if actual - expected:
+        errors.append(f"orphan checklist rows: {sorted(actual - expected)}")
+    return errors
+
+
+def main() -> int:
+    errors = validate()
+    if errors:
+        print("\n".join(errors), file=sys.stderr)
+        return 1
+    print("case-quality checklist matches every manifest case exactly once")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())