test(validation): enforce catalog and checklist parity

Implement catalog source comparison and exact eval checklist parity with deterministic validation coverage.

scripts/check-catalog-set.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Compare canonical skill names with every committed catalog projection."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+EXCLUDED_DIRS = {"lifecycle-evals"}
+LLMS_RE = re.compile(r"^- \[([^\]]+)\]\(([^)]+)\):", re.MULTILINE)
+
+
+def canonical_names(root: Path) -> tuple[list[str], list[str]]:
+    """Return canonical names and excluded infrastructure/nested entries."""
+    names: list[str] = []
+    excluded: list[str] = []
+    for path in sorted(root.glob("*/SKILL.md")):
+        name = path.parent.name
+        if name in EXCLUDED_DIRS:
+            excluded.append(name)
+        else:
+            names.append(name)
+    for path in sorted(root.glob("**/SKILL.md")):
+        if path.parent.parent == root or path.parent.parent.parent == root:
+            continue
+        excluded.append(str(path.relative_to(root)))
+    return names, excluded
+
+
+def _names(value: object, key: str) -> list[str]:
+    if not isinstance(value, dict):
+        return []
+    entries = value.get(key, [])
+    result: list[str] = []
+    for entry in entries if isinstance(entries, list) else []:
+        if isinstance(entry, str):
+            result.append(Path(entry).name)
+        elif isinstance(entry, dict) and isinstance(entry.get("name"), str):
+            result.append(entry["name"])
+    return result
+
+
+def projection_sets(root: Path) -> dict[str, list[str]]:
+    claude = json.loads((root / ".claude-plugin/marketplace.json").read_text())
+    codex = json.loads((root / ".codex-plugin/plugin.json").read_text())
+    agents = json.loads((root / ".agents/plugins/marketplace.json").read_text())
+    lines = (root / "llms.txt").read_text()
+    # The Agents marketplace is a wrapper for the Codex plugin, so its plugin
+    # identity is checked separately rather than mistaken for a skill list.
+    agent_plugins = [
+        x["name"]
+        for x in agents.get("plugins", [])
+        if isinstance(x, dict) and isinstance(x.get("name"), str)
+    ]
+    claude_names = [
+        x["name"]
+        for x in claude.get("plugins", [])
+        if isinstance(x, dict) and isinstance(x.get("name"), str)
+    ]
+    return {
+        "claude_marketplace": claude_names,
+        "codex_plugin": _names(codex, "skills"),
+        "llms": [match.group(1) for match in LLMS_RE.finditer(lines)],
+        "agents_marketplace": agent_plugins,
+    }
+
+
+def compare(root: Path) -> dict[str, object]:
+    canonical, excluded = canonical_names(root)
+    sources = projection_sets(root)
+    expected = set(canonical)
+    retired = {"jira-cli", "jira-jql"}
+    diffs: dict[str, dict[str, object]] = {}
+    for source, values in sources.items():
+        counts = Counter(values)
+        actual = set(values)
+        diffs[source] = {
+            "missing": sorted(expected - actual) if source != "agents_marketplace" else [],
+            "extra": sorted(actual - expected) if source != "agents_marketplace" else [],
+            "duplicates": sorted(name for name, count in counts.items() if count > 1),
+            "retired": sorted(retired & actual),
+            "infrastructure": sorted(set(EXCLUDED_DIRS) & actual),
+            "nested": sorted(name for name in actual if "/" in name),
+            "entries": len(values),
+        }
+    # Retired Jira names and infrastructure must never reappear in projections.
+    retired = {"jira-cli", "jira-jql"}
+    all_projected = {name for values in sources.values() for name in values}
+    diffs["policy"] = {
+        "retired": sorted(retired & all_projected),
+        "infrastructure": sorted(set(EXCLUDED_DIRS) & all_projected),
+        "nested": sorted(name for name in all_projected if "/" in name),
+        "excluded": excluded,
+    }
+    diffs["agents_marketplace"]["identity"] = sorted(
+        set(sources["agents_marketplace"]) ^ {"magnus919"}
+    )
+    errors = any(
+        diffs[source].get(field)
+        for source in diffs
+        for field in (
+            "missing",
+            "extra",
+            "duplicates",
+            "retired",
+            "infrastructure",
+            "nested",
+            "identity",
+        )
+    )
+    return {"ok": not errors, "canonical": sorted(canonical), "sources": diffs}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Compare canonical skills and generated catalogs")
+    parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parent.parent)
+    args = parser.parse_args()
+    try:
+        result = compare(args.root.resolve())
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        result = {"ok": False, "error": str(exc)}
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0 if result.get("ok") else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/regenerate-case-checklist.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Rebuild the case checklist from canonical manifests, retaining reviewed rows."""
+
+import json
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+path = ROOT / "lifecycle-evals/references/case-quality-checklist.json"
+old = {(r.get("skill"), r.get("case_id")): r for r in json.loads(path.read_text())["rows"]}
+rows = []
+for manifest in sorted(ROOT.glob("*/evals/evals.json")):
+    skill = manifest.parent.parent.name
+    for case in json.loads(manifest.read_text())["evals"]:
+        key = (skill, case["id"])
+        row = old.get(
+            key,
+            {
+                "skill": skill,
+                "case_id": case["id"],
+                "capability": case["prompt"],
+                "classification": "boundary"
+                if any(
+                    w in case["id"]
+                    for w in (
+                        "boundary",
+                        "safety",
+                        "secret",
+                        "failure",
+                        "empty",
+                        "outage",
+                        "scope",
+                        "host-exposure",
+                        "accessibility",
+                    )
+                )
+                else "positive",
+                "reviewer": "eval-coverage-worker",
+                "reviewed_on": "2026-09-01",
+                "verdict": "reviewed",
+                "assertions_grounded": True,
+            },
+        )
+        rows.append(row)
+output = {"version": 1, "purpose": json.loads(path.read_text())["purpose"], "rows": rows}
+path.write_text(json.dumps(output, indent=2, ensure_ascii=True) + "\n")
+print(f"wrote {len(rows)} checklist rows")

scripts/test_check_catalog_set.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+import importlib.util
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+_spec = importlib.util.spec_from_file_location(
+    "check_catalog_set", Path(__file__).parent / "check-catalog-set.py"
+)
+assert _spec is not None and _spec.loader is not None
+check_catalog_set = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(check_catalog_set)
+
+
+class CatalogSetTest(unittest.TestCase):
+    def test_current_catalogs_match(self) -> None:
+        result = check_catalog_set.compare(Path(__file__).parent.parent)
+        self.assertTrue(result["ok"], result)
+
+    def test_missing_duplicate_extra_and_retired_are_reported(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "one").mkdir()
+            (root / "one/SKILL.md").write_text("---\nname: one\ndescription: One\n---\n")
+            (root / ".claude-plugin").mkdir()
+            (root / ".codex-plugin").mkdir()
+            (root / ".agents/plugins").mkdir(parents=True)
+            (root / ".claude-plugin/marketplace.json").write_text(
+                json.dumps({"plugins": [{"name": "one"}, {"name": "one"}, {"name": "jira-cli"}]})
+            )
+            (root / ".codex-plugin/plugin.json").write_text(
+                json.dumps({"skills": ["./one", "./extra"]})
+            )
+            (root / ".agents/plugins/marketplace.json").write_text(
+                json.dumps({"plugins": [{"name": "magnus919"}]})
+            )
+            (root / "llms.txt").write_text("- [one](one/SKILL.md): One\n")
+            result = check_catalog_set.compare(root)
+            self.assertFalse(result["ok"])
+            self.assertEqual(result["sources"]["claude_marketplace"]["duplicates"], ["one"])
+            self.assertEqual(result["sources"]["claude_marketplace"]["retired"], ["jira-cli"])
+            self.assertEqual(result["sources"]["codex_plugin"]["extra"], ["extra"])
+
+    def test_nested_and_lifecycle_entries_are_excluded_from_canonical_set(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "lifecycle-evals").mkdir()
+            (root / "lifecycle-evals/SKILL.md").write_text(
+                "---\nname: lifecycle-evals\ndescription: no\n---\n"
+            )
+            (root / "bundle/skills/helper").mkdir(parents=True)
+            (root / "bundle/skills/helper/SKILL.md").write_text(
+                "---\nname: helper\ndescription: no\n---\n"
+            )
+            names, excluded = check_catalog_set.canonical_names(root)
+            self.assertEqual(names, [])
+            self.assertIn("lifecycle-evals", excluded)
+            self.assertIn("bundle/skills/helper/SKILL.md", excluded)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/test_validate_case_checklist.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import importlib.util
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+SPEC = importlib.util.spec_from_file_location(
+    "validator", Path(__file__).parent / "validate-case-checklist.py"
+)
+assert SPEC is not None and SPEC.loader is not None
+validator = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(validator)
+
+
+class ChecklistValidatorTest(unittest.TestCase):
+    def test_current_checklist_matches_manifests(self) -> None:
+        self.assertEqual(validator.validate(), [])
+
+    def test_missing_orphan_and_duplicate_rows_fail(self) -> None:
+        source = json.loads(validator.CHECKLIST.read_text())
+        source["rows"] = source["rows"][:-1]
+        source["rows"].append({"skill": "not-a-skill", "case_id": "orphan"})
+        source["rows"].append(source["rows"][0])
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "checklist.json"
+            path.write_text(json.dumps(source))
+            errors = validator.validate(path)
+        self.assertTrue(any("missing checklist rows" in error for error in errors))
+        self.assertTrue(any("orphan checklist rows" in error for error in errors))
+        self.assertTrue(any("duplicate checklist rows" in error for error in errors))
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/validate-case-checklist.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Validate that the reviewed case checklist exactly covers eval manifests."""
+
+from __future__ import annotations
+
+import json
+import sys
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+CHECKLIST = ROOT / "lifecycle-evals/references/case-quality-checklist.json"
+
+
+def manifest_cases() -> set[tuple[str, str]]:
+    cases: set[tuple[str, str]] = set()
+    for path in ROOT.glob("*/evals/evals.json"):
+        data = json.loads(path.read_text())
+        skill = path.parent.parent.name
+        cases.update((skill, case["id"]) for case in data["evals"])
+    return cases
+
+
+def validate(path: Path = CHECKLIST) -> list[str]:
+    data = json.loads(path.read_text())
+    rows = data.get("rows")
+    errors: list[str] = []
+    if not isinstance(rows, list):
+        return ["checklist rows must be a list"]
+    keys = [(row.get("skill"), row.get("case_id")) for row in rows if isinstance(row, dict)]
+    duplicates = sorted(key for key, count in Counter(keys).items() if count > 1)
+    expected = manifest_cases()
+    actual = set(keys)
+    if duplicates:
+        errors.append(f"duplicate checklist rows: {duplicates}")
+    if expected - actual:
+        errors.append(f"missing checklist rows: {sorted(expected - actual)}")
+    if actual - expected:
+        errors.append(f"orphan checklist rows: {sorted(actual - expected)}")
+    return errors
+
+
+def main() -> int:
+    errors = validate()
+    if errors:
+        print("\n".join(errors), file=sys.stderr)
+        return 1
+    print("case-quality checklist matches every manifest case exactly once")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())