mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-11 19:47:12 +03:00
Magnus HedemarkandGitHub
32eaa07c7e
Implement catalog source comparison and exact eval checklist parity with deterministic validation coverage.
54 lines
1.6 KiB
Python
54 lines
1.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Validate that the reviewed case checklist exactly covers eval manifests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
CHECKLIST = ROOT / "lifecycle-evals/references/case-quality-checklist.json"
|
|
|
|
|
|
def manifest_cases() -> set[tuple[str, str]]:
|
|
cases: set[tuple[str, str]] = set()
|
|
for path in ROOT.glob("*/evals/evals.json"):
|
|
data = json.loads(path.read_text())
|
|
skill = path.parent.parent.name
|
|
cases.update((skill, case["id"]) for case in data["evals"])
|
|
return cases
|
|
|
|
|
|
def validate(path: Path = CHECKLIST) -> list[str]:
|
|
data = json.loads(path.read_text())
|
|
rows = data.get("rows")
|
|
errors: list[str] = []
|
|
if not isinstance(rows, list):
|
|
return ["checklist rows must be a list"]
|
|
keys = [(row.get("skill"), row.get("case_id")) for row in rows if isinstance(row, dict)]
|
|
duplicates = sorted(key for key, count in Counter(keys).items() if count > 1)
|
|
expected = manifest_cases()
|
|
actual = set(keys)
|
|
if duplicates:
|
|
errors.append(f"duplicate checklist rows: {duplicates}")
|
|
if expected - actual:
|
|
errors.append(f"missing checklist rows: {sorted(expected - actual)}")
|
|
if actual - expected:
|
|
errors.append(f"orphan checklist rows: {sorted(actual - expected)}")
|
|
return errors
|
|
|
|
|
|
def main() -> int:
|
|
errors = validate()
|
|
if errors:
|
|
print("\n".join(errors), file=sys.stderr)
|
|
return 1
|
|
print("case-quality checklist matches every manifest case exactly once")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|