From be0c8df5e20d75761cf540e36a5fded303f86c51 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Tue, 21 Jul 2026 21:03:21 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20eval=20coverage=20ratchet=20=E2=80=94?= =?UTF-8?q?=20gate=20new=20skills,=20track=20coverage,=20ratchet=20thresho?= =?UTF-8?q?lds=20(#99)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1: New skills (not in grandfathered-skills.txt) must have evals/evals.json with at least 5 test cases. All 107 existing skills are grandfathered. Phase 2: scripts/eval-coverage.py reports coverage (skills with/without evals, case counts, reference-priority sorting). Added as informational CI step. Phase 3: Ratchet thresholds — at 25% coverage, modified skills without evals get a warning; at 50%, they fail CI. Enforced via --modified-from flag for PR-scoped checks. Closes #90 --- .github/workflows/validate.yml | 2 + scripts/eval-coverage.py | 206 +++++++++++++++++++++++++++++++ scripts/grandfathered-skills.txt | 107 ++++++++++++++++ scripts/validate-skills.rb | 25 ++++ 4 files changed, 340 insertions(+) create mode 100644 scripts/eval-coverage.py create mode 100644 scripts/grandfathered-skills.txt diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 60bb1e8..968d579 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -34,6 +34,8 @@ jobs: run: ruby scripts/gen-claude-marketplace.rb - name: Validate Codex plugin packaging run: ruby scripts/gen-codex-plugin.rb + - name: Report eval coverage + run: python3 scripts/eval-coverage.py - name: Test llms.txt generator run: ruby scripts/test-gen-llms-txt.rb - name: Validate llms.txt catalog diff --git a/scripts/eval-coverage.py b/scripts/eval-coverage.py new file mode 100644 index 0000000..f3a4c93 --- /dev/null +++ b/scripts/eval-coverage.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Report eval coverage across skills and enforce ratchet thresholds. + +Phase 2: informational coverage report (always passes). +Phase 3: ratchet — warn at 25%, fail-on-modify at 50%. + +Usage: + python3 scripts/eval-coverage.py # human-readable report + python3 scripts/eval-coverage.py --json # machine-readable + python3 scripts/eval-coverage.py --modified-from REF # ratchet check +""" + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +GRANDFATHER_FILE = ROOT / "scripts" / "grandfathered-skills.txt" + +# Phase 3 ratchet thresholds (percent of skills with evals) +WARN_THRESHOLD = 25 # modified skills without evals get a warning +FAIL_THRESHOLD = 50 # modified skills without evals fail CI + + +def find_skills() -> list[Path]: + """Find all canonical skill directories via git-tracked SKILL.md files.""" + result = subprocess.run( + ["git", "ls-files", "-z", "--", "*/SKILL.md"], + cwd=ROOT, + check=True, + capture_output=True, + ) + skills = [] + for name in result.stdout.decode().split("\0"): + if not name or "/agent-council/profiles/skills/" in name: + continue + skills.append(Path(name).parent) + return sorted(skills) + + +def load_grandfathered() -> set[str]: + if not GRANDFATHER_FILE.exists(): + return set() + return { + line.strip() + for line in GRANDFATHER_FILE.read_text().splitlines() + if line.strip() and not line.strip().startswith("#") + } + + +def check_evals(skill_dir: Path) -> tuple[bool, int]: + """Return (has_valid_evals, case_count) for a skill directory.""" + evals_file = ROOT / skill_dir / "evals" / "evals.json" + if not evals_file.exists(): + return False, 0 + try: + data = json.loads(evals_file.read_text(encoding="utf-8")) + if isinstance(data, dict) and isinstance(data.get("evals"), list): + count = len(data["evals"]) + return count > 0, count + return False, 0 + except (json.JSONDecodeError, OSError): + return False, 0 + + +def count_references(skill_name: str, all_skill_dirs: list[Path]) -> int: + """Count how many other SKILL.md files mention this skill name.""" + count = 0 + for skill_dir in all_skill_dirs: + skill_md = ROOT / skill_dir / "SKILL.md" + if not skill_md.exists(): + continue + try: + if skill_name in skill_md.read_text(encoding="utf-8"): + count += 1 + except OSError: + pass + return count + + +def modified_skills(base_ref: str) -> set[Path]: + """Return skill directories with changes between base_ref and HEAD.""" + result = subprocess.run( + ["git", "diff", "--name-only", base_ref, "HEAD", "--", "*/SKILL.md"], + cwd=ROOT, + capture_output=True, + text=True, + ) + modified = set() + for line in result.stdout.strip().splitlines(): + if line and "/agent-council/profiles/skills/" not in line: + modified.add(Path(line).parent) + return modified + + +def main() -> int: + parser = argparse.ArgumentParser(description="Eval coverage report and ratchet") + parser.add_argument("--json", action="store_true", help="JSON output") + parser.add_argument( + "--modified-from", + metavar="REF", + help="Apply ratchet only to skills modified since REF", + ) + args = parser.parse_args() + + skills = find_skills() + grandfathered = load_grandfathered() + + total = len(skills) + with_evals: list[dict] = [] + without_evals: list[str] = [] + + for skill_dir in skills: + has, count = check_evals(skill_dir) + name = str(skill_dir) + if has: + with_evals.append({"skill": name, "cases": count}) + else: + without_evals.append(name) + + coverage_pct = (len(with_evals) / total * 100) if total else 0.0 + + # Sort skills without evals: most-referenced first, then alphabetical + ref_counts = { + name: count_references(Path(name).name, skills) for name in without_evals + } + without_evals.sort(key=lambda n: (-ref_counts[n], n)) + + # Phase 3 ratchet check + ratchet_warnings: list[str] = [] + ratchet_errors: list[str] = [] + if args.modified_from: + modified = modified_skills(args.modified_from) + for skill_dir in sorted(modified): + name = str(skill_dir) + has, _ = check_evals(skill_dir) + if not has: + if coverage_pct >= FAIL_THRESHOLD: + ratchet_errors.append( + f"{name}: modified skill has no evals " + f"(coverage {coverage_pct:.1f}% >= {FAIL_THRESHOLD}% — evals required on modification)" + ) + elif coverage_pct >= WARN_THRESHOLD: + ratchet_warnings.append( + f"{name}: modified skill has no evals " + f"(coverage {coverage_pct:.1f}% >= {WARN_THRESHOLD}% — evals recommended)" + ) + + if args.json: + print( + json.dumps( + { + "total_skills": total, + "skills_with_evals": len(with_evals), + "skills_without_evals": len(without_evals), + "coverage_pct": round(coverage_pct, 1), + "with_evals": with_evals, + "without_evals": [ + {"skill": n, "references": ref_counts.get(n, 0)} + for n in without_evals + ], + "ratchet": { + "warn_threshold": WARN_THRESHOLD, + "fail_threshold": FAIL_THRESHOLD, + "warnings": ratchet_warnings, + "errors": ratchet_errors, + }, + }, + indent=2, + ) + ) + else: + print(f"Eval coverage: {len(with_evals)}/{total} skills ({coverage_pct:.1f}%)") + print() + if with_evals: + print("Skills WITH evals:") + for entry in with_evals: + print(f" + {entry['skill']} ({entry['cases']} cases)") + print() + print(f"Skills WITHOUT evals ({len(without_evals)}), by reference count:") + for name in without_evals: + refs = ref_counts.get(name, 0) + print(f" - {name} (referenced by {refs} skills)") + print() + print( + f"Ratchet: warn at {WARN_THRESHOLD}%, " + f"fail-on-modify at {FAIL_THRESHOLD}%" + ) + if ratchet_warnings: + print() + print("Ratchet warnings:") + for w in ratchet_warnings: + print(f" WARNING: {w}") + if ratchet_errors: + print() + print("Ratchet errors:") + for e in ratchet_errors: + print(f" ERROR: {e}") + + return 1 if ratchet_errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/grandfathered-skills.txt b/scripts/grandfathered-skills.txt new file mode 100644 index 0000000..f3d1b97 --- /dev/null +++ b/scripts/grandfathered-skills.txt @@ -0,0 +1,107 @@ +adr-authoring +agent-council +agent-evals-and-observability +agent-skills +api-design-and-evolution +artifact-pyramids +autogen +backend-engineering +brand-designer +bundles/neckbeard +bundles/research-and-vault +bundles/tailscale +bundles/tailscale/skills/headscale-backup +bundles/tailscale/skills/headscale-deploy +bundles/tailscale/skills/headscale-derp +bundles/tailscale/skills/headscale-node-lifecycle +bundles/tailscale/skills/headscale-routing +bundles/tailscale/skills/tailnet-policy +bundles/tailscale/skills/tailscale-client +bundles/workflow-architect +bundles/workflow-architect/skills/bundle-builder +bundles/workflow-architect/skills/interviewer +bundles/workflow-architect/skills/observer +c4-diagramming +chief-of-staff-methodology +cli-builder +color-management +confluence-cli +crewai +crowdsec +daily-life-discovery +data-architect +data-engineering +data-scientist +de-spin +docker-compose +dspy +epub +esp32-development +financial-modeling +fireflies +flaresolverr +flaresolverr-cli +forgejo-cli +frontend-engineering +ghost-cli +github-runner +go-to-market +gutenberg +haystack +hugo-theme +jellyfin-cli +jira-cli +jira-jql +kanban-guru +kubernetes +langchain +langgraph +lastfm +legal-strategy +linear +llamaindex +mermaid-diagrams +meshcore-packet-capture +ml-engineering +nous-branding +open-knowledge-format +openlibrary-cli +opensource-contributions +operational-design +org-design +peertube +platform-engineering +product-design-and-ux +product-discovery +product-methodology +product-strategy +programming-principles +pydanticai +qa-methodology +raleigh +remote-systems-administration +research-methodology +restic +secure-software-engineering +security-audit-methodology +seo-audit +site-reliability-engineering +software-architecture-analysis +spec-driven-development +strategy-frameworks +supabase +systematic-debugging +technical-documentation +technology-radar +tempest-cli +three +tmdb-cli +traefik +trakt +transistor +vercel-eve +verification-methodology +web-accessibility +woodpecker-ci +yc-default-alive-calculator +yc-weekly-growth-compass diff --git a/scripts/validate-skills.rb b/scripts/validate-skills.rb index 83acd31..aab38f9 100755 --- a/scripts/validate-skills.rb +++ b/scripts/validate-skills.rb @@ -2,10 +2,14 @@ # frozen_string_literal: true require "yaml" +require "set" +require "json" ROOT = File.expand_path("..", __dir__) ALLOWED_FIELDS = %w[name description license compatibility metadata allowed-tools].freeze README_HEADINGS = ["Why Install This Skill", "What You Get", "Quick Start", "Triggers", "Requirements"].freeze +GRANDFATHER_FILE = File.join(ROOT, "scripts", "grandfathered-skills.txt") +MIN_EVAL_CASES = 5 errors = [] skills = Dir.glob("#{ROOT}/**/SKILL.md").sort.reject do |skill| @@ -44,6 +48,8 @@ unless mislabeled_catalog_entries.empty? errors << "README.md: catalog label/path mismatch(es): #{labels.join(', ')}" end +grandfathered = File.exist?(GRANDFATHER_FILE) ? File.readlines(GRANDFATHER_FILE, chomp: true).reject { |l| l.strip.empty? || l.start_with?("#") }.to_set : Set.new + skills.each do |skill| relative = skill.delete_prefix("#{ROOT}/") text = File.read(skill) @@ -96,6 +102,25 @@ skills.each do |skill| README_HEADINGS.each do |heading| errors << "#{relative}: README missing #{heading}" unless readme_text.match?(/^#+ #{Regexp.escape(heading)}\s*$/) end + + # Phase 1: new skills (not grandfathered) must have evals with >= MIN_EVAL_CASES cases + skill_dir = File.dirname(skill).delete_prefix("#{ROOT}/") + unless grandfathered.include?(skill_dir) + evals_path = File.join(root, "evals", "evals.json") + unless File.file?(evals_path) + errors << "#{relative}: new skill must have evals/evals.json (not in grandfathered-skills.txt)" + next + end + begin + evals_data = JSON.parse(File.read(evals_path)) + case_count = evals_data.is_a?(Hash) && evals_data["evals"].is_a?(Array) ? evals_data["evals"].length : 0 + if case_count < MIN_EVAL_CASES + errors << "#{relative}: evals/evals.json has #{case_count} case(s), minimum is #{MIN_EVAL_CASES}" + end + rescue JSON::ParserError => e + errors << "#{relative}: evals/evals.json is invalid JSON: #{e.message.lines.first.strip}" + end + end end if errors.empty?