magnus919_agent-skills/scripts/test-eval-coverage.py

#!/usr/bin/env python3
"""Tests for scripts/eval-coverage.py ratchet logic.

Uses a temporary git repository to verify modified-skill detection,
coverage-decrease detection, and threshold behavior.
"""

import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

# Import the module under test.  The script is named eval-coverage.py
# (hyphenated), so we load it via importlib rather than a normal import.
import importlib.util  # noqa: E402

SCRIPT_DIR = Path(__file__).resolve().parent
_spec = importlib.util.spec_from_file_location(
    "eval_coverage", SCRIPT_DIR / "eval-coverage.py"
)
eval_coverage = importlib.util.module_from_spec(_spec)
sys.modules["eval_coverage"] = eval_coverage
_spec.loader.exec_module(eval_coverage)  # type: ignore[union-attr]


def git(repo: str, *args: str) -> str:
    """Run a git command in *repo* and return stdout."""
    result = subprocess.run(
        ["git", *args],
        cwd=repo,
        capture_output=True,
        text=True,
        check=True,
    )
    return result.stdout.strip()


def write_skill(repo: str, name: str, evals: list[dict] | None = None) -> None:
    """Create a minimal skill directory with an optional evals manifest."""
    skill_dir = Path(repo) / name
    skill_dir.mkdir(parents=True, exist_ok=True)
    (skill_dir / "SKILL.md").write_text(
        f"---\nname: {name}\ndescription: Test skill {name}.\n---\n\n# {name}\n"
    )
    if evals is not None:
        evals_dir = skill_dir / "evals"
        evals_dir.mkdir(exist_ok=True)
        (evals_dir / "evals.json").write_text(
            json.dumps({"skill_name": name, "evals": evals}, indent=2)
        )


def make_case(case_id: str) -> dict:
    return {"id": case_id, "prompt": "test", "expected_output": "ok", "assertions": ["a"]}


class TestModifiedSkills(unittest.TestCase):
    """modified_skills() must detect changes to any file under a skill dir."""

    def setUp(self) -> None:
        self.tmp = tempfile.TemporaryDirectory()
        self.repo = self.tmp.name
        git(self.repo, "init", "-b", "main")
        git(self.repo, "config", "user.email", "test@test.invalid")
        git(self.repo, "config", "user.name", "Test")

    def tearDown(self) -> None:
        self.tmp.cleanup()

    def _commit(self, msg: str) -> str:
        git(self.repo, "add", "-A")
        git(self.repo, "commit", "-m", msg)
        return git(self.repo, "rev-parse", "HEAD")

    def test_skill_md_change_detected(self) -> None:
        write_skill(self.repo, "alpha")
        base = self._commit("initial")
        (Path(self.repo) / "alpha" / "SKILL.md").write_text("changed")
        self._commit("edit SKILL.md")
        old_root = eval_coverage.ROOT
        eval_coverage.ROOT = Path(self.repo)
        try:
            mods = eval_coverage.modified_skills(base)
        finally:
            eval_coverage.ROOT = old_root
        self.assertIn(Path("alpha"), mods)

    def test_reference_change_detected(self) -> None:
        write_skill(self.repo, "beta")
        ref_dir = Path(self.repo) / "beta" / "references"
        ref_dir.mkdir()
        (ref_dir / "guide.md").write_text("v1")
        base = self._commit("initial")
        (ref_dir / "guide.md").write_text("v2")
        self._commit("edit reference")
        old_root = eval_coverage.ROOT
        eval_coverage.ROOT = Path(self.repo)
        try:
            mods = eval_coverage.modified_skills(base)
        finally:
            eval_coverage.ROOT = old_root
        self.assertIn(Path("beta"), mods)

    def test_eval_manifest_deletion_detected(self) -> None:
        write_skill(self.repo, "gamma", evals=[make_case("c1")])
        base = self._commit("initial")
        (Path(self.repo) / "gamma" / "evals" / "evals.json").unlink()
        self._commit("delete evals")
        old_root = eval_coverage.ROOT
        eval_coverage.ROOT = Path(self.repo)
        try:
            mods = eval_coverage.modified_skills(base)
        finally:
            eval_coverage.ROOT = old_root
        self.assertIn(Path("gamma"), mods)

    def test_unrelated_file_not_detected(self) -> None:
        write_skill(self.repo, "delta")
        (Path(self.repo) / "README.md").write_text("root readme")
        base = self._commit("initial")
        (Path(self.repo) / "README.md").write_text("changed readme")
        self._commit("edit root readme")
        old_root = eval_coverage.ROOT
        eval_coverage.ROOT = Path(self.repo)
        try:
            mods = eval_coverage.modified_skills(base)
        finally:
            eval_coverage.ROOT = old_root
        self.assertNotIn(Path("delta"), mods)


class TestCoverageDecreased(unittest.TestCase):
    """coverage_decreased() must detect a drop in eval coverage."""

    def setUp(self) -> None:
        self.tmp = tempfile.TemporaryDirectory()
        self.repo = self.tmp.name
        git(self.repo, "init", "-b", "main")
        git(self.repo, "config", "user.email", "test@test.invalid")
        git(self.repo, "config", "user.name", "Test")

    def tearDown(self) -> None:
        self.tmp.cleanup()

    def _commit(self, msg: str) -> str:
        git(self.repo, "add", "-A")
        git(self.repo, "commit", "-m", msg)
        return git(self.repo, "rev-parse", "HEAD")

    def test_decrease_detected(self) -> None:
        write_skill(self.repo, "s1", evals=[make_case("c1")])
        write_skill(self.repo, "s2", evals=[make_case("c2")])
        base = self._commit("two skills with evals")
        # Remove evals from s2.
        (Path(self.repo) / "s2" / "evals" / "evals.json").unlink()
        self._commit("remove s2 evals")
        old_root = eval_coverage.ROOT
        eval_coverage.ROOT = Path(self.repo)
        try:
            decreased, base_pct, head_pct = eval_coverage.coverage_decreased(base)
        finally:
            eval_coverage.ROOT = old_root
        self.assertTrue(decreased)
        self.assertAlmostEqual(base_pct, 100.0)
        self.assertAlmostEqual(head_pct, 50.0)

    def test_no_decrease_when_stable(self) -> None:
        write_skill(self.repo, "s1", evals=[make_case("c1")])
        write_skill(self.repo, "s2")
        base = self._commit("one with evals, one without")
        (Path(self.repo) / "s2" / "SKILL.md").write_text("edited")
        self._commit("edit s2")
        old_root = eval_coverage.ROOT
        eval_coverage.ROOT = Path(self.repo)
        try:
            decreased, base_pct, head_pct = eval_coverage.coverage_decreased(base)
        finally:
            eval_coverage.ROOT = old_root
        self.assertFalse(decreased)
        self.assertAlmostEqual(base_pct, 50.0)
        self.assertAlmostEqual(head_pct, 50.0)

    def test_increase_not_flagged(self) -> None:
        write_skill(self.repo, "s1")
        base = self._commit("no evals")
        write_skill(self.repo, "s1", evals=[make_case("c1")])
        self._commit("add evals")
        old_root = eval_coverage.ROOT
        eval_coverage.ROOT = Path(self.repo)
        try:
            decreased, base_pct, head_pct = eval_coverage.coverage_decreased(base)
        finally:
            eval_coverage.ROOT = old_root
        self.assertFalse(decreased)
        self.assertAlmostEqual(base_pct, 0.0)
        self.assertAlmostEqual(head_pct, 100.0)


if __name__ == "__main__":
    unittest.main()