magnus919_agent-skills/tests/integration/test_validation_pipeline.py

"""Integration tests for the agent-skills validation pipeline.

Tests the full validation flow end-to-end: creating a skill, validating its
structure, checking eval manifests, and verifying coverage reporting.
"""

from __future__ import annotations

import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

SCRIPT_DIR = Path(__file__).resolve().parents[2] / "scripts"
ROOT = SCRIPT_DIR.parent

sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(SCRIPT_DIR))

from eval_validation import find_skill_manifests, validate_manifest


def _git(repo: Path, *args: str) -> None:
    subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True)


def _make_skill_dir(tmp: Path, name: str) -> Path:
    """Create a minimal valid skill directory."""
    skill = tmp / name
    skill.mkdir(parents=True)

    # SKILL.md
    (skill / "SKILL.md").write_text(
        f"---\nname: {name}\ndescription: Test skill for integration testing.\n---\n\n"
        "# {name}\n\nA test skill for the integration test suite.\n"
    )

    # README.md
    (skill / "README.md").write_text(
        f"# {name}\n\n## Why Install This Skill\n\n"
        "This skill helps with testing the validation pipeline.\n\n"
        "## What You Get\n\n"
        "| File | Purpose |\n|------|--------|\n| SKILL.md | Instructions |\n\n"
        "## Quick Start\n\nNo setup needed.\n\n"
        "## Triggers\n\n- When integration tests run\n\n"
        "## Requirements\n\nPython 3.10+\n"
    )

    # evals/evals.json
    (skill / "evals").mkdir(exist_ok=True)
    manifest = {
        "schema_version": 1,
        "skill_name": name,
        "evals": [
            {
                "id": "int-01",
                "prompt": "Run the skill.",
                "expected_output": "The skill ran successfully.",
                "assertions": ["Skill completes without error."],
            },
            {
                "id": "int-02",
                "prompt": "Test edge case.",
                "expected_output": "Edge case handled.",
                "assertions": ["No crash on edge input."],
            },
            {
                "id": "int-03",
                "prompt": "Verify output format.",
                "expected_output": "Valid JSON output.",
                "assertions": ["Output is valid JSON."],
            },
            {
                "id": "int-04",
                "prompt": "Test with empty input.",
                "expected_output": "Graceful handling.",
                "assertions": ["Returns appropriate error."],
            },
            {
                "id": "int-05",
                "prompt": "Test concurrent access.",
                "expected_output": "Thread-safe operation.",
                "assertions": ["No race conditions."],
            },
        ],
    }
    (skill / "evals" / "evals.json").write_text(json.dumps(manifest, indent=2))

    return skill


class TestValidateEvalsIntegration(unittest.TestCase):
    """Integration tests for eval validation."""

    def setUp(self) -> None:
        self.tmp = tempfile.TemporaryDirectory()
        self.root = Path(self.tmp.name)
        _git(self.root, "init", "-q")
        _git(self.root, "config", "user.email", "test@example.invalid")
        _git(self.root, "config", "user.name", "Test")

    def tearDown(self) -> None:
        self.tmp.cleanup()

    def test_valid_skill_passes_validation(self) -> None:
        """A properly constructed skill should pass eval validation."""
        skill = _make_skill_dir(self.root, "test-skill")
        manifest_path = skill / "evals" / "evals.json"

        self.assertTrue(manifest_path.exists())
        result = validate_manifest(manifest_path, self.root)
        self.assertEqual(len(result.errors), 0, f"Validation errors: {result.errors}")
        self.assertEqual(result.states.get("schema_valid"), True)

    def test_manifest_with_missing_evals_fails(self) -> None:
        """A manifest without the 'evals' key should fail validation."""
        skill = _make_skill_dir(self.root, "bad-skill")
        (skill / "evals" / "evals.json").write_text(
            json.dumps({"schema_version": 1, "skill_name": "bad-skill"})
        )
        result = validate_manifest(skill / "evals" / "evals.json", self.root)
        self.assertGreater(len(result.errors), 0)

    def test_manifest_with_empty_evals_fails(self) -> None:
        """A manifest with empty evals array should fail."""
        skill = _make_skill_dir(self.root, "empty-skill")
        (skill / "evals" / "evals.json").write_text(
            json.dumps(
                {
                    "schema_version": 1,
                    "skill_name": "empty-skill",
                    "evals": [],
                }
            )
        )
        result = validate_manifest(skill / "evals" / "evals.json", self.root)
        self.assertGreater(len(result.errors), 0)

    def test_manifest_with_invalid_schema_version_fails(self) -> None:
        """A manifest with wrong schema_version should fail."""
        skill = _make_skill_dir(self.root, "wrong-version")
        manifest = json.loads((skill / "evals" / "evals.json").read_text())
        manifest["schema_version"] = 99
        (skill / "evals" / "evals.json").write_text(json.dumps(manifest))
        result = validate_manifest(skill / "evals" / "evals.json", self.root)
        self.assertGreater(len(result.errors), 0)
        self.assertNotEqual(result.states.get("schema_valid"), True)

    def test_find_skill_manifests_discovers_all(self) -> None:
        """find_skill_manifests should discover all skill eval manifests."""
        _make_skill_dir(self.root, "skill-a")
        _make_skill_dir(self.root, "skill-b")

        manifests = find_skill_manifests(self.root)
        self.assertGreaterEqual(len(manifests), 2)

        names = {m.parent.parent.name for m in manifests}
        self.assertIn("skill-a", names)
        self.assertIn("skill-b", names)

    def test_skill_without_evals_dir_is_handled(self) -> None:
        """Skills without evals dir should not cause errors in discovery."""
        skill = self.root / "no-evals-skill"
        skill.mkdir()
        (skill / "SKILL.md").write_text(
            "---\nname: no-evals-skill\ndescription: Test.\n---\n# Test\n"
        )
        manifests = find_skill_manifests(self.root)
        no_eval_paths = [m for m in manifests if "no-evals-skill" in str(m)]
        self.assertEqual(len(no_eval_paths), 0)


class TestValidationPipelineEndToEnd(unittest.TestCase):
    """End-to-end tests for the full validation pipeline."""

    def setUp(self) -> None:
        self.tmp = tempfile.TemporaryDirectory()
        self.root = Path(self.tmp.name)
        _git(self.root, "init", "-q")
        _git(self.root, "config", "user.email", "test@example.invalid")
        _git(self.root, "config", "user.name", "Test")

    def tearDown(self) -> None:
        self.tmp.cleanup()

    def test_validate_evals_accepts_valid_skill(self) -> None:
        """validate-evals.py should accept a valid skill."""
        _make_skill_dir(self.root, "test-skill")
        _git(self.root, "add", "test-skill/")
        _git(self.root, "commit", "-m", "add skill")

        result = subprocess.run(
            [sys.executable, str(SCRIPT_DIR / "validate-evals.py")],
            cwd=self.root,
            capture_output=True,
            text=True,
        )
        self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")

    def test_eval_coverage_reports_on_skills(self) -> None:
        """eval-coverage.py should report coverage for skills."""
        _make_skill_dir(self.root, "covered-skill")
        _git(self.root, "add", "covered-skill/")
        _git(self.root, "commit", "-m", "add covered skill")

        result = subprocess.run(
            [sys.executable, str(SCRIPT_DIR / "eval-coverage.py")],
            cwd=self.root,
            capture_output=True,
            text=True,
        )
        # Coverage script reports (always exits 0 for informational mode)
        self.assertEqual(result.returncode, 0)
        self.assertIn("schema-valid", result.stdout.lower())


class TestScanTechDebt(unittest.TestCase):
    """Integration test for the tech debt scanner."""

    def setUp(self) -> None:
        self.tmp = tempfile.TemporaryDirectory()
        self.root = Path(self.tmp.name)
        _git(self.root, "init", "-q")
        _git(self.root, "config", "user.email", "test@example.invalid")
        _git(self.root, "config", "user.name", "Test")

    def tearDown(self) -> None:
        self.tmp.cleanup()

    def test_detects_todo_markers(self) -> None:
        """scan-tech-debt.py should detect TODO markers in Python files."""
        test_file = self.root / "module.py"
        test_file.write_text("# TODO(#42): Fix this later\ndef foo(): pass\n")
        _git(self.root, "add", "module.py")

        result = subprocess.run(
            [sys.executable, str(SCRIPT_DIR / "scan-tech-debt.py")],
            cwd=self.root,
            capture_output=True,
            text=True,
        )
        self.assertIn("TODO", result.stdout)

    def test_reports_zero_for_clean_code(self) -> None:
        """scan-tech-debt.py should report zero for clean code."""
        test_file = self.root / "clean.py"
        test_file.write_text("def foo():\n    return 42\n")
        _git(self.root, "add", "clean.py")

        result = subprocess.run(
            [sys.executable, str(SCRIPT_DIR / "scan-tech-debt.py")],
            cwd=self.root,
            capture_output=True,
            text=True,
        )
        self.assertIn("No technical debt markers found", result.stdout)