magnus919_agent-skills/eval_runner/tests/test_release.py

"""Tests for release-grade evaluation: trials, rubric, pairwise, release matrix."""

from __future__ import annotations

import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))

from eval_runner.release import (
    CalibrationRecord,
    CaseAggregation,
    FreezeSnapshot,
    RubricGrader,
    TrialRecord,
    aggregate_trials,
    apply_rubric,
    build_release_report,
    compute_release_decision,
    dataset_hash,
    evaluate_pairwise,
    plan_pairwise,
    write_release_report,
)


def _trial(
    case_id: str, status: str = "completed", passed: bool = True, missing: list[str] | None = None
) -> dict:
    return {
        "trial_id": f"t-{case_id}-{status}",
        "case": {"case_id": case_id, "prompt_hash": "abc", "fixture_hashes": {}},
        "status": status,
        "failures": []
        if passed and status == "completed"
        else [{"type": "assertion", "message": "fail"}],
        "missing_evidence": missing or [],
    }


def test_aggregate_trials_groups_by_case():
    manifests = [
        _trial("case-a", passed=True),
        _trial("case-a", passed=False),
        _trial("case-b", passed=True),
    ]
    results = aggregate_trials(manifests)
    assert len(results) == 2
    case_a = next(r for r in results if r.case_id == "case-a")
    assert case_a.trial_count == 2
    assert case_a.success_count == 1
    assert case_a.failure_count == 1
    assert case_a.success_frequency == 0.5
    assert not case_a.consistent


def test_aggregate_trials_consistency():
    manifests = [_trial("c1", passed=True), _trial("c1", passed=True), _trial("c1", passed=True)]
    results = aggregate_trials(manifests)
    assert results[0].consistent
    assert results[0].success_frequency == 1.0


def test_aggregate_trials_counts_errors_and_timeouts():
    manifests = [
        _trial("c1", status="completed", passed=True),
        _trial("c1", status="error", passed=False),
        _trial("c1", status="timeout", passed=False),
    ]
    results = aggregate_trials(manifests)
    agg = results[0]
    assert agg.success_count == 1
    assert agg.error_count == 1
    assert agg.timeout_count == 1
    assert agg.trial_count == 3


def test_aggregate_trials_preserves_all_failures():
    manifests = [
        _trial("c1", passed=False),
        _trial("c1", passed=False),
        _trial("c1", status="error", passed=False),
    ]
    results = aggregate_trials(manifests)
    agg = results[0]
    assert agg.failure_count == 2
    assert agg.error_count == 1
    assert agg.success_count == 0


def test_aggregate_trials_missing_evidence():
    manifests = [
        _trial("c1", passed=True, missing=["response"]),
        _trial("c1", passed=True, missing=["token_usage"]),
    ]
    results = aggregate_trials(manifests)
    assert set(results[0].all_missing_evidence) == {"response", "token_usage"}


def test_aggregate_trials_case_sets():
    manifests = [_trial("c1"), _trial("c2")]
    results = aggregate_trials(manifests, case_sets={"c1": "release", "c2": "regression"})
    assert results[0].case_set == "release"
    assert results[1].case_set == "regression"


def test_case_aggregation_to_dict():
    agg = CaseAggregation(
        case_id="c1",
        case_set="release",
        trials=[
            TrialRecord("t1", "c1", "completed", True),
            TrialRecord("t2", "c1", "completed", True),
        ],
    )
    d = agg.to_dict(paired_delta="both_pass")
    assert d["case_id"] == "c1"
    assert d["case_set"] == "release"
    assert d["trial_count"] == 2
    assert d["success_frequency"] == 1.0
    assert d["consistent"] is True
    assert d["paired_delta"] == "both_pass"


def test_rubric_grader_pass():
    grader = RubricGrader("test-grader", "1.0", ("relevance", "completeness"))
    result = grader.grade(
        "the expected output is here and complete", "expected output is here and complete"
    )
    assert result["verdict"] == "pass"
    assert result["grader_id"] == "test-grader"
    assert result["grader_version"] == "1.0"
    assert result["blinded"] is True


def test_rubric_grader_abstain_on_empty():
    grader = RubricGrader("test-grader", "1.0", ("relevance",))
    result = grader.grade("   ", "expected")
    assert result["verdict"] == "abstain"


def test_rubric_grader_insufficient_evidence_on_none():
    grader = RubricGrader("test-grader", "1.0", ("relevance",))
    result = grader.grade(None, "expected")
    assert result["verdict"] == "insufficient_evidence"


def test_rubric_grader_versioned():
    g1 = RubricGrader("g", "1.0", ("relevance",))
    g2 = RubricGrader("g", "2.0", ("relevance",))
    r1 = g1.grade("hello world", "hello world")
    r2 = g2.grade("hello world", "hello world")
    assert r1["grader_version"] == "1.0"
    assert r2["grader_version"] == "2.0"


def test_apply_rubric_adds_case_id():
    grader = RubricGrader("g", "1", ("relevance",))
    result = apply_rubric(grader, "case-42", "response text", "expected text")
    assert result["case_id"] == "case-42"


def test_plan_pairwise_deterministic():
    plans1 = plan_pairwise(["c1", "c2", "c3"], seed=42)
    plans2 = plan_pairwise(["c1", "c2", "c3"], seed=42)
    assert [(p.case_id, p.position_a) for p in plans1] == [
        (p.case_id, p.position_a) for p in plans2
    ]


def test_plan_pairwise_blinded_positions():
    plans = plan_pairwise(["c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8"], seed=99)
    positions = [p.position_a for p in plans]
    assert "candidate" in positions
    assert "baseline" in positions


def test_plan_pairwise_reversal():
    plan = plan_pairwise(["c1"], seed=1)[0]
    rev = plan.reversed
    assert rev.position_a == plan.position_b
    assert rev.position_b == plan.position_a


def test_evaluate_pairwise_order_reversal():
    plan = plan_pairwise(["c1"], seed=7)[0]
    result = evaluate_pairwise(
        plan,
        response_a="the complete expected answer with all details",
        response_b="partial answer",
        expected="the complete expected answer with all details",
    )
    assert result["order_reversal_tested"] is True
    assert result["order_reversal_consistent"] is not None
    assert result["case_id"] == "c1"


def test_evaluate_pairwise_abstain_on_none():
    plan = plan_pairwise(["c1"], seed=1)[0]
    result = evaluate_pairwise(plan, response_a=None, response_b="something", expected="something")
    assert result["winner"] == "abstain"


def test_calibration_not_calibrated():
    cal = CalibrationRecord()
    assert not cal.calibrated
    d = cal.to_dict()
    assert d["calibrated"] is False


def test_calibration_calibrated():
    cal = CalibrationRecord(human_sample_count=50, judge_agreement_rate=0.85)
    assert cal.calibrated
    d = cal.to_dict()
    assert d["calibrated"] is True
    assert d["judge_agreement_rate"] == 0.85


def test_calibration_low_agreement_not_calibrated():
    cal = CalibrationRecord(human_sample_count=50, judge_agreement_rate=0.5)
    assert not cal.calibrated


def test_release_decision_pass():
    cases = [
        {
            "case_id": "c1",
            "case_set": "release",
            "success_frequency": 1.0,
            "consistent": True,
            "missing_evidence": [],
            "paired_delta": "both_pass",
        },
    ]
    cal = CalibrationRecord(human_sample_count=20, judge_agreement_rate=0.9)
    decision = compute_release_decision(cases, [], cal)
    assert decision["outcome"] == "PASS"


def test_release_decision_block_on_hard_gate():
    decision = compute_release_decision(
        [], [], CalibrationRecord(), hard_gate_violations=["privacy violation"]
    )
    assert decision["outcome"] == "BLOCK"
    assert "privacy violation" in decision["reasons"][0]


def test_release_decision_hold_on_missing_evidence():
    cases = [
        {
            "case_id": "c1",
            "case_set": "release",
            "success_frequency": 1.0,
            "consistent": True,
            "missing_evidence": ["response"],
            "paired_delta": "both_pass",
        },
    ]
    decision = compute_release_decision(cases, [], CalibrationRecord())
    assert decision["outcome"] == "HOLD"


def test_release_decision_block_on_low_frequency():
    cases = [
        {
            "case_id": "c1",
            "case_set": "release",
            "success_frequency": 0.5,
            "consistent": False,
            "missing_evidence": [],
            "paired_delta": "both_pass",
        },
    ]
    cal = CalibrationRecord(human_sample_count=20, judge_agreement_rate=0.9)
    decision = compute_release_decision(cases, [], cal)
    assert decision["outcome"] == "BLOCK"


def test_release_decision_block_on_regression():
    cases = [
        {
            "case_id": "c1",
            "case_set": "regression",
            "success_frequency": 1.0,
            "consistent": True,
            "missing_evidence": [],
            "paired_delta": "candidate_regression",
        },
    ]
    cal = CalibrationRecord(human_sample_count=20, judge_agreement_rate=0.9)
    decision = compute_release_decision(cases, [], cal)
    assert decision["outcome"] == "BLOCK"


def test_release_decision_conditional_on_inconsistency():
    cases = [
        {
            "case_id": "c1",
            "case_set": "release",
            "success_frequency": 0.9,
            "consistent": False,
            "missing_evidence": [],
            "paired_delta": "both_pass",
        },
    ]
    cal = CalibrationRecord(human_sample_count=20, judge_agreement_rate=0.9)
    decision = compute_release_decision(cases, [], cal)
    assert decision["outcome"] == "CONDITIONAL"


def test_release_decision_conditional_on_uncalibrated_judge():
    cases = [
        {
            "case_id": "c1",
            "case_set": "release",
            "success_frequency": 1.0,
            "consistent": True,
            "missing_evidence": [],
            "paired_delta": "both_pass",
        },
    ]
    rubric = [{"case_id": "c1", "verdict": "pass"}]
    cal = CalibrationRecord()
    decision = compute_release_decision(cases, rubric, cal)
    assert decision["outcome"] == "CONDITIONAL"
    assert any("advisory" in r for r in decision["reasons"])


def test_release_decision_conditional_on_rubric_abstain():
    cases = [
        {
            "case_id": "c1",
            "case_set": "release",
            "success_frequency": 1.0,
            "consistent": True,
            "missing_evidence": [],
            "paired_delta": "both_pass",
        },
    ]
    rubric = [{"case_id": "c1", "verdict": "abstain"}]
    cal = CalibrationRecord(human_sample_count=20, judge_agreement_rate=0.9)
    decision = compute_release_decision(cases, rubric, cal)
    assert decision["outcome"] == "CONDITIONAL"


def test_freeze_snapshot_completeness():
    freeze = FreezeSnapshot(
        candidate_tree_hash="abc123",
        baseline_tree_hash="def456",
        dataset_hash="ghi789",
        grader_versions={"default": "1.0"},
        randomization_seed=42,
    )
    assert freeze.complete
    d = freeze.to_dict()
    assert d["candidate_tree_hash"] == "abc123"
    assert d["randomization_seed"] == 42


def test_freeze_snapshot_incomplete():
    freeze = FreezeSnapshot(
        candidate_tree_hash="",
        baseline_tree_hash="def456",
        dataset_hash="ghi789",
        grader_versions={},
        randomization_seed=42,
    )
    assert not freeze.complete


def test_build_release_report_structure():
    freeze = FreezeSnapshot("a", "b", "c", {"g": "1"}, 42)
    cal = CalibrationRecord(human_sample_count=10, judge_agreement_rate=0.8)
    report = build_release_report(
        skill_name="test-skill",
        freeze=freeze,
        case_results=[],
        rubric_results=[],
        pairwise_results=[],
        calibration=cal,
    )
    assert report["schema_version"] == 1
    assert report["skill_name"] == "test-skill"
    assert "report_id" in report
    assert "generated_at" in report
    assert report["release_decision"]["outcome"] in ("PASS", "CONDITIONAL", "HOLD", "BLOCK")


def test_build_release_report_validates_against_schema():
    try:
        from jsonschema import Draft202012Validator
    except ImportError:
        print("SKIP: jsonschema not installed")
        return

    schema_path = (
        Path(__file__).resolve().parent.parent.parent / "schemas" / "release-eval-v1.schema.json"
    )
    schema = json.loads(schema_path.read_text())
    Draft202012Validator.check_schema(schema)
    validator = Draft202012Validator(schema)

    freeze = FreezeSnapshot("abc", "def", "ghi", {"default-pairwise": "1"}, 42)
    cal = CalibrationRecord(human_sample_count=10, judge_agreement_rate=0.8)
    case_results = [
        {
            "case_id": "c1",
            "case_set": "release",
            "trial_count": 3,
            "success_count": 3,
            "failure_count": 0,
            "error_count": 0,
            "timeout_count": 0,
            "success_frequency": 1.0,
            "consistent": True,
            "missing_evidence": [],
            "paired_delta": "both_pass",
        }
    ]
    rubric_results = [
        {
            "case_id": "c1",
            "grader_id": "default-pairwise",
            "grader_version": "1",
            "blinded": True,
            "verdict": "pass",
            "scores": {"relevance": 0.9},
            "rationale": "good",
        }
    ]
    pairwise_results = [
        {
            "case_id": "c1",
            "position_a": "candidate",
            "position_b": "baseline",
            "winner": "a",
            "order_reversal_tested": True,
            "order_reversal_consistent": True,
        }
    ]

    report = build_release_report(
        skill_name="test-skill",
        freeze=freeze,
        case_results=case_results,
        rubric_results=rubric_results,
        pairwise_results=pairwise_results,
        calibration=cal,
    )

    errors = list(validator.iter_errors(report))
    assert not errors, f"Schema validation failed: {[e.message for e in errors]}"


def test_release_schema_matches_runtime_case_ids():
    try:
        from jsonschema import Draft202012Validator
    except ImportError:
        print("SKIP: jsonschema not installed")
        return

    schema_path = (
        Path(__file__).resolve().parent.parent.parent / "schemas" / "release-eval-v1.schema.json"
    )
    schema = json.loads(schema_path.read_text())
    for definition in ["case_result", "rubric_result", "pairwise_result"]:
        case_id_schema = schema["$defs"][definition]["properties"]["case_id"]
        validator = Draft202012Validator(case_id_schema)
        assert not list(validator.iter_errors("valid-case-1")), definition
        for unsafe_case_id in ["../case", "Uppercase", "case_id", "case\n"]:
            assert list(validator.iter_errors(unsafe_case_id)), (
                definition,
                unsafe_case_id,
            )


def test_write_release_report():
    import tempfile

    freeze = FreezeSnapshot("a", "b", "c", {"g": "1"}, 42)
    cal = CalibrationRecord()
    report = build_release_report(
        skill_name="my-skill",
        freeze=freeze,
        case_results=[],
        rubric_results=[],
        pairwise_results=[],
        calibration=cal,
    )
    with tempfile.TemporaryDirectory() as tmp:
        path = write_release_report(report, Path(tmp))
        assert path.is_file()
        assert "my-skill" in path.name
        loaded = json.loads(path.read_text())
        assert loaded["schema_version"] == 1


def test_dataset_hash_deterministic():
    import tempfile

    with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
        f.write('{"evals": []}')
        f.flush()
        h1 = dataset_hash(Path(f.name))
        h2 = dataset_hash(Path(f.name))
        assert h1 == h2
        assert len(h1) == 16
    Path(f.name).unlink()


if __name__ == "__main__":
    test_aggregate_trials_groups_by_case()
    test_aggregate_trials_consistency()
    test_aggregate_trials_counts_errors_and_timeouts()
    test_aggregate_trials_preserves_all_failures()
    test_aggregate_trials_missing_evidence()
    test_aggregate_trials_case_sets()
    test_case_aggregation_to_dict()
    test_rubric_grader_pass()
    test_rubric_grader_abstain_on_empty()
    test_rubric_grader_insufficient_evidence_on_none()
    test_rubric_grader_versioned()
    test_apply_rubric_adds_case_id()
    test_plan_pairwise_deterministic()
    test_plan_pairwise_blinded_positions()
    test_plan_pairwise_reversal()
    test_evaluate_pairwise_order_reversal()
    test_evaluate_pairwise_abstain_on_none()
    test_calibration_not_calibrated()
    test_calibration_calibrated()
    test_calibration_low_agreement_not_calibrated()
    test_release_decision_pass()
    test_release_decision_block_on_hard_gate()
    test_release_decision_hold_on_missing_evidence()
    test_release_decision_block_on_low_frequency()
    test_release_decision_block_on_regression()
    test_release_decision_conditional_on_inconsistency()
    test_release_decision_conditional_on_uncalibrated_judge()
    test_release_decision_conditional_on_rubric_abstain()
    test_freeze_snapshot_completeness()
    test_freeze_snapshot_incomplete()
    test_build_release_report_structure()
    test_build_release_report_validates_against_schema()
    test_release_schema_matches_runtime_case_ids()
    test_write_release_report()
    test_dataset_hash_deterministic()
    print("All release evaluation tests passed.")