mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-11 19:47:12 +03:00
usernameandfactory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
bfe05ef18a
Add mypy strict type checking, radon cyclomatic complexity analysis, and deptry unused dependency detection. Fix type annotations in eval-coverage.py and release.py to pass strict mypy checks. Resolves 4 agent-readiness signals: type_check, strict_typing, cyclomatic_complexity, unused_dependencies_detection. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
491 lines
15 KiB
Python
491 lines
15 KiB
Python
"""Release-grade evaluation: repeated trials, rubric graders, and release matrix.
|
|
|
|
Builds on the deterministic paired evaluation infrastructure to add:
|
|
- Multi-trial aggregation per case (success frequency, consistency)
|
|
- Versioned rubric graders with abstain/insufficient-evidence
|
|
- Blinded pairwise comparison with position randomization and order-reversal
|
|
- Human calibration tracking with disagreement slices
|
|
- Case set separation (dev, regression, release)
|
|
- Freeze snapshot validation
|
|
- Release decision: PASS, CONDITIONAL, HOLD, BLOCK
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import random
|
|
import uuid
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
RELEASE_SCHEMA_VERSION = 1
|
|
|
|
CASE_SETS = ("dev", "regression", "release")
|
|
|
|
RELEASE_OUTCOMES = ("PASS", "CONDITIONAL", "HOLD", "BLOCK")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FreezeSnapshot:
|
|
candidate_tree_hash: str
|
|
baseline_tree_hash: str
|
|
dataset_hash: str
|
|
grader_versions: dict[str, str]
|
|
randomization_seed: int
|
|
exclusions: tuple[str, ...] = ()
|
|
frozen_at: str = ""
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"candidate_tree_hash": self.candidate_tree_hash,
|
|
"baseline_tree_hash": self.baseline_tree_hash,
|
|
"dataset_hash": self.dataset_hash,
|
|
"grader_versions": dict(self.grader_versions),
|
|
"randomization_seed": self.randomization_seed,
|
|
"exclusions": list(self.exclusions),
|
|
"frozen_at": self.frozen_at or datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
|
|
@property
|
|
def complete(self) -> bool:
|
|
return bool(
|
|
self.candidate_tree_hash
|
|
and self.baseline_tree_hash
|
|
and self.dataset_hash
|
|
and self.grader_versions
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class TrialRecord:
|
|
trial_id: str
|
|
case_id: str
|
|
status: str
|
|
passed: bool
|
|
missing_evidence: list[str] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class CaseAggregation:
|
|
case_id: str
|
|
case_set: str
|
|
trials: list[TrialRecord] = field(default_factory=list)
|
|
|
|
@property
|
|
def trial_count(self) -> int:
|
|
return len(self.trials)
|
|
|
|
@property
|
|
def success_count(self) -> int:
|
|
return sum(1 for t in self.trials if t.status == "completed" and t.passed)
|
|
|
|
@property
|
|
def failure_count(self) -> int:
|
|
return sum(1 for t in self.trials if t.status == "completed" and not t.passed)
|
|
|
|
@property
|
|
def error_count(self) -> int:
|
|
return sum(1 for t in self.trials if t.status == "error")
|
|
|
|
@property
|
|
def timeout_count(self) -> int:
|
|
return sum(1 for t in self.trials if t.status == "timeout")
|
|
|
|
@property
|
|
def success_frequency(self) -> float:
|
|
if not self.trials:
|
|
return 0.0
|
|
return self.success_count / self.trial_count
|
|
|
|
@property
|
|
def consistent(self) -> bool:
|
|
completed = [t for t in self.trials if t.status == "completed"]
|
|
if not completed:
|
|
return False
|
|
first = completed[0].passed
|
|
return all(t.passed == first for t in completed)
|
|
|
|
@property
|
|
def all_missing_evidence(self) -> list[str]:
|
|
seen: set[str] = set()
|
|
result: list[str] = []
|
|
for t in self.trials:
|
|
for m in t.missing_evidence:
|
|
if m not in seen:
|
|
seen.add(m)
|
|
result.append(m)
|
|
return result
|
|
|
|
def to_dict(self, paired_delta: str = "insufficient_data") -> dict[str, Any]:
|
|
return {
|
|
"case_id": self.case_id,
|
|
"case_set": self.case_set,
|
|
"trial_count": self.trial_count,
|
|
"success_count": self.success_count,
|
|
"failure_count": self.failure_count,
|
|
"error_count": self.error_count,
|
|
"timeout_count": self.timeout_count,
|
|
"success_frequency": round(self.success_frequency, 4),
|
|
"consistent": self.consistent,
|
|
"missing_evidence": self.all_missing_evidence,
|
|
"paired_delta": paired_delta,
|
|
}
|
|
|
|
|
|
def aggregate_trials(
|
|
manifests: list[dict[str, Any]],
|
|
case_sets: dict[str, str] | None = None,
|
|
) -> list[CaseAggregation]:
|
|
"""Group run manifests by case_id and aggregate trial outcomes."""
|
|
by_case: dict[str, list[TrialRecord]] = {}
|
|
for m in manifests:
|
|
case_id = m.get("case", {}).get("case_id", "")
|
|
if not case_id:
|
|
continue
|
|
trial = TrialRecord(
|
|
trial_id=m.get("trial_id", ""),
|
|
case_id=case_id,
|
|
status=m.get("status", "error"),
|
|
passed=_manifest_passed(m),
|
|
missing_evidence=m.get("missing_evidence", []),
|
|
)
|
|
by_case.setdefault(case_id, []).append(trial)
|
|
|
|
results = []
|
|
for case_id, trials in sorted(by_case.items()):
|
|
case_set = "dev"
|
|
if case_sets and case_id in case_sets:
|
|
case_set = case_sets[case_id]
|
|
results.append(CaseAggregation(case_id=case_id, case_set=case_set, trials=trials))
|
|
return results
|
|
|
|
|
|
def _manifest_passed(m: dict[str, Any]) -> bool:
|
|
if m.get("status") != "completed":
|
|
return False
|
|
failures = m.get("failures", [])
|
|
return len(failures) == 0
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RubricGrader:
|
|
grader_id: str
|
|
version: str
|
|
criteria: tuple[str, ...]
|
|
|
|
def grade(
|
|
self,
|
|
response: str | None,
|
|
expected: str,
|
|
*,
|
|
blinded: bool = True,
|
|
) -> dict[str, Any]:
|
|
if response is None:
|
|
return self._result("insufficient_evidence", {}, "no response to evaluate", blinded)
|
|
if not response.strip():
|
|
return self._result("abstain", {}, "empty response; cannot judge quality", blinded)
|
|
scores: dict[str, float] = {}
|
|
for criterion in self.criteria:
|
|
scores[criterion] = self._score_criterion(criterion, response, expected)
|
|
avg = sum(scores.values()) / len(scores) if scores else 0.0
|
|
if avg >= 0.7:
|
|
verdict = "pass"
|
|
elif avg >= 0.4:
|
|
verdict = "fail"
|
|
else:
|
|
verdict = "fail"
|
|
rationale = f"mean score {avg:.2f} across {len(self.criteria)} criteria"
|
|
return self._result(verdict, scores, rationale, blinded)
|
|
|
|
def _score_criterion(self, criterion: str, response: str, expected: str) -> float:
|
|
response_lower = response.lower()
|
|
expected_lower = expected.lower()
|
|
expected_tokens = set(expected_lower.split())
|
|
response_tokens = set(response_lower.split())
|
|
if not expected_tokens:
|
|
return 0.5
|
|
overlap = len(expected_tokens & response_tokens) / len(expected_tokens)
|
|
return min(1.0, overlap)
|
|
|
|
def _result(
|
|
self,
|
|
verdict: str,
|
|
scores: dict[str, float],
|
|
rationale: str,
|
|
blinded: bool,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"grader_id": self.grader_id,
|
|
"grader_version": self.version,
|
|
"blinded": blinded,
|
|
"verdict": verdict,
|
|
"scores": scores,
|
|
"rationale": rationale,
|
|
}
|
|
|
|
|
|
def apply_rubric(
|
|
grader: RubricGrader,
|
|
case_id: str,
|
|
response: str | None,
|
|
expected: str,
|
|
*,
|
|
blinded: bool = True,
|
|
) -> dict[str, Any]:
|
|
result = grader.grade(response, expected, blinded=blinded)
|
|
result["case_id"] = case_id
|
|
return result
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PairwisePlan:
|
|
case_id: str
|
|
position_a: str
|
|
position_b: str
|
|
seed: int
|
|
|
|
@property
|
|
def reversed(self) -> PairwisePlan:
|
|
return PairwisePlan(
|
|
case_id=self.case_id,
|
|
position_a=self.position_b,
|
|
position_b=self.position_a,
|
|
seed=self.seed,
|
|
)
|
|
|
|
|
|
def plan_pairwise(case_ids: list[str], seed: int) -> list[PairwisePlan]:
|
|
"""Assign randomized A/B positions for each case, blinded to identity."""
|
|
rng = random.Random(seed)
|
|
plans = []
|
|
for case_id in case_ids:
|
|
if rng.random() < 0.5:
|
|
plans.append(PairwisePlan(case_id, "candidate", "baseline", seed))
|
|
else:
|
|
plans.append(PairwisePlan(case_id, "baseline", "candidate", seed))
|
|
return plans
|
|
|
|
|
|
def evaluate_pairwise(
|
|
plan: PairwisePlan,
|
|
response_a: str | None,
|
|
response_b: str | None,
|
|
expected: str,
|
|
*,
|
|
judge: RubricGrader | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Compare two responses under a blinded pairwise plan."""
|
|
if judge is None:
|
|
judge = RubricGrader("default-pairwise", "1", ("relevance", "completeness"))
|
|
|
|
grade_a = judge.grade(response_a, expected, blinded=True)
|
|
grade_b = judge.grade(response_b, expected, blinded=True)
|
|
|
|
score_a = _mean_score(grade_a)
|
|
score_b = _mean_score(grade_b)
|
|
|
|
if (
|
|
grade_a["verdict"] == "abstain"
|
|
or grade_b["verdict"] == "abstain"
|
|
or grade_a["verdict"] == "insufficient_evidence"
|
|
or grade_b["verdict"] == "insufficient_evidence"
|
|
):
|
|
winner = "abstain"
|
|
elif abs(score_a - score_b) < 0.05:
|
|
winner = "tie"
|
|
elif score_a > score_b:
|
|
winner = "a"
|
|
else:
|
|
winner = "b"
|
|
|
|
reversed_plan = plan.reversed # noqa: F841 (used for future blinded grading)
|
|
grade_rev_a = judge.grade(response_b, expected, blinded=True)
|
|
grade_rev_b = judge.grade(response_a, expected, blinded=True)
|
|
score_rev_a = _mean_score(grade_rev_a)
|
|
score_rev_b = _mean_score(grade_rev_b)
|
|
|
|
if abs(score_rev_a - score_rev_b) < 0.05:
|
|
rev_winner = "tie"
|
|
elif score_rev_a > score_rev_b:
|
|
rev_winner = "a"
|
|
else:
|
|
rev_winner = "b"
|
|
|
|
order_consistent = _winners_consistent(winner, rev_winner)
|
|
|
|
return {
|
|
"case_id": plan.case_id,
|
|
"position_a": plan.position_a,
|
|
"position_b": plan.position_b,
|
|
"winner": winner,
|
|
"order_reversal_tested": True,
|
|
"order_reversal_consistent": order_consistent,
|
|
}
|
|
|
|
|
|
def _mean_score(grade: dict[str, Any]) -> float:
|
|
scores: dict[str, float] = grade.get("scores", {})
|
|
if not scores:
|
|
return 0.0
|
|
return sum(scores.values()) / len(scores)
|
|
|
|
|
|
def _winners_consistent(forward: str, reversed_result: str) -> bool | None:
|
|
if forward == "abstain" or reversed_result == "abstain":
|
|
return None
|
|
if forward == "tie" and reversed_result == "tie":
|
|
return True
|
|
if forward == "tie" or reversed_result == "tie":
|
|
return False
|
|
return forward == reversed_result
|
|
|
|
|
|
@dataclass
|
|
class CalibrationRecord:
|
|
human_sample_count: int = 0
|
|
judge_agreement_rate: float | None = None
|
|
disagreement_slices: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
@property
|
|
def calibrated(self) -> bool:
|
|
return (
|
|
self.human_sample_count > 0
|
|
and self.judge_agreement_rate is not None
|
|
and self.judge_agreement_rate >= 0.7
|
|
)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"human_sample_count": self.human_sample_count,
|
|
"judge_agreement_rate": self.judge_agreement_rate,
|
|
"disagreement_slices": self.disagreement_slices,
|
|
"calibrated": self.calibrated,
|
|
}
|
|
|
|
|
|
def compute_release_decision(
|
|
case_results: list[dict[str, Any]],
|
|
rubric_results: list[dict[str, Any]],
|
|
calibration: CalibrationRecord,
|
|
hard_gate_violations: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Compute PASS, CONDITIONAL, HOLD, or BLOCK from the release matrix."""
|
|
violations = hard_gate_violations or []
|
|
reasons: list[str] = []
|
|
|
|
if violations:
|
|
return {
|
|
"outcome": "BLOCK",
|
|
"reasons": [f"hard gate violation: {v}" for v in violations],
|
|
"hard_gate_violations": violations,
|
|
}
|
|
|
|
release_cases = [c for c in case_results if c["case_set"] == "release"]
|
|
regression_cases = [c for c in case_results if c["case_set"] == "regression"]
|
|
|
|
missing_evidence_cases = [c["case_id"] for c in case_results if c.get("missing_evidence")]
|
|
if missing_evidence_cases:
|
|
reasons.append(f"missing evidence in cases: {', '.join(missing_evidence_cases)}")
|
|
return {
|
|
"outcome": "HOLD",
|
|
"reasons": reasons,
|
|
"hard_gate_violations": [],
|
|
}
|
|
|
|
inconsistent = [c["case_id"] for c in case_results if not c["consistent"]]
|
|
low_frequency = [c["case_id"] for c in release_cases if c["success_frequency"] < 0.8]
|
|
|
|
rubric_abstains = [
|
|
r["case_id"] for r in rubric_results if r["verdict"] in ("abstain", "insufficient_evidence")
|
|
]
|
|
|
|
uncalibrated_advisory = not calibration.calibrated
|
|
|
|
if low_frequency:
|
|
reasons.append(f"release cases below 80% success: {', '.join(low_frequency)}")
|
|
return {
|
|
"outcome": "BLOCK",
|
|
"reasons": reasons,
|
|
"hard_gate_violations": [],
|
|
}
|
|
|
|
regressions = [
|
|
c["case_id"] for c in regression_cases if c["paired_delta"] == "candidate_regression"
|
|
]
|
|
if regressions:
|
|
reasons.append(f"regressions detected: {', '.join(regressions)}")
|
|
return {
|
|
"outcome": "BLOCK",
|
|
"reasons": reasons,
|
|
"hard_gate_violations": [],
|
|
}
|
|
|
|
conditional = False
|
|
if inconsistent:
|
|
reasons.append(f"inconsistent cases: {', '.join(inconsistent)}")
|
|
conditional = True
|
|
if rubric_abstains:
|
|
reasons.append(f"rubric abstained on: {', '.join(rubric_abstains)}")
|
|
conditional = True
|
|
if uncalibrated_advisory:
|
|
reasons.append("judge not calibrated against human labels; rubric results advisory only")
|
|
conditional = True
|
|
|
|
if conditional:
|
|
return {
|
|
"outcome": "CONDITIONAL",
|
|
"reasons": reasons,
|
|
"hard_gate_violations": [],
|
|
}
|
|
|
|
if not reasons:
|
|
reasons.append("all gates satisfied")
|
|
|
|
return {
|
|
"outcome": "PASS",
|
|
"reasons": reasons,
|
|
"hard_gate_violations": [],
|
|
}
|
|
|
|
|
|
def build_release_report(
|
|
*,
|
|
skill_name: str,
|
|
freeze: FreezeSnapshot,
|
|
case_results: list[dict[str, Any]],
|
|
rubric_results: list[dict[str, Any]],
|
|
pairwise_results: list[dict[str, Any]],
|
|
calibration: CalibrationRecord,
|
|
hard_gate_violations: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
decision = compute_release_decision(
|
|
case_results, rubric_results, calibration, hard_gate_violations
|
|
)
|
|
return {
|
|
"schema_version": RELEASE_SCHEMA_VERSION,
|
|
"report_id": str(uuid.uuid4()),
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"skill_name": skill_name,
|
|
"freeze_snapshot": freeze.to_dict(),
|
|
"case_results": case_results,
|
|
"rubric_results": rubric_results,
|
|
"pairwise_results": pairwise_results,
|
|
"calibration": calibration.to_dict(),
|
|
"release_decision": decision,
|
|
}
|
|
|
|
|
|
def write_release_report(report: dict[str, Any], output_dir: Path) -> Path:
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
path = output_dir / f"{report['skill_name']}--release-report.json"
|
|
path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def dataset_hash(manifest_path: Path) -> str:
|
|
content = manifest_path.read_bytes()
|
|
return hashlib.sha256(content).hexdigest()[:16]
|