mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-11 19:47:12 +03:00
101 lines
3.7 KiB
Python
101 lines
3.7 KiB
Python
"""Comparison report generation for paired evaluation trials."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .grader import GradeResult
|
|
from .path_safety import contained_path, validate_case_id
|
|
|
|
COMPARISON_SCHEMA_VERSION = 1
|
|
|
|
|
|
def build_comparison_report(
|
|
*,
|
|
skill_name: str,
|
|
case_id: str,
|
|
candidate_grade: GradeResult,
|
|
baseline_grade: GradeResult,
|
|
candidate_manifest: dict[str, Any],
|
|
baseline_manifest: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
candidate_passed = candidate_grade.passed
|
|
baseline_passed = baseline_grade.passed
|
|
|
|
if candidate_passed and not baseline_passed:
|
|
delta = "candidate_improvement"
|
|
elif not candidate_passed and baseline_passed:
|
|
delta = "candidate_regression"
|
|
elif candidate_passed and baseline_passed:
|
|
delta = "both_pass"
|
|
else:
|
|
delta = "both_fail"
|
|
|
|
return {
|
|
"schema_version": COMPARISON_SCHEMA_VERSION,
|
|
"report_id": str(uuid.uuid4()),
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"skill_name": skill_name,
|
|
"case_id": case_id,
|
|
"candidate": {
|
|
"trial_id": candidate_manifest.get("trial_id", ""),
|
|
"passed": candidate_passed,
|
|
"infra_error": candidate_grade.infra_error,
|
|
"pass_count": candidate_grade.pass_count,
|
|
"fail_count": candidate_grade.fail_count,
|
|
"manual_count": candidate_grade.manual_count,
|
|
"assertions": [
|
|
{"assertion": r.assertion, "verdict": r.verdict.value, "detail": r.detail}
|
|
for r in candidate_grade.results
|
|
],
|
|
"manifest": candidate_manifest,
|
|
},
|
|
"baseline": {
|
|
"trial_id": baseline_manifest.get("trial_id", ""),
|
|
"passed": baseline_passed,
|
|
"infra_error": baseline_grade.infra_error,
|
|
"pass_count": baseline_grade.pass_count,
|
|
"fail_count": baseline_grade.fail_count,
|
|
"manual_count": baseline_grade.manual_count,
|
|
"assertions": [
|
|
{"assertion": r.assertion, "verdict": r.verdict.value, "detail": r.detail}
|
|
for r in baseline_grade.results
|
|
],
|
|
"manifest": baseline_manifest,
|
|
},
|
|
"paired_delta": delta,
|
|
}
|
|
|
|
|
|
def write_comparison_report(report: dict[str, Any], output_dir: Path) -> Path:
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
case_id = report.get("case_id", "unknown")
|
|
validate_case_id(case_id)
|
|
report_id = report.get("report_id", "unknown")[:8]
|
|
path = contained_path(output_dir, f"{case_id}--{report_id}.comparison.json")
|
|
path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def format_comparison_summary(report: dict[str, Any]) -> str:
|
|
lines = [
|
|
f"Skill: {report['skill_name']} Case: {report['case_id']}",
|
|
f"Delta: {report['paired_delta']}",
|
|
"",
|
|
f" Candidate: {'PASS' if report['candidate']['passed'] else 'FAIL'}"
|
|
f" ({report['candidate']['pass_count']} pass, {report['candidate']['fail_count']} fail,"
|
|
f" {report['candidate']['manual_count']} manual)",
|
|
f" Baseline: {'PASS' if report['baseline']['passed'] else 'FAIL'}"
|
|
f" ({report['baseline']['pass_count']} pass, {report['baseline']['fail_count']} fail,"
|
|
f" {report['baseline']['manual_count']} manual)",
|
|
]
|
|
if report["candidate"]["infra_error"]:
|
|
lines.append(" [!] Candidate had infrastructure error")
|
|
if report["baseline"]["infra_error"]:
|
|
lines.append(" [!] Baseline had infrastructure error")
|
|
return "\n".join(lines)
|