mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-18 15:06:28 +03:00
0a80bda800
Add release evaluation layer on top of deterministic paired execution: - schemas/release-eval-v1.schema.json: release report schema with freeze snapshot, per-case trial aggregation, rubric graders, blinded pairwise comparison, calibration tracking, and PASS/CONDITIONAL/HOLD/BLOCK outcomes - eval_runner/release.py: core module for multi-trial aggregation, versioned rubric graders with abstain/insufficient-evidence, pairwise planning with position randomization and order-reversal testing, calibration records, and release decision computation - eval_runner/tests/test_release.py: 34 tests covering all acceptance criteria - schemas/evals-v1.schema.json: optional case_set field (dev/regression/release) - eval_runner/models.py: case_set on EvalCase - eval_runner/runner.py: load case_set from manifest - .github/workflows/skill-eval.yml: run release tests in CI Gate semantics: hard invariants (privacy, auth, destructive) tolerate zero violations and cannot be averaged away. Missing evidence produces HOLD, not PASS. Uncalibrated judge results are advisory only. Closes #106 Co-authored-by: magnus919 <magnus919>
90 lines
2.5 KiB
Python
90 lines
2.5 KiB
Python
"""Typed data models for the eval runner adapter contract."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from dataclasses import dataclass, field
|
|
from enum import Enum
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
class ExitStatus(str, Enum):
|
|
COMPLETED = "completed"
|
|
ERROR = "error"
|
|
TIMEOUT = "timeout"
|
|
STOPPED = "stopped"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EvalCase:
|
|
id: str
|
|
prompt: str
|
|
expected_output: str
|
|
assertions: list[str]
|
|
files: list[str] = field(default_factory=list)
|
|
case_set: str = "dev"
|
|
|
|
@property
|
|
def prompt_hash(self) -> str:
|
|
return hashlib.sha256(self.prompt.encode()).hexdigest()[:16]
|
|
|
|
def fixture_hashes(self, skill_root: Path) -> dict[str, str]:
|
|
hashes: dict[str, str] = {}
|
|
for rel in self.files:
|
|
target = skill_root / rel
|
|
if target.is_file():
|
|
hashes[rel] = hashlib.sha256(target.read_bytes()).hexdigest()[:16]
|
|
else:
|
|
hashes[rel] = "missing"
|
|
return hashes
|
|
|
|
|
|
@dataclass
|
|
class AdapterInput:
|
|
skill_path: Path
|
|
case: EvalCase
|
|
work_dir: Path
|
|
output_dir: Path
|
|
permissions: dict[str, Any] = field(default_factory=dict)
|
|
limits: dict[str, Any] = field(default_factory=dict)
|
|
env: dict[str, str] = field(default_factory=dict)
|
|
model: str = ""
|
|
harness_config: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass
|
|
class ToolEvent:
|
|
name: str
|
|
arguments: dict[str, Any] = field(default_factory=dict)
|
|
result_summary: str = ""
|
|
timestamp: str = ""
|
|
|
|
|
|
@dataclass
|
|
class AdapterOutput:
|
|
exit_status: ExitStatus
|
|
response: str | None = None
|
|
activation_evidence: str | None = None
|
|
artifacts: list[str] = field(default_factory=list)
|
|
environment_state: dict[str, Any] | None = None
|
|
tool_events: list[ToolEvent] = field(default_factory=list)
|
|
duration_ms: float = 0.0
|
|
token_usage: dict[str, Any] | None = None
|
|
raw_trace_path: str | None = None
|
|
error: str | None = None
|
|
|
|
def missing_evidence(self) -> list[str]:
|
|
missing: list[str] = []
|
|
if self.response is None:
|
|
missing.append("response")
|
|
if self.activation_evidence is None:
|
|
missing.append("activation_evidence")
|
|
if self.environment_state is None:
|
|
missing.append("environment_state")
|
|
if self.token_usage is None:
|
|
missing.append("token_usage")
|
|
if not self.tool_events:
|
|
missing.append("tool_events")
|
|
return missing
|