mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-12 12:06:29 +03:00
e83558a6e3
Implements #104. Adds a repository-level evaluation runner with: - Typed HarnessAdapter Protocol (adapter.py) - Dataclass models for AdapterInput/AdapterOutput (models.py) - FakeAdapter for deterministic CI without credentials (fake_adapter.py) - CliSubprocessAdapter for non-interactive CLI harnesses (cli_adapter.py) - Run manifest builder with schema validation (manifest.py) - Runner CLI entry point (runner.py, __main__.py) - JSON Schema for trial manifests (schemas/run-manifest-v1.schema.json) - Unit tests including schema validation (tests/test_runner.py) Co-authored-by: magnus919 <magnus919>
38 lines
1.1 KiB
Python
38 lines
1.1 KiB
Python
"""Adapter protocol defining the harness contract."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Protocol, runtime_checkable
|
|
|
|
from .models import AdapterInput, AdapterOutput
|
|
|
|
|
|
@runtime_checkable
|
|
class HarnessAdapter(Protocol):
|
|
"""Contract for executing an eval case against a candidate skill.
|
|
|
|
Implementations must be non-interactive and deterministic given the same
|
|
inputs (modulo model non-determinism in real adapters). Raw traces may
|
|
remain adapter-specific; only evidence required by declared graders is
|
|
normalized in AdapterOutput.
|
|
"""
|
|
|
|
@property
|
|
def name(self) -> str:
|
|
"""Stable adapter identifier, e.g. 'fake', 'cli-subprocess'."""
|
|
...
|
|
|
|
@property
|
|
def version(self) -> str:
|
|
"""Adapter implementation version."""
|
|
...
|
|
|
|
def execute(self, input: AdapterInput) -> AdapterOutput:
|
|
"""Run one eval case and return normalized results.
|
|
|
|
Must not raise on expected failure modes (timeout, model error, skill
|
|
not found). Encode failures in AdapterOutput.exit_status and .error.
|
|
Unexpected infrastructure errors may raise.
|
|
"""
|
|
...
|