mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-19 23:46:28 +03:00
Magnus HedemarkandGitHub
7964c060d3
Define semantic progress and stage finality for asynchronous work, add bounded-check guidance, and cover terminal advancement, productive complexity, and stuck-work regressions with deterministic transcript tests. Signed-off-by: Magnus Hedemark <magnus919@pm.me>
87 lines
3.0 KiB
Python
87 lines
3.0 KiB
Python
"""Deterministic RED/GREEN regression tests for the progress/finality oracle."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
|
|
|
|
from eval_runner.progress_finality import evaluate_transcript
|
|
|
|
FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "progress_finality"
|
|
|
|
|
|
def _load(name: str) -> dict:
|
|
with (FIXTURE_DIR / name).open(encoding="utf-8") as fh:
|
|
return json.load(fh)
|
|
|
|
|
|
def test_accepted_output_advances_without_poll():
|
|
verdict = evaluate_transcript(_load("accepted-terminal.json"))
|
|
assert verdict.passed, verdict.reason
|
|
assert verdict.checks_for_state == 0
|
|
assert verdict.identical_repolls == 0
|
|
|
|
|
|
def test_complex_work_passes_when_semantic_progress_changes():
|
|
transcript = _load("complex-progress.json")
|
|
assert len(transcript["turns"]) >= 3
|
|
verdict = evaluate_transcript(transcript)
|
|
assert verdict.passed, verdict.reason
|
|
assert verdict.identical_repolls == 0
|
|
assert verdict.checks_for_state == 0
|
|
|
|
|
|
def test_one_check_is_allowed_after_a_real_pending_state():
|
|
verdict = evaluate_transcript(_load("pending-state-change.json"))
|
|
assert verdict.passed, verdict.reason
|
|
assert verdict.checks_for_state == 1
|
|
assert verdict.identical_repolls == 0
|
|
|
|
|
|
def test_identical_repoll_fails_even_when_volatile_fields_change():
|
|
verdict = evaluate_transcript(_load("identical-repoll.json"))
|
|
assert not verdict.passed
|
|
assert verdict.identical_repolls >= 1
|
|
assert "identical" in verdict.reason.lower()
|
|
assert "re-poll" in verdict.reason.lower()
|
|
|
|
|
|
def test_identical_work_turns_fail_as_stuck_work():
|
|
verdict = evaluate_transcript(_load("stuck-work.json"))
|
|
assert not verdict.passed
|
|
assert "stuck work" in verdict.reason.lower()
|
|
assert verdict.identical_repolls == 0
|
|
|
|
|
|
def test_mechanism_change_and_escalate_are_allowed_on_unchanged_state():
|
|
transcript = _load("stuck-work.json")
|
|
first, second = transcript["turns"][0], transcript["turns"][1]
|
|
transcript["turns"] = [
|
|
first,
|
|
{**second, "action": {"kind": "mechanism_change", "target": "switch to sampling profiler"}},
|
|
{**second, "action": {"kind": "escalate", "target": "owner review"}},
|
|
]
|
|
verdict = evaluate_transcript(transcript)
|
|
assert verdict.passed, verdict.reason
|
|
assert verdict.checks_for_state == 0
|
|
assert verdict.identical_repolls == 0
|
|
|
|
|
|
def test_missing_dependency_contract_is_rejected():
|
|
missing = _load("accepted-terminal.json")
|
|
for turn in missing["turns"]:
|
|
del turn["observation"]["dependency_required_for_next_stage"]
|
|
with pytest.raises(ValueError, match="dependency_required_for_next_stage"):
|
|
evaluate_transcript(missing)
|
|
|
|
wrong_type = _load("accepted-terminal.json")
|
|
for turn in wrong_type["turns"]:
|
|
turn["observation"]["dependency_required_for_next_stage"] = "no"
|
|
with pytest.raises(ValueError, match="dependency_required_for_next_stage"):
|
|
evaluate_transcript(wrong_type)
|