magnus919_agent-skills/eval_runner/progress_finality.py

"""Portable semantic-progress and finality oracle for scripted agent transcripts.

This is a generic transcript grader, not an agent runtime and not an
integration with any specific harness. It evaluates the repository-agnostic
semantic-progress contract: judge semantic state, declared dependencies, and
declared stage actions — never elapsed time, raw iteration counts, request
identifiers, or repeated observation of unchanged state. Identical repetition
is a failure for checks (identical re-polls) and equally for unproductive work
turns (stuck work), while mechanism changes, escalations, and any number of
turns that change the semantic state remain allowed.
"""

from __future__ import annotations

import hashlib
import json
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any

ACCEPTED_STATES = frozenset({"accepted", "finished"})
BLOCKED_DEPENDENCY_STATES = frozenset({"pending", "failed"})
TERMINAL_ACTIONS = frozenset({"advance", "deliver", "complete"})
CHECK_ACTION = "check"
UNPRODUCTIVE_ACTIONS = frozenset({"work", "check"})
KNOWN_ACTIONS = frozenset(
    {"work", "check", "advance", "deliver", "complete", "mechanism_change", "escalate"}
)


@dataclass(frozen=True)
class ProgressVerdict:
    passed: bool
    reason: str
    checks_for_state: int
    identical_repolls: int


def _validate_transcript(transcript: Mapping[str, Any]) -> None:
    if not isinstance(transcript, Mapping):
        raise ValueError("transcript: must be a mapping")
    case_id = transcript.get("case_id")
    if not isinstance(case_id, str) or not case_id.strip():
        raise ValueError("case_id: must be a non-empty string")
    turns = transcript.get("turns")
    if not isinstance(turns, list) or not turns:
        raise ValueError("turns: must be a non-empty list")
    for index, turn in enumerate(turns):
        prefix = f"turns[{index}]"
        if not isinstance(turn, Mapping):
            raise ValueError(f"{prefix}: must be a mapping")
        for required in ("observation", "action"):
            if required not in turn:
                raise ValueError(f"{prefix}: missing '{required}'")
            if not isinstance(turn[required], Mapping):
                raise ValueError(f"{prefix}.{required}: must be a mapping")
        observation = turn["observation"]
        for required in ("stage", "deliverable_state", "dependency_state"):
            value = observation.get(required)
            if not isinstance(value, str) or not value:
                raise ValueError(f"{prefix}.observation.{required}: must be a non-empty string")
        if "dependency_required_for_next_stage" not in observation:
            raise ValueError(f"{prefix}.observation: missing 'dependency_required_for_next_stage'")
        if not isinstance(observation["dependency_required_for_next_stage"], bool):
            raise ValueError(
                f"{prefix}.observation.dependency_required_for_next_stage: must be a boolean"
            )
        if "semantic_state" not in observation:
            raise ValueError(f"{prefix}.observation: missing 'semantic_state'")
        if not isinstance(observation["semantic_state"], Mapping):
            raise ValueError(f"{prefix}.observation.semantic_state: must be a mapping")
        kind = turn["action"].get("kind")
        if not isinstance(kind, str) or kind not in KNOWN_ACTIONS:
            raise ValueError(f"{prefix}.action.kind: unrecognized action kind {kind!r}")


def _semantic_fingerprint(observation: Mapping[str, Any]) -> str:
    payload = {
        "stage": observation["stage"],
        "deliverable_state": observation["deliverable_state"],
        "dependency_state": observation["dependency_state"],
        "dependency_required_for_next_stage": observation["dependency_required_for_next_stage"],
        "semantic_state": observation["semantic_state"],
    }
    blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
    return hashlib.sha256(blob.encode("utf-8")).hexdigest()


def evaluate_transcript(transcript: Mapping[str, Any]) -> ProgressVerdict:
    """Evaluate a portable semantic-progress/finality transcript."""
    _validate_transcript(transcript)
    checks_by_fingerprint: dict[str, int] = {}
    identical_repolls = 0
    previous_fingerprint: str | None = None
    previous_action_kind: str | None = None

    for index, turn in enumerate(transcript["turns"]):
        observation = turn["observation"]
        action_kind = turn["action"]["kind"]
        fingerprint = _semantic_fingerprint(observation)
        deliverable_state = observation["deliverable_state"]
        dependency_state = observation["dependency_state"]
        dependency_required = observation["dependency_required_for_next_stage"]

        if (
            deliverable_state in ACCEPTED_STATES
            and not dependency_required
            and action_kind not in TERMINAL_ACTIONS
        ):
            return ProgressVerdict(
                passed=False,
                reason=(
                    f"turns[{index}]: accepted/finished deliverable with no dependency "
                    f"required for the next stage must advance, deliver, or complete "
                    f"immediately; found action '{action_kind}'"
                ),
                checks_for_state=checks_by_fingerprint.get(fingerprint, 0),
                identical_repolls=identical_repolls,
            )

        if (
            dependency_required
            and dependency_state in BLOCKED_DEPENDENCY_STATES
            and action_kind == CHECK_ACTION
        ):
            checks_by_fingerprint[fingerprint] = checks_by_fingerprint.get(fingerprint, 0) + 1
            count = checks_by_fingerprint[fingerprint]
            if count > 1:
                identical_repolls += 1
                return ProgressVerdict(
                    passed=False,
                    reason=(
                        f"turns[{index}]: identical re-poll — check repeated for an unchanged "
                        f"semantic state (fingerprint {fingerprint[:12]}) while the declared "
                        f"stage action remains untaken; a volatile change such as a new "
                        f"timestamp or request ID is not progress; require a mechanism "
                        f"change or escalation"
                    ),
                    checks_for_state=count,
                    identical_repolls=identical_repolls,
                )

        if (
            previous_fingerprint is not None
            and fingerprint == previous_fingerprint
            and action_kind in UNPRODUCTIVE_ACTIONS
            and previous_action_kind == action_kind
        ):
            return ProgressVerdict(
                passed=False,
                reason=(
                    f"turns[{index}]: stuck work — action '{action_kind}' repeated against "
                    f"an unchanged semantic state (fingerprint {fingerprint[:12]}) while the "
                    f"declared stage action remains untaken; repeating identical effort "
                    f"without a semantic change is not progress; declare a mechanism "
                    f"change, escalate, or produce a changed semantic state"
                ),
                checks_for_state=checks_by_fingerprint.get(fingerprint, 0),
                identical_repolls=identical_repolls,
            )

        previous_fingerprint = fingerprint
        previous_action_kind = action_kind

    return ProgressVerdict(
        passed=True,
        reason="all turns satisfy the semantic-progress and finality contract",
        checks_for_state=max(checks_by_fingerprint.values(), default=0),
        identical_repolls=identical_repolls,
    )