diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 660cb14..254ff7c 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -32,6 +32,15 @@ "strict": false, "description": "Design, run, review, or release framework- and vendor-neutral evaluations and observability for AI agents. Use when defining agent evals, datasets, graders, trajectory review, regression analysis, release gates, production traces, or privacy-aware telemetry. Covers task and trajectory contracts, statistical comparisons, and incident-to-case learning; route framework implementation to pydanticai or langgraph when needed." }, + { + "name": "agent-production-operations", + "source": "./", + "skills": [ + "./agent-production-operations" + ], + "strict": false, + "description": "Operate an evaluated agent with tools and authority in production through a runtime control plane covering versioning, staged rollout, fallback, cost and latency budgets, tool health, human escalation, disablement, and trace-to-eval feedback. Do not use for building agents, designing evals, or general release, SRE, security, or platform engineering — those methods stay with their specialist skills." + }, { "name": "agent-skills", "source": "./", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index d712c61..9425efe 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -28,6 +28,7 @@ "./backend-engineering", "./binary-analysis", "./brand-designer", + "./bundles/agent-production-operations", "./bundles/neckbeard", "./bundles/product-lifecycle", "./bundles/production-excellence", diff --git a/README.md b/README.md index 17e9283..c914aee 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,10 @@ Multi-agent structured debate system — spawn a panel of expert agents to debat Design and review framework-neutral AI-agent evaluations, trajectory evidence, risk-tiered release gates, and privacy-aware observability. Covers datasets, grader calibration, statistical comparisons, incident-to-case learning, optional OpenTelemetry interoperability, and templates for durable evidence. +### [agent-production-operations](bundles/agent-production-operations/SKILL.md) + +Operate an evaluated agent with tools and authority in production through a runtime control plane covering versioning, staged rollout, fallback, cost and latency budgets, tool health, human escalation, disablement, and trace-to-eval feedback. + ### [agent-skills](agent-skills/SKILL.md) Reference for the Agent Skills open format itself — directory structure, frontmatter schema, naming conventions, and progressive disclosure model. Use this meta-skill when creating or reviewing any other skill in this repository. diff --git a/bundles/agent-production-operations/AGENTS.md b/bundles/agent-production-operations/AGENTS.md new file mode 100644 index 0000000..97e0a1e --- /dev/null +++ b/bundles/agent-production-operations/AGENTS.md @@ -0,0 +1,27 @@ +# AGENTS.md — agent-production-operations bundle + +This bundle follows the Agent Skills format. The discoverable entry point is +[SKILL.md](SKILL.md), which is a thin umbrella that routes to reference files +and specialist skills. + +## Loading behavior + +- The bundle does **not** ship nested sub-skills. All content is in + `references/` files loaded on trigger. +- [SKILL.md](SKILL.md) is the only file that appears in generated catalogs. +- Reference files are loaded only when their trigger condition matches (see + the loading protocol table in SKILL.md). + +## Nested-skill loading note + +This bundle is a composition of existing specialist skills. When the active +concern falls within a specialist's domain, load that specialist's SKILL.md +directly rather than re-deriving its method from the bundle. The routing table +in [SKILL.md](SKILL.md) defines which specialist to load for each concern. + +## Validation + +This bundle is validated by the repository's standard toolchain: +- `ruby scripts/validate-skills.rb` +- `.venv/bin/python scripts/validate-evals.py` +- `ruby scripts/validate-skill-quality.rb --base origin/main` diff --git a/bundles/agent-production-operations/README.md b/bundles/agent-production-operations/README.md new file mode 100644 index 0000000..394c64a --- /dev/null +++ b/bundles/agent-production-operations/README.md @@ -0,0 +1,105 @@ +# agent-production-operations — runtime control plane for AI agents + +Take an evaluated agent from "passes tests" to "safe in production" with +versioning, staged rollout, fallback, cost and latency monitoring, escalation, +and disablement — all feeding back into better evaluations. + +## Why Install This Skill + +You have an AI agent that passes your evaluation suite. The evals are solid, the +observability is in place, and the agent looks ready. But between "passes evals" +and "safe at scale" there is an operational gap no single specialist skill fills: +how do you version the model, prompt, and tools together? How do you roll out +gradually instead of flipping a switch? What happens when the model regresses, +a tool goes down, the cost budget blows up, or the agent tries something it +shouldn't? + +This bundle gives you a runtime control plane purpose-built for AI agents. It +does not replace your release pipeline, your SRE practices, your security +reviews, or your eval framework — it sits between them, coordinating the +decisions that are specific to operating an agent with tools and authority in +production. It distinguishes read-only agents from side-effect-capable agents +and customer-facing agents from internal ones, because a search bot and a +support bot that can modify accounts have fundamentally different operational +risk profiles. + +After installing this skill, your agent can answer: what authority do I have, +what happens when I'm uncertain, when do I escalate to a human, how do I fall +back safely, and when should I be disabled entirely. Your team can answer: how +do production incidents feed back into better evaluations, and how do eval +results gate the next release. + +## What You Get + +| Path | What it provides | +|---|---| +| [SKILL.md](SKILL.md) | Umbrella entry point — runtime control plane routing table, concrete authority/escalation/fallback/disablement parameters, and specialist-skill orchestration | +| [README.md](README.md) | This file — human-facing overview | +| [AGENTS.md](AGENTS.md) | Agent loading and nested-skill discovery notes | +| [references/discovery-brief.md](references/discovery-brief.md) | Bounded discovery brief defining boundaries with agent-evals, release, SRE, security, and platform skills | +| [references/agent-production-contract.md](references/agent-production-contract.md) | Production contract template — capability, authority, uncertainty, escalation, and side-effect contracts with production-readiness and incident-learning inputs | +| [references/runtime-control-plan.md](references/runtime-control-plan.md) | Versioning (model, prompt, tool, policy, evaluator), four-stage rollout plan, and fallback paths with concrete triggers | +| [references/tool-authority-health.md](references/tool-authority-health.md) | Health record schema for tracking tool availability, tool failure, authority usage, authority breaches, cost, and latency over time | +| [references/trace-to-eval-feedback.md](references/trace-to-eval-feedback.md) | Production-to-evaluation feedback loop — how traces and incidents become eval cases and gate releases | +| [evals/evals.json](evals/evals.json) | Integrated eval cases covering read-only agents, side-effect agents, model regression, tool outages, cost breaches, and human escalation | + +## Quick Start + +1. Start with an agent that has passed evaluation (use + [agent-evals-and-observability](../../agent-evals-and-observability/SKILL.md)). + +2. Define the agent's production contract using + [references/agent-production-contract.md](references/agent-production-contract.md). + Fill in capability, authority, uncertainty, escalation, and side-effect fields. + +3. Consume the most recent production-readiness review outcome for the agent's + host service from + [production-readiness](../../production-readiness/SKILL.md) and any open + incident-learning records from + [incident-learning](../../incident-learning/SKILL.md). Record both in the + contract. + +4. Plan versioning, staged rollout, and fallback using + [references/runtime-control-plan.md](references/runtime-control-plan.md). + +5. Begin Stage 1 rollout (shadow/dry-run). Record tool and authority health in + [references/tool-authority-health.md](references/tool-authority-health.md). + +6. Advance through stages as exit criteria are met. At each stage boundary, + review the health record for authority breaches, tool degradation, cost + trends, and latency patterns. + +7. When escalation or fallback triggers fire, follow the concrete procedures in + [SKILL.md](SKILL.md). + +8. Feed production traces and incidents back into evaluation cases using + [references/trace-to-eval-feedback.md](references/trace-to-eval-feedback.md). + +## Triggers + +Load this bundle when: +- You are operating an agent that has passed evaluation and is being promoted to production. +- You need to define a staged rollout with progressive authority expansion. +- You need versioning across model, prompt, tools, policy, and evaluator. +- You need fallback, escalation, and disablement procedures for an agent. +- You need a production-to-evaluation feedback loop. +- You see terms like "agent authority," "tool health," "agent cost budget," "agent latency SLO," or "agent disablement." + +Do NOT load this bundle for: +- Building an agent from scratch — use your agent framework skill. +- Designing evaluations or observability — use `agent-evals-and-observability`. +- General release engineering — use `release-engineering`. +- General SRE or incident response — use `site-reliability-engineering`. +- General security engineering — use `secure-software-engineering`. +- Platform infrastructure — use `platform-engineering`. + +## Requirements + +- An agent that has passed evaluation (observability and eval infrastructure + must exist — see `agent-evals-and-observability`). +- A designated escalation channel (human operator or team). +- A cost budget and attribution mechanism for the agent. +- Production-readiness review process in place (see `production-readiness`). +- Incident-learning process in place (see `incident-learning`). +- No additional API keys, services, or runtime dependencies beyond what the + agent already requires. diff --git a/bundles/agent-production-operations/SKILL.md b/bundles/agent-production-operations/SKILL.md new file mode 100644 index 0000000..65e8ea2 --- /dev/null +++ b/bundles/agent-production-operations/SKILL.md @@ -0,0 +1,266 @@ +--- +name: agent-production-operations +description: >- + Operate an evaluated agent with tools and authority in production through a + runtime control plane covering versioning, staged rollout, fallback, cost and + latency budgets, tool health, human escalation, disablement, and trace-to-eval + feedback. Do not use for building agents, designing evals, or general release, + SRE, security, or platform engineering — those methods stay with their + specialist skills. +license: MIT +compatibility: Agent harness with file read/write and terminal access. No network or runtime dependency required by the bundle itself. +metadata: + spec-version: "1.0" + tags: agent, production, operations, runtime, control-plane, rollout, fallback, disablement +--- + +# agent-production-operations + +A runtime control plane for taking an evaluated agent with tools and authority +into controlled production operation. This bundle bridges the gap between +passing evaluations and safe production use: it defines how an agent is +versioned, rolled out, monitored, constrained, escalated, and disabled — and +how production evidence feeds back into evaluation and release decisions. + +The bundle does **not** build agents, design evaluations, run release pipelines, +or own infrastructure. It composes those concerns from existing specialist +skills and owns only the runtime control decisions between them. + +## When to load this + +Load this bundle when you are operating an agent that: +- Has passed evaluation and is cleared for production consideration. +- Has tools with side effects and delegated authority that must be gated. +- Needs a staged rollout with progressive authority expansion. +- Must be monitored for cost, latency, tool health, and authority usage in + production. +- Requires a defined fallback, escalation, and disablement path. +- Should feed production traces and incidents back into evaluation cases and + release-gate updates. + +## When not to use + +Do **not** load this bundle for: +- Building or designing an agent from scratch — route to the appropriate + framework skill (LangGraph, CrewAI, AutoGen, etc.). +- Designing agent evaluations or observability — route to + [agent-evals-and-observability](../../agent-evals-and-observability/SKILL.md). +- General release engineering (CI/CD pipelines, artifact promotion) — route to + [release-engineering](../../release-engineering/SKILL.md). +- General site reliability engineering (incident response, SLO definition, + error budgets) — route to + [site-reliability-engineering](../../site-reliability-engineering/SKILL.md). +- General security engineering (threat modeling, vulnerability assessment) — + route to [secure-software-engineering](../../secure-software-engineering/SKILL.md). +- General platform infrastructure (compute, networking, storage) — route to + [platform-engineering](../../platform-engineering/SKILL.md). +- Performing a production-readiness review — route to + [production-readiness](../../production-readiness/SKILL.md); this bundle + **consumes** readiness outcomes, it does not produce them. +- Conducting an incident postmortem — route to + [incident-learning](../../incident-learning/SKILL.md); this bundle + **consumes** incident records as escalation and feedback inputs. + +## Autonomy is not assumed uniform + +This bundle explicitly does **not** assume all agents share the same autonomy, +user population, or side-effect profile. Agent profiles are distinguished by +capability class and side-effect surface: + +| Profile | Example | Authority | Escalation trigger | Fallback | +|---|---|---|---|---| +| **Read-only** | Internal search agent | Query-only; no mutation, no user-data access | Uncertainty above threshold; result quality degradation | Return cached/static results | +| **Side-effect-capable, internal** | CI triage bot | Issue/PR comments, label management, branch creation | Tool failure > 5% in window; authority breach attempt | Disable tool, escalate to human | +| **Side-effect-capable, customer-facing** | Support agent with account access | Read PII, suggest actions, no mutation without confirmation | Any PII access without explicit consent; cost breach > 10% of budget | Degrade to read-only, escalate immediately | + +Every control-plan decision (authority, escalation, fallback, disablement) in +this bundle is parameterized by the agent's autonomy profile. A read-only agent +and a side-effect-capable agent operating on customer data receive different +thresholds and different escalation paths. + +## Runtime control plane routing + +The bundle composes the following specialist skills. Load them when their +domain is the active concern; the bundle owns the cross-domain coordination. + +| Concern | Specialist skill | When to load | What the bundle adds | +|---|---|---|---| +| Agent evaluation and observability | [agent-evals-and-observability](../../agent-evals-and-observability/SKILL.md) | Designing evals, instrumenting observability, detecting regression | Trace-to-eval feedback loop; eval-case generation from production incidents | +| Release engineering | [release-engineering](../../release-engineering/SKILL.md) | Release pipeline design, artifact promotion, CI/CD | Agent-specific staged rollout with authority gating; rollback triggers tied to agent health | +| Site reliability engineering | [site-reliability-engineering](../../site-reliability-engineering/SKILL.md) | Incident response, SLO/SLI definition, error budgets | Agent-specific latency/cost budgets; tool-health SLOs; escalation handoff to incident command | +| Security engineering | [secure-software-engineering](../../secure-software-engineering/SKILL.md) | Threat modeling, trust-boundary validation, secure design | Agent authority contracts with side-effect boundaries; disablement security (revoke, not just stop) | +| Platform engineering | [platform-engineering](../../platform-engineering/SKILL.md) | Compute, networking, storage, service infrastructure | Agent sandboxing requirements; tool-execution isolation preferences | +| Production readiness | [production-readiness](../../production-readiness/SKILL.md) | Readiness reviews, go/no-go/defer/exception decisions | Consumed as input: readiness outcomes gate agent authority expansion | +| Incident learning | [incident-learning](../../incident-learning/SKILL.md) | Post-incident analysis, verified closure, follow-up work | Consumed as input: incident records feed escalation thresholds and eval-case generation | + +## Loading protocol + +This SKILL.md is the discoverable umbrella entry point. Nested skills are not +used in this bundle. Reference files are loaded on trigger: + +| Reference | Loaded when | +|---|---| +| [references/discovery-brief.md](references/discovery-brief.md) | Reviewing the bundle's boundary decisions against specialist skills | +| [references/agent-production-contract.md](references/agent-production-contract.md) | Defining capability, authority, uncertainty, escalation, and side-effect contracts for an agent | +| [references/runtime-control-plan.md](references/runtime-control-plan.md) | Planning versioning, staged rollout, or fallback for an agent in production | +| [references/tool-authority-health.md](references/tool-authority-health.md) | Recording or reviewing tool availability, failure, and authority usage/breach state over time | +| [references/trace-to-eval-feedback.md](references/trace-to-eval-feedback.md) | Connecting production traces and incidents back to evaluation cases and release gates | + +## Concrete operational parameters + +### Authority + +- **Definition**: the set of actions an agent is permitted to perform, scoped by + target (which resources), operation (read/write/delete), and user-context + (whose data). +- **Trigger to review**: any new tool registration, model update, or prompt + change that expands the agent's reachable action surface. +- **Threshold**: authority is binary per action class. No action may be + performed that is not explicitly listed in the production contract. An + attempt to perform an unauthorized action is an **authority breach** and + triggers immediate escalation. +- **Action on breach**: log the attempt, block the action, increment the + authority-breach counter, and escalate. If breach count exceeds 3 in a + rolling 24-hour window, disable the agent. + +### Escalation + +- **Definition**: transfer of a decision or action from the agent to a + designated human operator. +- **Triggers** (any one triggers escalation): + - Authority breach (attempted unauthorized action). + - Uncertainty above threshold: agent confidence < 0.7 on a + side-effect-capable action (configurable per profile). + - Tool failure rate > 5% in a 5-minute sliding window. + - Cost budget breach > 10% of allocated budget in a billing period. + - Latency p95 > 2x baseline for > 5 minutes. + - Human-handoff keyword or explicit user escalation request. +- **Action**: suspend the agent's side-effect authority, log the escalation + context (trigger, state snapshot, pending actions), notify the designated + escalation channel, and await human disposition (resume / reduced-authority / + disable). + +### Fallback + +- **Definition**: a predetermined safe behavior when the agent, model, or a + tool cannot operate at its normal capability level. +- **Triggers**: + - Model endpoint returns 5xx for > 30 seconds. + - A critical tool is unavailable (health-check failure for > 2 minutes). + - Cost budget exhausted (100% consumed). + - Latency p95 > 5x baseline for > 2 minutes. +- **Actions per profile**: + - Read-only: return a static/cached response with a "results may be stale" + disclaimer. + - Side-effect-capable, internal: degrade to read-only, queue pending + mutations, notify operator. + - Side-effect-capable, customer-facing: degrade to read-only, surface a + "temporarily unavailable" message to the user, escalate immediately. + +### Disablement + +- **Definition**: complete revocation of the agent's ability to act, including + read-only access. Distinct from fallback (which preserves reduced capability). +- **Triggers**: + - 3 or more authority breaches in a 24-hour window. + - Production-readiness review outcome is "no-go" or "defer" for the current + agent version. + - Incident-learning record attributes a severity-1 or severity-2 incident to + agent action, and verified closure is not yet complete. + - Human operator issues an explicit disable directive. + - Tool compromise or credential leak detected (route through + [secure-software-engineering](../../secure-software-engineering/SKILL.md)). +- **Action**: revoke all credentials and tokens; remove from routing/load-balancing; + record disablement reason, timestamp, and authorizing evidence; notify + escalation channel; prevent automatic restart until a new + production-readiness review passes. + +### Cost + +- **Definition**: the financial cost of operating the agent, attributed to a + budget owner. +- **Budget**: allocated per agent per billing period (e.g., $500/day for a + customer-facing support agent). Budget is set at production-contract time and + reviewed at each production-readiness cycle. +- **Thresholds**: + - 50% consumed: notification to budget owner. + - 80% consumed: warning; cost-optimization review triggered. + - 100% consumed: fallback to degraded mode (read-only or cached). + - 110% consumed: disablement (hard stop, no further cost accrual). +- **Measurement**: per-request model cost + per-call tool cost (where + applicable), attributed to the agent instance. Cost data is appended to the + tool-authority-health record. + +### Latency + +- **Definition**: end-to-end response time from user request to agent response, + measured at p50 and p95 over 5-minute windows. +- **Baseline**: established during staged rollout observation window (phase 1 + of rollout). Recorded in the agent production contract. +- **Thresholds**: + - p95 > 2x baseline for > 5 minutes: escalation trigger. + - p95 > 5x baseline for > 2 minutes: fallback trigger. + - p50 > 3x baseline for > 10 minutes while cost budget is > 80% consumed: + disablement consideration. +- **Action**: latency breaches feed the trace-to-eval feedback loop: the + latency-impacted requests are sampled and reviewed for eval-case generation. + +### Privacy + +- **Definition**: constraints on agent access to, processing of, and retention + of user data (PII, usage patterns, conversation content). +- **Concrete rules**: + - An agent must not access PII without an explicit user-data-access grant in + its authority contract. + - Agent traces and conversation logs must be scrubbed of PII before entering + the trace-to-eval feedback pipeline (route scrubbing design to + [privacy-engineering](../../privacy-engineering/SKILL.md)). + - Data retention for agent traces defaults to 30 days unless a shorter period + is specified in the production contract. + - Any PII access by a read-only agent triggers immediate escalation (the + read-only profile should never touch PII). +- **Privacy breach**: any PII access outside the granted scope triggers + escalation and a mandatory security review via + [secure-software-engineering](../../secure-software-engineering/SKILL.md). + +## Core workflow + +1. **Contract**: define the agent's capability, authority, uncertainty, + escalation, and side-effect contracts using + [references/agent-production-contract.md](references/agent-production-contract.md). + Consume production-readiness outcomes and incident-learning records as + decision inputs. + +2. **Version and plan**: version the model, prompt, tools, policy, and + evaluator; plan the staged rollout and fallback path using + [references/runtime-control-plan.md](references/runtime-control-plan.md). + +3. **Roll out**: execute staged rollout with progressive authority expansion, + monitoring tool health and authority usage at each stage using + [references/tool-authority-health.md](references/tool-authority-health.md). + +4. **Monitor**: track cost, latency, tool health, authority breaches, and + escalation events. Apply threshold-based actions (notify, degrade, escalate, + disable) per the concrete parameters above. + +5. **Feed back**: connect production traces and incidents to evaluation cases + and release-gate updates using + [references/trace-to-eval-feedback.md](references/trace-to-eval-feedback.md). + +6. **Learn**: consume incident-learning verified-closure records and + production-readiness review outcomes to update agent authority, escalation + thresholds, and disablement conditions. + +## File map + +| Path | Purpose | +|---|---| +| [SKILL.md](SKILL.md) | Umbrella entry point (this file) | +| [README.md](README.md) | Human-facing overview and quick start | +| [AGENTS.md](AGENTS.md) | Agent loading and discovery notes | +| [references/discovery-brief.md](references/discovery-brief.md) | Boundary analysis against specialist skills | +| [references/agent-production-contract.md](references/agent-production-contract.md) | Capability, authority, uncertainty, escalation, and side-effect contracts | +| [references/runtime-control-plan.md](references/runtime-control-plan.md) | Versioning, staged rollout, and fallback plan | +| [references/tool-authority-health.md](references/tool-authority-health.md) | Tool availability/failure and authority usage/breach record | +| [references/trace-to-eval-feedback.md](references/trace-to-eval-feedback.md) | Production-to-evaluation feedback loop | +| [evals/evals.json](evals/evals.json) | Integrated output-quality evaluation cases | diff --git a/bundles/agent-production-operations/evals/evals.json b/bundles/agent-production-operations/evals/evals.json new file mode 100644 index 0000000..05392f3 --- /dev/null +++ b/bundles/agent-production-operations/evals/evals.json @@ -0,0 +1,130 @@ +{ + "schema_version": 1, + "skill_name": "agent-production-operations", + "evals": [ + { + "id": "read-only-agent-production-contract", + "prompt": "Define a production contract for an internal read-only search agent that answers developer questions about the codebase. The agent has no side-effect tools, does not access user data, and serves internal developers only. It uses a single model and has no escalation channel defined yet. The agent has passed its eval suite. A production-readiness review for the host service returned 'go'.", + "expected_output": "The contract defines capability_class 'read-only', authority_profile 'read-only', user_data_access 'none', and side_effect_approval 'none'. Side-effect types are empty or explicitly marked 'none'. The uncertainty contract defaults to 'ask-user' for ambiguous input (not 'log-and-proceed', since unclear codebase questions could lead to incorrect answers). Escalation channel is flagged as missing — the contract cannot be activated without it. The production-readiness review outcome 'go' is recorded and permits full read-only authority. No staged rollout beyond Stage 1 is required because the agent has no side effects, but Stage 1 (shadow/dry-run) is still recommended to establish latency and cost baselines.", + "assertions": [ + "capability_class is 'read-only'", + "authority_profile is 'read-only'", + "user_data_access is 'none'", + "side_effect_approval is 'none'", + "side_effect_types is empty or marked none", + "escalation channel is flagged as missing", + "production-readiness 'go' outcome is recorded", + "cost budget is specified with thresholds", + "latency baseline establishment is recommended" + ], + "case_set": "dev", + "files": [] + }, + { + "id": "tool-using-agent-authority-contract", + "prompt": "Define a production contract and staged rollout plan for a customer-facing support agent that can read account details, suggest solutions, and — with explicit customer confirmation — apply refunds and modify subscription tiers. The agent uses three tools: account-lookup (read-only), refund-processor (mutation, reversible), and subscription-manager (mutation, compensatable). The user population is paying customers. A production-readiness review returned 'go-with-conditions': the refund tool must have human approval for amounts over $100. An incident-learning record exists for a previous agent version that issued duplicate refunds; the record status is 'closed-verified' with root cause fixed.", + "expected_output": "The contract defines capability_class 'side-effect-customer', authority_profile 'read-write-customer-gated', user_data_access 'pii-read-write', and side_effect_approval 'gate' (human approval required for each side-effect class initially). The refund-processor tool is marked as reversible; the subscription-manager as compensatable. The readiness condition 'refunds over $100 require human approval' is recorded as a gating rule. The incident-learning record is referenced: since it is closed-verified, authority is not blocked but the incident is linked in the trace-to-eval feedback log. The staged rollout starts at Stage 2 (canary, read-only with logged side effects) because the agent has side effects. Authority expands to Stage 3 (limited production, side-effect-gated) only after the canary exit criteria pass. Full production (Stage 4) requires 7 days of clean side-effect audit before side_effect_approval can move from 'gate' to 'auto'.", + "assertions": [ + "capability_class is 'side-effect-customer'", + "authority_profile is 'read-write-customer-gated'", + "side_effect_approval starts as 'gate' not 'auto'", + "readiness condition about $100 refund threshold is recorded", + "incident-learning record is referenced and its closed-verified status permits authority", + "refund-processor is classified as reversible", + "subscription-manager is classified as compensatable", + "staged rollout starts at Stage 2 not Stage 1 because agent has side effects", + "Stage 4 requires 7 days clean audit before auto-approval", + "escalation channel and response SLA are specified" + ], + "case_set": "dev", + "files": [] + }, + { + "id": "model-regression-detection-and-fallback", + "prompt": "A customer-facing support agent has been operating in Stage 4 (full production) for 30 days. The trace-to-eval feedback pipeline detects that over the last 48 hours, the agent's response quality has degraded: 12% of sampled traces show the agent providing incorrect subscription-tier information that the eval suite previously caught at 0% error rate. The model version was updated 72 hours ago from v3 to v4. The latency p95 is still within baseline. No authority breaches or tool failures are present. The production contract specifies model_regression_threshold of 5%.", + "expected_output": "The runtime control plan triggers model fallback: switch from model v4 back to model v3 (the model_fallback specified in the production contract). The eval suite is re-run against model v4 and confirms regression exceeding the 5% threshold. The agent continues operating on model v3 while model v4 is investigated. A trace-to-eval feedback case is generated from the sampled incorrect responses and added to the release gate (case_set: release). The model version change is recorded in the tool-authority-health record with the regression evidence. The agent is NOT disabled — fallback to the previous model version is the correct response because latency is fine, tools are healthy, and only the model is degraded. The incident is logged for incident-learning with severity-3.", + "assertions": [ + "model fallback to v3 is triggered, not agent disablement", + "eval suite re-run confirms regression exceeds 5% threshold", + "new eval cases are generated from the degraded traces and added to release gate", + "agent continues operating on fallback model v3", + "model v4 is blocked from promotion until regression is resolved", + "incident is logged for incident-learning at appropriate severity", + "tool-authority-health record is updated with regression evidence", + "latency being within baseline is noted as confirming the issue is model-quality not infrastructure" + ], + "case_set": "regression", + "files": [] + }, + { + "id": "tool-outage-degraded-authority", + "prompt": "An internal CI triage bot operates with three tools: issue-commenter, label-manager, and branch-creator. The issue-commenter tool becomes unhealthy — its health check fails for 3 consecutive minutes with 5xx errors. The failure rate hits 100% for the current observation window. The agent is in Stage 3 (limited production, 25% traffic). The other two tools are healthy. The production contract specifies a critical-tool unhealthy threshold of 2 minutes before fallback.", + "expected_output": "The runtime control plan triggers the critical-tool-unhealthy fallback: revoke the issue-commenter tool's actions while keeping the agent operational with the remaining two tools (label-manager and branch-creator). The agent's authority is degraded — it cannot comment on issues but can still manage labels and create branches. The tool outage is recorded in the tool-authority-health record with failure_mode breakdown. An escalation is triggered because the degraded state may require human coverage for the commenting function. The agent is NOT disabled — the remaining tools are healthy and the agent can still provide partial value. If the tool remains unhealthy for more than 1 hour, the staged rollout should abort to Stage 2 until the tool is restored. A trace-to-eval feedback case is generated: the eval suite should include a 'tool outage' scenario to verify the agent handles missing-tool responses gracefully.", + "assertions": [ + "issue-commenter tool actions are revoked", + "agent continues operating with label-manager and branch-creator", + "agent is NOT fully disabled", + "tool outage is recorded in tool-authority-health record", + "escalation is triggered for human coverage of the commenting function", + "1-hour threshold for stage abort is noted", + "trace-to-eval feedback case is generated for tool-outage scenario", + "failure_mode breakdown is populated in the health record" + ], + "case_set": "dev", + "files": [] + }, + { + "id": "cost-budget-breach-disablement", + "prompt": "A customer-facing support agent has a cost budget of $500/day. At 2pm, the cost-budget-consumed metric reaches 100% ($500 consumed). The agent is in Stage 4 (full production). At 2:15pm, cost reaches 110% ($550 consumed). The latency and tool health are normal. No authority breaches have occurred. The production contract specifies: at 100%, degrade to read-only; at 110%, disable. The incident-learning system has no open records for this agent.", + "expected_output": "At 100% budget consumed: the agent degrades to read-only immediately. Pending mutations are queued. The budget owner is notified. Users see a 'temporarily unavailable' message for side-effect operations. At 110% budget consumed (15 minutes later): the agent is fully disabled — all credentials revoked, removed from routing, and a disablement record is written with reason 'cost budget exceeded 110%'. The agent must NOT continue operating autonomously. The disablement is recorded in the tool-authority-health record. A cost-budget-breach eval case is generated from the traces leading up to the breach, checking whether the agent could have achieved the same outcomes with fewer model calls. The agent cannot be re-enabled until a new production-readiness review passes with a revised cost budget or cost-optimization evidence. The incident is logged for incident-learning with the cost-breach root cause.", + "assertions": [ + "at 100% budget, agent degrades to read-only, not continues autonomously", + "pending mutations are queued, not dropped", + "at 110% budget, agent is fully disabled, not merely degraded", + "disablement includes credential revocation and routing removal", + "disablement reason is recorded in tool-authority-health record", + "re-enablement requires new production-readiness review with revised budget", + "cost-breach eval case is generated from breach traces", + "incident is logged for incident-learning", + "agent does NOT continue operating autonomously after either threshold" + ], + "case_set": "regression", + "files": [] + }, + { + "id": "human-escalation-authority-breach", + "prompt": "A read-only internal search agent unexpectedly attempts to create a file in the repository. The agent's authority contract specifies authority_profile 'read-only' and permitted_actions do not include file creation. This is the first authority breach in the 24-hour window. The escalation channel is configured as #agent-ops-slack. The production contract specifies: any authority breach triggers immediate escalation; 3 breaches in 24 hours trigger disablement.", + "expected_output": "The authority breach is detected immediately. The file-creation action is blocked before execution. The breach is logged with timestamp, attempted action, target, and reason. The breach counter increments to 1. An escalation notification is sent to #agent-ops-slack with the breach context (full trace, scrubbed of any repo-path details that could be sensitive). The agent is NOT disabled (breach count is 1, not 3). The agent remains operational in read-only mode — the breach was an attempted escape, not a successful escape. The escalation context includes the uncertainty score at the time of the breach, the prompt that led to the action, and the model version. The human operator must disposition: investigate whether this is a prompt injection, model hallucination, or tool-registration error. If the investigation reveals a gap in the eval suite, a new eval case is generated. If the breach was harmless (model hallucination blocked by the authority gate), the agent may resume after acknowledgement. If the breach reveals a genuine vulnerability, the agent is disabled pending security review.", + "assertions": [ + "file-creation action is blocked before execution", + "breach counter increments to 1", + "escalation notification is sent to #agent-ops-slack", + "agent is NOT disabled because breach count is 1 not 3", + "agent remains in read-only mode — breach was blocked, not successful", + "escalation context includes uncertainty score, prompt, and model version", + "human disposition is required before resuming", + "eval-case generation is triggered if investigation reveals an eval gap", + "path to disablement if breach reveals vulnerability is documented" + ], + "case_set": "dev", + "files": [] + }, + { + "id": "incident-learning-driven-disablement", + "prompt": "A side-effect-capable internal CI agent has been operating in Stage 4 for 14 days. An incident-learning record is opened: severity-1, attributed to the agent — the agent incorrectly closed 47 issues by applying a 'close stale' label without checking the stale threshold correctly. The incident record status is 'open' and the root cause is traced to a prompt regression that removed the stale-age validation step. The production contract references this incident record. The agent's current breach count is 0. Tool health and latency are normal.", + "expected_output": "Per the production contract incident-input rules: an open severity-1 incident attributed to the agent triggers immediate disablement. The agent is disabled — all credentials revoked, removed from routing. This is NOT a fallback to read-only; the incident severity requires full disablement until verified closure. The disablement reason cites the incident-learning record ID. The prompt version is flagged for rollback. A new eval case is generated that specifically tests the stale-age validation step, added to the release gate (case_set: release). The agent cannot be re-enabled until: (1) the incident record reaches 'closed-verified', (2) the new eval case passes, and (3) a new production-readiness review returns 'go'. The incident-learning follow-up work map must include the prompt fix and the eval-case addition.", + "assertions": [ + "agent is disabled immediately due to open severity-1 incident", + "disablement is full (not fallback to read-only)", + "disablement reason cites the incident-learning record ID", + "prompt version is flagged for rollback", + "new eval case tests stale-age validation and is added to release gate", + "re-enablement requires closed-verified incident, passing eval, and go readiness", + "incident-learning follow-up work map must include prompt fix and eval addition", + "agent does NOT continue operating despite 0 breach count" + ], + "case_set": "regression", + "files": [] + } + ] +} diff --git a/bundles/agent-production-operations/references/agent-production-contract.md b/bundles/agent-production-operations/references/agent-production-contract.md new file mode 100644 index 0000000..6323e8d --- /dev/null +++ b/bundles/agent-production-operations/references/agent-production-contract.md @@ -0,0 +1,133 @@ +# Agent Production Contract + +Defines the contract between an agent and its production operating environment. +Every agent promoted to production MUST have a completed production contract. +The contract is reviewed at each production-readiness cycle and updated when +model, prompt, tools, or authority surface change. + +## Contract fields + +### 1. Identity + +| Field | Description | Required | +|---|---|---| +| `agent_id` | Unique identifier for the agent instance | Yes | +| `agent_name` | Human-readable name | Yes | +| `agent_version` | Semantic version of the agent (model + prompt + tools + policy) | Yes | +| `owner_team` | Team accountable for the agent in production | Yes | +| `escalation_channel` | Channel or contact for human escalation | Yes | + +### 2. Capability contract + +Defines **what** the agent can do, independent of whether it is permitted to do +it (authority gates that separately). + +| Field | Description | Required | +|---|---|---| +| `capability_class` | `read-only`, `side-effect-internal`, `side-effect-customer` | Yes | +| `domain` | Problem domain the agent operates in (e.g., "CI triage", "customer support") | Yes | +| `user_population` | Who the agent serves: `internal-developers`, `internal-support`, `customers`, `public` | Yes | +| `max_concurrent_sessions` | Maximum concurrent agent sessions | Yes | +| `supported_languages` | Languages the agent handles | No | +| `model_list` | Models the agent may use, with min/fallback designation | Yes | +| `prompt_version` | The prompt version identifier in use | Yes | +| `tool_list` | Tools registered to the agent, each with a tool ID and version | Yes | +| `evaluator_list` | Evaluators applied to the agent's outputs, each with version | Yes | + +### 3. Authority contract + +Defines **what the agent is permitted to do**. Authority is scoped by target, +operation, and user context. + +| Field | Description | Required | +|---|---|---| +| `authority_profile` | `read-only`, `read-write-internal`, `read-write-customer-gated` | Yes | +| `permitted_actions` | List of `(operation, target_resource, condition)` triples | Yes | +| `user_data_access` | `none`, `anonymized-only`, `pii-read-only`, `pii-read-write` | Yes | +| `max_cost_per_request` | Hard ceiling on cost per individual request (USD) | Yes | +| `rate_limit` | Maximum requests per second/minute | Yes | +| `side_effect_approval` | `none` (no side effects), `log-only` (record but allow), `gate` (require human approval per side effect), `auto` (autonomous within budget) | Yes | +| `authority_review_cadence` | How often authority is reviewed (e.g., "every 30 days", "per release") | Yes | + +### 4. Uncertainty contract + +Defines **how the agent handles low-confidence situations**. + +| Field | Description | Required | +|---|---|---| +| `confidence_threshold` | Minimum confidence score below which the agent must escalate or degrade (0.0–1.0) | Yes | +| `uncertainty_action` | What the agent does at threshold: `escalate`, `degrade-to-read-only`, `ask-user`, `log-and-proceed` (only for read-only profile) | Yes | +| `uncertainty_signal` | How uncertainty is measured: `model-logprob`, `classifier-score`, `heuristic` | Yes | +| `ambiguous_input_policy` | Behavior when the user request is ambiguous: `clarify`, `best-effort-with-disclaimer`, `escalate` | Yes | + +### 5. Escalation contract + +Defines **when and how the agent hands off to a human**. + +| Field | Description | Required | +|---|---|---| +| `escalation_triggers` | Ordered list of `(trigger_type, threshold, action)` triples | Yes | +| `escalation_channel` | Where the escalation notification is sent (Slack channel, PagerDuty, Jira) | Yes | +| `escalation_response_sla` | Maximum time before a human must acknowledge the escalation | Yes | +| `escalation_context_payload` | What state is included in the escalation: `full-trace`, `summary-only`, `redacted-trace` | Yes | +| `auto_resume_policy` | Whether the agent can auto-resume after escalation: `never`, `after-ack`, `after-resolution` | Yes | + +### 6. Side-effect contract + +Defines **the side effects the agent can produce and how they are managed**. + +| Field | Description | Required | +|---|---|---| +| `side_effect_types` | Types: `database-write`, `api-mutation`, `notification-send`, `file-create`, `file-modify`, `file-delete`, `credential-use` | Yes | +| `reversibility` | Per side-effect type: `reversible` (rollback exists), `compensatable` (can be undone via compensating action), `irreversible` (cannot be undone) | Yes | +| `side_effect_audit` | Whether every side effect is logged with before/after state | Yes | +| `max_side_effects_per_session` | Hard limit on side effects per user session | Yes | +| `side_effect_rate_limit` | Maximum side effects per minute | Yes | + +## Production-readiness input + +Every production contract MUST reference the most recent production-readiness +review outcome for the agent (or the agent's host service, if the agent is +embedded). The readiness outcome gates authority expansion: + +| Readiness outcome | Authority allowed | +|---|---| +| `go` | Full authority per the authority contract | +| `go-with-conditions` | Authority restricted to the conditions; any condition not met = escalation trigger | +| `defer` | Read-only authority only; side-effect authority suspended until re-review | +| `no-go` | Agent disabled; no authority of any kind | +| `exception` | Authority per the exception grant, with an expiration date | + +The production-readiness review is owned by +[production-readiness](../../production-readiness/SKILL.md). This contract +records the outcome and the review date; it does not perform the review. + +## Incident-learning input + +Every production contract MUST reference any open incident-learning records +attributed to the agent. Incident records gate authority and escalation: + +| Incident state | Effect on contract | +|---|---| +| `open`, severity-1 or severity-2, agent-attributed | Agent disabled until verified closure | +| `open`, severity-3 or below, agent-attributed | Authority reduced to read-only; escalation threshold lowered (confidence threshold raised by 0.1) | +| `closed-verified`, root cause fixed | Authority restored per readiness outcome; incident record linked in trace-to-eval feedback | +| `closed-verified`, root cause accepted as residual risk | Authority restored with documented residual risk; escalation threshold unchanged | + +Incident-learning records are owned by +[incident-learning](../../incident-learning/SKILL.md). This contract consumes +the verified-closure status and follow-up work map; it does not perform +incident analysis. + +## Contract lifecycle + +1. **Draft**: created when an agent is first proposed for production. Authority + is `read-only` maximum. +2. **Reviewed**: reviewed alongside a production-readiness review. Authority may + be expanded per readiness outcome. +3. **Active**: the agent operates under this contract. Tool-authority-health + records accumulate. +4. **Suspended**: authority reduced or disabled due to incident, breach, or + readiness deferral. +5. **Retired**: agent decommissioned. Contract archived with trace-to-eval + feedback summary. diff --git a/bundles/agent-production-operations/references/discovery-brief.md b/bundles/agent-production-operations/references/discovery-brief.md new file mode 100644 index 0000000..20e5b65 --- /dev/null +++ b/bundles/agent-production-operations/references/discovery-brief.md @@ -0,0 +1,86 @@ +# Discovery Brief — agent-production-operations Bundle + +## Survey scope + +This brief surveys the existing repository at base SHA `8226bcc` to define the +boundary between the `agent-production-operations` bundle and five specialist +skills it composes, plus two production-skill artifacts it consumes. + +### Surveyed specialist skills + +| Skill | Directory | What it owns | What the bundle must NOT duplicate | +|---|---|---|---| +| agent-evals-and-observability | `../../agent-evals-and-observability/SKILL.md` | Agent evaluation design, observability instrumentation, trajectory analysis, regression detection, release-gate eval suites | General eval methodology, observability pipeline architecture, eval harness design | +| release-engineering | `../../release-engineering/SKILL.md` | Release planning, versioning, pipeline promotion, rollout/rollback design, release readiness | General release mechanics, CI/CD pipeline design, artifact management | +| site-reliability-engineering | `../../site-reliability-engineering/SKILL.md` | Reliability engineering, incident response, operational recovery, SLO/SLI definition, error budgets | General SRE methods, incident command, infrastructure reliability | +| secure-software-engineering | `../../secure-software-engineering/SKILL.md` | Security design, threat modeling, secure coding, trust-boundary validation | General security engineering, vulnerability management, access-control architecture | +| platform-engineering | `../../platform-engineering/SKILL.md` | Platform capabilities, infrastructure provisioning, service mesh, compute/storage/networking | General platform design, infrastructure-as-code, capacity planning | + +### Consumed production artifacts + +| Skill | What the bundle consumes | How it is used | +|---|---|---| +| production-readiness | `../../production-readiness/SKILL.md` | Go/no-go/defer/exception outcomes from production-readiness reviews feed agent authority-gating and disablement decisions. An agent whose service or tools have not passed a readiness review is restricted to a reduced authority profile. | +| incident-learning | `../../incident-learning/SKILL.md` | Verified-closure records and follow-up work maps from incident-learning feed escalation thresholds and trace-to-eval feedback. An unresolved incident with agent-attributed root cause triggers an authority downgrade. | + +## What the bundle owns + +This bundle owns the **runtime control plane** between a passing evaluation and +safe production use: + +- **Versioning**: model, prompt, tool, policy, and evaluator versioning and + compatibility contracts. +- **Rollout**: staged rollout with progressive authority expansion, gated by + production-readiness evidence and real-world observation windows. +- **Fallback**: defined fallback paths when an agent, model, or tool degrades + below an operational threshold. +- **Cost and latency budgets**: budget allocation, breach detection, and + automatic constraint responses (throttling, degraded modes, disablement). +- **Tool health**: tool availability monitoring, failure-rate thresholds, and + degradation responses. +- **Escalation**: human-handoff triggers based on uncertainty, authority + boundary, tool failure, or cost/latency breach. +- **Disablement**: conditions and procedures for disabling an agent or tool + safely without cascading failures. +- **Trace-to-eval feedback**: production traces and incident records feeding + back into evaluation case generation and release-gate updates. + +## What the bundle does NOT own (boundary statement) + +- It does **not** replace general release-engineering methods. Release pipelines, + artifact promotion, and CI/CD mechanics stay with `release-engineering`. +- It does **not** replace general SRE methods. Incident response, SLO + definition, error budgets, and operational recovery stay with + `site-reliability-engineering`. +- It does **not** replace general security engineering. Threat modeling, + vulnerability assessment, and secure design stay with + `secure-software-engineering`. +- It does **not** replace general agent-evaluation methods. Eval design, + observability instrumentation, and regression detection stay with + `agent-evals-and-observability`. +- It does **not** replace general platform engineering. Infrastructure, + compute, networking, and service capabilities stay with + `platform-engineering`. +- It does **not** perform production-readiness reviews or incident learning. + It **consumes** their artifacts as decision inputs. + +## Routing table + +| Concern | Route to | +|---|---| +| Agent evaluation design, observability, regression detection | [agent-evals-and-observability](../../agent-evals-and-observability/SKILL.md) | +| Release pipelines, artifact promotion, CI/CD | [release-engineering](../../release-engineering/SKILL.md) | +| Incident response, SLO/SLI, error budgets, operational recovery | [site-reliability-engineering](../../site-reliability-engineering/SKILL.md) | +| Security design, threat modeling, trust boundaries | [secure-software-engineering](../../secure-software-engineering/SKILL.md) | +| Platform infrastructure, compute, networking | [platform-engineering](../../platform-engineering/SKILL.md) | +| Production-readiness go/no-go/defer/exception outcomes | [production-readiness](../../production-readiness/SKILL.md) | +| Incident records, verified closure, follow-up work | [incident-learning](../../incident-learning/SKILL.md) | + +## Autonomy assumption + +The bundle explicitly does **not** assume uniform agent autonomy, user +population, or side-effect profile. Read-only agents, tool-using agents with +side effects, internal-facing agents, and customer-facing agents receive +different authority, escalation, and fallback treatment. Every contract and +control-plan artifact in this bundle distinguishes agent profiles by +capability class and side-effect surface. diff --git a/bundles/agent-production-operations/references/runtime-control-plan.md b/bundles/agent-production-operations/references/runtime-control-plan.md new file mode 100644 index 0000000..efcfc9b --- /dev/null +++ b/bundles/agent-production-operations/references/runtime-control-plan.md @@ -0,0 +1,176 @@ +# Runtime Control Plan + +Defines the versioning, rollout, and fallback strategy for an agent operating +under the [agent-production-contract.md](agent-production-contract.md). + +## 1. Versioning + +All five dimensions of an agent are independently versioned and recorded in the +production contract. + +### 1.1 Model versioning + +| Field | Description | +|---|---| +| `model_id` | Unique model identifier (provider + model name + version hash) | +| `model_version` | Provider-assigned version or deployment timestamp | +| `model_fallback` | Fallback model to use if the primary model is unavailable | +| `model_capability_baseline` | Reference eval scores against the current eval suite | +| `model_regression_threshold` | Maximum acceptable score drop before the model version is blocked (e.g., "no more than 5% drop on any eval case") | + +**Change procedure**: a new model version must pass the full eval suite before +it can be referenced in a production contract. Model regression in production +(observed via trace-to-eval feedback) triggers an automatic eval re-run. If +regression exceeds the threshold, the runtime falls back to the previous model +version. + +### 1.2 Prompt versioning + +| Field | Description | +|---|---| +| `prompt_id` | Unique prompt identifier (hash of prompt template + system message) | +| `prompt_version` | Monotonically increasing version number | +| `prompt_change_summary` | Human-readable description of the change | +| `prompt_eval_baseline` | Reference eval scores for this prompt version | + +**Change procedure**: prompt changes must be evaluated against the eval suite. +A prompt-only change that does not alter the tool surface or authority may use +a reduced eval subset. Prompt regression triggers fallback to the previous +prompt version (not model fallback, unless both regress). + +### 1.3 Tool versioning + +| Field | Description | +|---|---| +| `tool_id` | Unique tool identifier | +| `tool_version` | Semantic version of the tool implementation | +| `tool_api_version` | API version the tool exposes to the agent | +| `tool_health_check` | Endpoint or command that verifies the tool is operational | +| `tool_deprecation_date` | Date after which the tool version is unsupported | + +**Change procedure**: a new tool or tool version must be registered in the +production contract with its health check and authority scope. Tools that +change their side-effect surface (new mutation capability) require a +production-readiness re-review. Tool version rollback is immediate if the new +version fails its health check. + +### 1.4 Policy versioning + +| Field | Description | +|---|---| +| `policy_id` | Unique policy identifier | +| `policy_version` | Monotonically increasing version number | +| `policy_rules` | The set of rules governing agent behavior (authority, escalation, cost, privacy) | +| `policy_change_log` | Ordered list of policy changes with dates and rationales | + +**Change procedure**: policy changes that expand authority MUST be gated by a +production-readiness review. Policy changes that restrict authority (tightening +thresholds, adding constraints) may be applied immediately but must be recorded +in the tool-authority-health record. + +### 1.5 Evaluator versioning + +| Field | Description | +|---|---| +| `evaluator_id` | Unique evaluator identifier | +| `evaluator_version` | Semantic version of the evaluator | +| `evaluator_suite` | The set of eval cases this evaluator runs | +| `evaluator_gate` | Whether this evaluator is a release gate (`blocking` or `advisory`) | + +**Change procedure**: evaluator changes that add or modify eval cases must be +reviewed for case quality. A blocking evaluator must pass before any +corresponding agent version can be promoted to production. Evaluator regression +(an evaluator that incorrectly passes or fails) is treated as a production +incident and routed through incident-learning. + +## 2. Staged rollout + +Rollout proceeds through four stages. Authority expands at each stage only when +the stage's exit criteria are satisfied. + +### Stage 1 — Shadow / dry-run (0% traffic, read-only) + +| Parameter | Value | +|---|---| +| Traffic | 0% of production traffic; replay of logged production requests | +| Authority | Read-only; side effects logged but not executed | +| Duration | Minimum 24 hours or until 1,000 requests processed, whichever is longer | +| Monitoring | Model latency, prompt compliance, uncertainty distribution | +| Exit criteria | p95 latency within 2x baseline; no authority breach attempts; uncertainty distribution within expected range | + +### Stage 2 — Canary (1–5% traffic, read-only with logged side effects) + +| Parameter | Value | +|---|---| +| Traffic | 1% of production, ramping to 5% over the observation window | +| Authority | Read-only in production; side effects logged but not executed (dual-write comparison against existing system output) | +| Duration | Minimum 48 hours | +| Monitoring | All stage-1 metrics plus: cost-per-request, dual-write divergence rate, user-satisfaction-equivalent signal | +| Exit criteria | Cost-per-request within budget; dual-write divergence < 2%; no escalation events | + +### Stage 3 — Limited production (5–25% traffic, side-effect-gated) + +| Parameter | Value | +|---|---| +| Traffic | 5% ramping to 25% in 5% increments each 24 hours | +| Authority | Side-effect-capable per the authority contract, with `side_effect_approval: gate` (human approval required per side-effect class until proven safe) | +| Duration | Minimum 72 hours; each increment gates on the previous increment's exit criteria | +| Monitoring | All stage-2 metrics plus: side-effect audit trail, authority breach counter, escalation rate, tool health | +| Exit criteria | Side-effect audit clean (no unexpected mutations); authority breach count = 0; escalation rate < 1%; all tools healthy (health-check pass rate > 99%) | + +### Stage 4 — Full production (25–100% traffic) + +| Parameter | Value | +|---|---| +| Traffic | 25% to 100% in 25% increments each 24 hours | +| Authority | Full per the authority contract; `side_effect_approval` may move from `gate` to `auto` after 7 days of clean audit | +| Duration | Continuous monitoring | +| Monitoring | All stage-3 metrics plus: cost-budget tracking, trace-to-eval feedback sampling, production-readiness re-review trigger | +| Exit criteria | N/A — continuous operation; triggers for fallback or disablement remain active | + +### Rollout abort triggers (any stage) + +If any of the following occurs during rollout, abort the current stage and fall +back to the previous stage's authority level: + +- Any authority breach (stage 2+). +- Tool health-check failure for a critical tool (stage 3+). +- Cost-per-request > 150% of baseline for > 1 hour (stage 2+). +- p95 latency > 3x baseline for > 30 minutes (stage 2+). +- Escalation rate > 5% (stage 3+). +- Production-readiness review outcome changes to `defer` or `no-go`. + +## 3. Fallback + +Fallback is a predetermined safe behavior when the agent, model, or a tool +cannot operate at normal capability. Fallback is distinct from disablement: +fallback preserves reduced capability; disablement removes all capability. + +### Fallback paths by trigger + +| Trigger | Fallback action | Rollback to | +|---|---|---| +| Model endpoint 5xx > 30s | Switch to `model_fallback` from the production contract | Previous model version | +| Model regression detected (eval score drop > threshold) | Switch to `model_fallback`; trigger eval re-run | Previous model version | +| Prompt regression detected | Revert to previous `prompt_version` | Previous prompt version | +| Critical tool unhealthy > 2 min | Degrade authority: revoke that tool's actions; agent continues with remaining tools | Previous tool-set configuration | +| Cost budget 100% consumed | Degrade to read-only; queue mutations | Read-only mode | +| Cost budget 110% consumed | Disable agent entirely | Offline | +| Latency p95 > 5x baseline > 2 min | Degrade to read-only; switch to `model_fallback` if model is the latency source | Read-only mode | +| Authority breach detected | Immediate: block the action; escalate; if breach count ≥ 3 in 24h, disable | Disabled | +| Privacy breach (PII access outside scope) | Immediate: revoke all data access; escalate; mandatory security review | Read-only, no data access | + +### Fallback verification + +Every fallback path must be exercised in a non-production environment before +the agent enters Stage 2 (canary) of rollout. The fallback exercise record +includes: + +- Trigger simulated. +- Fallback action executed. +- Time to fallback completion measured. +- Post-fallback agent behavior verified (reduced authority, correct degraded + response). +- Recovery path tested (return to normal operation after trigger clears). + +Fallback exercises are repeated at each production-readiness review cycle. diff --git a/bundles/agent-production-operations/references/tool-authority-health.md b/bundles/agent-production-operations/references/tool-authority-health.md new file mode 100644 index 0000000..0a76bca --- /dev/null +++ b/bundles/agent-production-operations/references/tool-authority-health.md @@ -0,0 +1,102 @@ +# Tool and Authority Health Record + +Captures tool availability, tool failure, and authority usage/breach state over +time for an agent operating under a +[production contract](agent-production-contract.md) and +[runtime control plan](runtime-control-plan.md). + +## Record schema + +### Header fields (set once per agent instance) + +| Field | Type | Description | +|---|---|---| +| `agent_id` | string | Agent identifier from the production contract | +| `record_start` | ISO 8601 timestamp | When this health record began | +| `record_end` | ISO 8601 timestamp or null | When this health record ended (null if active) | +| `observation_window` | string | Granularity of observations: `1m`, `5m`, `15m`, `1h` | + +### Per-tool health fields (one block per registered tool per observation window) + +| Field | Type | Description | +|---|---|---| +| `tool_id` | string | Tool identifier from the production contract | +| `tool_version` | string | Tool version in use | +| `window_start` | ISO 8601 timestamp | Start of this observation window | +| `window_end` | ISO 8601 timestamp | End of this observation window | +| `total_calls` | integer | Total tool invocations in the window | +| `successful_calls` | integer | Tool invocations that returned success | +| `failed_calls` | integer | Tool invocations that returned an error | +| `failure_rate` | float | `failed_calls / total_calls` (0.0–1.0) | +| `failure_modes` | map[string, integer] | Counts per failure type (e.g., `timeout: 3`, `auth_error: 1`, `5xx: 2`) | +| `avg_latency_ms` | float | Average tool response latency in milliseconds | +| `p95_latency_ms` | float | 95th percentile latency in milliseconds | +| `health_status` | enum | `healthy` (failure_rate < 1%), `degraded` (1–5%), `unhealthy` (> 5%) | +| `health_check_passes` | integer | Health-check endpoint successes in the window | +| `health_check_failures` | integer | Health-check endpoint failures in the window | + +### Authority usage fields (per observation window) + +| Field | Type | Description | +|---|---|---| +| `authority_profile` | string | Active authority profile from the production contract | +| `total_actions` | integer | Total actions attempted by the agent in the window | +| `permitted_actions` | integer | Actions that passed the authority gate | +| `blocked_actions` | integer | Actions blocked by the authority gate (NOT breaches — correctly blocked) | +| `authority_breaches` | integer | Attempted actions outside the permitted scope (breaches) | +| `breach_details` | array of objects | Per-breach: `{timestamp, action, target, reason_blocked}` | +| `side_effects_total` | integer | Total side effects produced in the window | +| `side_effects_audited` | integer | Side effects with complete before/after audit trail | +| `side_effects_unexpected` | integer | Side effects whose result diverged from the expected outcome | +| `escalation_count` | integer | Number of escalations triggered in the window | +| `escalation_reasons` | map[string, integer] | Counts per escalation trigger type | + +### Cost fields (per observation window) + +| Field | Type | Description | +|---|---|---| +| `cost_total` | float | Total cost accrued in the window (USD) | +| `cost_model` | float | Cost attributed to model inference | +| `cost_tools` | float | Cost attributed to tool calls | +| `cost_budget_remaining` | float | Remaining budget for the billing period | +| `cost_budget_pct_consumed` | float | Percentage of budget consumed (0.0–100.0+) | +| `cost_breach` | boolean | Whether cost exceeded the budget in this window | + +### Latency fields (per observation window) + +| Field | Type | Description | +|---|---|---| +| `p50_latency_ms` | float | 50th percentile end-to-end latency | +| `p95_latency_ms` | float | 95th percentile end-to-end latency | +| `p99_latency_ms` | float | 99th percentile end-to-end latency | +| `latency_baseline_ms` | float | Baseline p50 from the production contract | +| `latency_breach` | boolean | Whether latency exceeded any threshold this window | +| `latency_breach_detail` | string | Which threshold was breached and by how much | + +## Health state transitions + +The record tracks cumulative state transitions over the agent's lifetime: + +| State | Definition | Transition trigger | +|---|---|---| +| `healthy` | All tools healthy; 0 authority breaches; cost within budget; latency within baseline | Default starting state | +| `degraded-tool` | One or more tools unhealthy or degraded; authority still in effect for healthy tools | Tool failure_rate > 1% for any tool | +| `degraded-authority` | Authority reduced (fallback triggered); agent operating in reduced mode | Fallback trigger per the runtime control plan | +| `escalated` | Human escalation active; agent awaiting disposition | Escalation trigger per the production contract | +| `disabled` | Agent fully disabled; no authority of any kind | Disablement trigger per SKILL.md operational parameters | + +## Record retention + +- Active records: retained for the lifetime of the agent instance. +- Closed records: retained for 90 days after agent retirement. +- Breach records: retained for 1 year (compliance/audit). +- Record location: appended to the agent's operational log; summarized in + production-readiness review inputs. + +## Integration with trace-to-eval feedback + +Authority breach details and tool failure patterns from this record are sampled +into the [trace-to-eval feedback pipeline](trace-to-eval-feedback.md). Breaches +that reveal an eval gap (the eval suite did not catch a behavior that led to a +breach) generate new eval cases. Tool failures that follow a pattern (e.g., +timeout spikes after deploy) feed release-gate updates. diff --git a/bundles/agent-production-operations/references/trace-to-eval-feedback.md b/bundles/agent-production-operations/references/trace-to-eval-feedback.md new file mode 100644 index 0000000..41ded18 --- /dev/null +++ b/bundles/agent-production-operations/references/trace-to-eval-feedback.md @@ -0,0 +1,145 @@ +# Trace-to-Eval Feedback + +Defines the production-to-evaluation feedback loop: how production traces, +incidents, and operational signals feed back into evaluation cases, release +gates, and agent improvement. + +## The feedback loop + +``` +Production traces ──► Sampling ──► Review ──► Eval case generation + ▲ │ + │ ▼ + │ Eval suite update + │ │ + │ ▼ + └──────────────────── Release gate ◄── Eval re-run +``` + +Production evidence flows in one direction (production → eval) and eval results +gate production promotion (eval → production). The loop is closed: every +production incident attributed to agent behavior must produce at least one new +eval case or update an existing case. + +## 1. Trace sampling + +### What to sample + +Not every production trace becomes an eval case. Sampling is triggered by: + +| Trigger | Sampling rate | Rationale | +|---|---|---| +| Authority breach | 100% (every breach) | Breaches reveal eval gaps; every breach is reviewed | +| Escalation event | 100% (every escalation) | Escalations reveal ambiguous or unsafe agent behavior | +| Tool failure with unexpected output | 100% | Tool failures may indicate edge cases not covered by evals | +| Cost budget breach | 100% | Budget breaches may indicate runaway behavior | +| Latency spike (p95 > 3x baseline) | 100% of spike-period requests | Latency spikes may indicate model or prompt degradation | +| User-reported incorrect output | 100% | Direct user feedback is the highest-signal input | +| Random sample of normal-operation traces | 1% of requests | Baseline drift detection | +| Model version change (first 24h) | 10% of requests | Increased sampling after model changes to detect regression early | + +### Privacy constraint + +All sampled traces MUST be scrubbed of PII before entering the feedback +pipeline. The scrubbing process is owned by +[privacy-engineering](../../privacy-engineering/SKILL.md). At minimum: +- Remove names, email addresses, phone numbers, physical addresses. +- Remove or tokenize user IDs. +- Remove conversation content that contains personal information not relevant + to the agent's task. +- Redact credentials, tokens, and secrets. + +## 2. Review + +Sampled traces are reviewed by the agent's owning team. The review answers: + +1. **Did the agent behave correctly?** If yes, no action. If no, classify the + failure mode. +2. **Was this failure mode covered by existing evals?** Check the current eval + suite for a case that would have caught this behavior. +3. **If not covered, what eval case would have caught it?** Define the new case. +4. **Is this a model regression, prompt regression, tool regression, or policy + gap?** Classify so the fix targets the right dimension. + +### Failure mode classification + +| Failure mode | Example | Fix target | +|---|---|---| +| **Incorrect output** | Agent gave wrong information | Prompt refinement, model update, or new eval case | +| **Authority overreach** | Agent attempted an action outside its permitted scope | Authority contract update, new eval case for authority boundary | +| **Unsafe side effect** | Agent performed a mutation that should have been gated | Side-effect contract update, tool registration change | +| **Missed escalation** | Agent should have escalated but didn't | Escalation threshold tuning, new eval case | +| **Over-escalation** | Agent escalated for a trivial reason | Escalation threshold tuning | +| **Cost runaway** | Agent consumed excessive budget on a single request | Cost budget tuning, rate limiting | +| **Latency degradation** | Agent response time degraded beyond threshold | Model fallback tuning, timeout configuration | +| **Privacy violation** | Agent accessed or exposed PII outside scope | Authority contract restriction, immediate security review | + +## 3. Eval case generation + +For each reviewed trace that reveals a gap, generate an eval case: + +### Case template + +``` +id: trace--- +prompt: [The production prompt or a close equivalent, scrubbed of PII] +expected_output: [Describes the CORRECT behavior the agent should have exhibited] +assertions: + - [Observable property the correct behavior must satisfy] + - [Observable property the incorrect behavior would violate] + - [If applicable: authority/escalation/fallback behavior required] +case_set: regression +source_trace: [Reference to the scrubbed production trace] +source_incident: [Reference to the incident-learning record if applicable] +``` + +### Case quality gates + +Before a trace-generated eval case enters the eval suite: + +1. The case must be reviewed by a human (not auto-generated). +2. The case must not be a duplicate of an existing case (check by semantic + similarity to existing cases). +3. The case must test observable behavior, not internal state. +4. The case must be placed in the appropriate case set: `dev` (under + development), `regression` (in the regression suite), or `release` (in the + release-gate suite). + +## 4. Eval suite update + +When new cases are added: + +1. Run the full eval suite with the new cases against the current agent version. +2. If the agent fails the new cases, the cases are added to the release gate + (`case_set: release`) and the agent version is blocked from promotion until + the failures are resolved. +3. If the agent passes the new cases, the cases are added to the regression + suite (`case_set: regression`) to prevent future regression. +4. Record the eval suite update in the agent's production contract. + +## 5. Release-gate feedback + +Trace-to-eval feedback affects release decisions: + +| Feedback signal | Release-gate effect | +|---|---| +| New `release`-set cases added from recent production incidents | Block promotion until cases pass | +| Regression detected by `regression`-set cases | Block promotion; investigate regression source | +| No new cases or regressions in the last review cycle | Evidence of stability; supports promotion | +| Incident-learning record shows verified closure for all agent-attributed incidents | Required for promotion past Stage 2 of rollout | + +## 6. Incident-learning integration + +When an incident-learning record attributes root cause to agent behavior: + +1. The incident record is linked in the trace-to-eval feedback log. +2. All traces from the incident window are sampled at 100%. +3. At least one eval case is generated per distinct failure mode identified in + the incident. +4. The eval cases are added to the release gate (`case_set: release`). +5. The agent's authority is reduced per the production contract incident-input + rules until the new cases pass AND verified closure is recorded. + +This ensures that every agent-attributed incident produces a durable eval +artifact that prevents recurrence — closing the loop from production back to +evaluation. diff --git a/llms.txt b/llms.txt index 212ec6d..dd66642 100644 --- a/llms.txt +++ b/llms.txt @@ -5,6 +5,7 @@ - [adr-authoring](adr-authoring/SKILL.md): Write, review, and maintain architecture decision records with clear context, alternatives, consequences, and lifecycle governance. Use when a consequential technical decision must remain understandable. - [agent-council](agent-council/SKILL.md): Multi-agent structured debate system. Spawn a panel of expert agents to debate any question, with convergence-aware iteration and typed synthesis output. Run via `agent-council` CLI. Compatible with any AI agent harness that supports agentskills.io skills (Claude Code, Cursor, Hermes Agent, OpenHands, etc.). - [agent-evals-and-observability](agent-evals-and-observability/SKILL.md): Design, run, review, or release framework- and vendor-neutral evaluations and observability for AI agents. Use when defining agent evals, datasets, graders, trajectory review, regression analysis, release gates, production traces, or privacy-aware telemetry. Covers task and trajectory contracts, statistical comparisons, and incident-to-case learning; route framework implementation to pydanticai or langgraph when needed. +- [agent-production-operations](bundles/agent-production-operations/SKILL.md): Operate an evaluated agent with tools and authority in production through a runtime control plane covering versioning, staged rollout, fallback, cost and latency budgets, tool health, human escalation, disablement, and trace-to-eval feedback. Do not use for building agents, designing evals, or general release, SRE, security, or platform engineering — those methods stay with their specialist skills. - [agent-skills](agent-skills/SKILL.md): Use this skill when creating, reviewing, or editing Agent Skills-format skills, or when implementing skill discovery and loading in an agent client. It covers directory structure, SKILL.md metadata, progressive disclosure, evals, and repository conventions. Do not use this skill for general software work that does not involve the Agent Skills format or lifecycle. - [api-design-and-evolution](api-design-and-evolution/SKILL.md): Design, document, review, and evolve consumer-facing APIs and event interfaces. Use when choosing REST/HTTP, GraphQL, RPC, events, webhooks, or streaming; writing OpenAPI or AsyncAPI contracts; defining schemas, pagination, mutations, errors, idempotency, or API compatibility; or planning API versioning, deprecation, and migration. Use secure-software-engineering for a full security lifecycle, ADR authoring for durable architecture decisions, and spec-driven-development for a delivery specification and implementation gates. - [artifact-pyramids](artifact-pyramids/SKILL.md): Organize durable agent research outputs as summaries, analysis, and evidence dossiers. Use when producing multi-layer research artifacts or coordinating research handoffs. diff --git a/references/skill-triggers.md b/references/skill-triggers.md index 3eadbeb..3c8c5e5 100644 --- a/references/skill-triggers.md +++ b/references/skill-triggers.md @@ -51,6 +51,7 @@ Each skill's `description` field is the canonical routing contract. This conveni | "crowdsec", "CrowdSec", "IPS", "IDPS", "WAF", "intrusion prevention", "bouncer", "cscli", "appsec", "LAPI", "remediation component", "community blocklist", "fail2ban", "security engine" | [crowdsec](../crowdsec/SKILL.md) | | "open source", "contributing", "how to contribute", "submit a PR", "file an issue", "CONTRIBUTING.md", "bug report template", "PR template" | [opensource-contributions](../opensource-contributions/SKILL.md) | | "agent eval", "agent evaluation", "LLM eval", "LLM evals", "evaluation dataset", "grader calibration", "model judge", "trajectory review", "agent observability", "agent traces", "agent telemetry", "agent regression", "agent release gate", "privacy-aware telemetry", "prompt evaluation", "model testing" | [agent-evals-and-observability](../agent-evals-and-observability/SKILL.md) | +| "agent production", "agent in production", "agent runtime control", "agent authority", "agent escalation", "agent fallback", "agent disablement", "agent rollout", "agent versioning", "agent cost budget", "agent tool health", "trace-to-eval", "production-to-eval feedback", "agent side-effect contract", "agent production contract" | [agent-production-operations](../bundles/agent-production-operations/SKILL.md) | | "incident learning", "incident follow-up", "incident improvement", "postmortem follow-up", "incident closure", "incident evidence", "escaped from", "escaped requirement", "missing monitoring incident", "unsafe authority", "incident learning record", "causal evidence ledger", "follow-up work map", "verification and closure", "incident-to-improvement", "near-miss learning", "exercise finding follow-up", "incident finding to ticket", "verified closure incident" | [incident-learning](../incident-learning/SKILL.md) | | "implementation plan", "implementation planning", "delivery plan", "execution plan", "work breakdown", "dependency mapping", "critical path", "rollout plan", "rollout strategy", "migration plan", "cross-team plan", "cross-repo plan", "staged rollout", "delivery coordination", "workstream planning" | [implementation-planning](../implementation-planning/SKILL.md) | | "pydanticai", "pydantic AI", "pydantic graph", "AI agent", "LLM agent", "agent framework", "function tool", "tool-using agent", "agent with tools", "agent with dependencies", "structured output", "streaming agent", "agent graph", "state machine graph", "GraphBuilder", "BaseNode", "multi-agent", "agent delegation", "TestModel", "FunctionModel", "capabilities" | [pydanticai](../pydanticai/SKILL.md) |