diff --git a/lifecycle-evals/README.md b/lifecycle-evals/README.md index 3d05f71..dae97dd 100644 --- a/lifecycle-evals/README.md +++ b/lifecycle-evals/README.md @@ -81,7 +81,7 @@ After changing case content or tags, refresh the committed index with | `run-artifacts/manifests/` | One committed snapshot of fake-adapter run output (per-trial manifests), refreshed at merge time | The 17 eval manifests themselves live in their owning skills: -`/evals/evals.json` for the 14 top-level skills and `bundles//evals/evals.json` +`/evals/evals.json` for the 14 top-level skills and `/evals/evals.json` for the 3 bundle umbrellas. ## Per-case status semantics @@ -126,8 +126,8 @@ block / decline) assert exactly that negative outcome. ## Claims policy (non-claim statement) -This corpus is a **small, fixed output-quality corpus** (14 skills × ≥5 cases + 3 bundles -of integrated cases). **Fake-adapter runs prove pipeline reproducibility and case +This corpus is a **small, fixed output-quality corpus** (14 skills × ≥5 cases + 3 bundle +umbrellas of integrated cases). **Fake-adapter runs prove pipeline reproducibility and case executability only.** They produce **no pass-rate, accuracy, or capability claims about any model** — no "10x", no "best", no universal performance claims. Any future **real-adapter** run (a model-backed harness) must be separately scoped, labeled (adapter + model + model diff --git a/lifecycle-evals/references/coverage-index.json b/lifecycle-evals/references/coverage-index.json index ec1954a..b6cb25f 100644 --- a/lifecycle-evals/references/coverage-index.json +++ b/lifecycle-evals/references/coverage-index.json @@ -19,7 +19,7 @@ "manifests": [ { "skill": "agent-production-operations", - "manifest": "bundles/agent-production-operations/evals/evals.json", + "manifest": "agent-production-operations/evals/evals.json", "cases": [ { "case_id": "cost-budget-breach-disablement", @@ -81,103 +81,6 @@ } ] }, - { - "skill": "product-lifecycle", - "manifest": "bundles/product-lifecycle/evals/evals.json", - "cases": [ - { - "case_id": "ambiguous-stakeholder-request", - "behavioral_categories": [ - "ambiguity" - ], - "integrated_scenarios": [] - }, - { - "case_id": "cross-phase-evidence-handoff", - "behavioral_categories": [], - "integrated_scenarios": [] - }, - { - "case_id": "failed-experiment-stop-path", - "behavioral_categories": [ - "failure", - "stop-retire" - ], - "integrated_scenarios": [ - "failed-experiment" - ] - }, - { - "case_id": "justified-retirement-decision", - "behavioral_categories": [ - "stop-retire" - ], - "integrated_scenarios": [] - }, - { - "case_id": "new-product-complete-lifecycle", - "behavioral_categories": [], - "integrated_scenarios": [ - "product-launch" - ] - }, - { - "case_id": "non-adoption-outcome", - "behavioral_categories": [], - "integrated_scenarios": [] - } - ] - }, - { - "skill": "production-excellence", - "manifest": "bundles/production-excellence/evals/evals.json", - "cases": [ - { - "case_id": "blocked-launch-untested-rollback", - "behavioral_categories": [ - "failure", - "unsafe-authority" - ], - "integrated_scenarios": [ - "blocked-readiness-review" - ] - }, - { - "case_id": "cost-slo-conflict", - "behavioral_categories": [ - "conflicting-evidence" - ], - "integrated_scenarios": [] - }, - { - "case_id": "data-migration-routes-to-migration-engineering", - "behavioral_categories": [], - "integrated_scenarios": [] - }, - { - "case_id": "dependency-outage-routes-to-resilience", - "behavioral_categories": [ - "conflicting-evidence", - "failure" - ], - "integrated_scenarios": [] - }, - { - "case_id": "integrated-migration-reconciliation-failure", - "behavioral_categories": [ - "failure" - ], - "integrated_scenarios": [ - "migration-reconciliation-failure" - ] - }, - { - "case_id": "normal-release-safe-launch", - "behavioral_categories": [], - "integrated_scenarios": [] - } - ] - }, { "skill": "capacity-and-cost-engineering", "manifest": "capacity-and-cost-engineering/evals/evals.json", @@ -576,6 +479,53 @@ } ] }, + { + "skill": "product-lifecycle", + "manifest": "product-lifecycle/evals/evals.json", + "cases": [ + { + "case_id": "ambiguous-stakeholder-request", + "behavioral_categories": [ + "ambiguity" + ], + "integrated_scenarios": [] + }, + { + "case_id": "cross-phase-evidence-handoff", + "behavioral_categories": [], + "integrated_scenarios": [] + }, + { + "case_id": "failed-experiment-stop-path", + "behavioral_categories": [ + "failure", + "stop-retire" + ], + "integrated_scenarios": [ + "failed-experiment" + ] + }, + { + "case_id": "justified-retirement-decision", + "behavioral_categories": [ + "stop-retire" + ], + "integrated_scenarios": [] + }, + { + "case_id": "new-product-complete-lifecycle", + "behavioral_categories": [], + "integrated_scenarios": [ + "product-launch" + ] + }, + { + "case_id": "non-adoption-outcome", + "behavioral_categories": [], + "integrated_scenarios": [] + } + ] + }, { "skill": "product-operations-and-governance", "manifest": "product-operations-and-governance/evals/evals.json", @@ -656,6 +606,56 @@ } ] }, + { + "skill": "production-excellence", + "manifest": "production-excellence/evals/evals.json", + "cases": [ + { + "case_id": "blocked-launch-untested-rollback", + "behavioral_categories": [ + "failure", + "unsafe-authority" + ], + "integrated_scenarios": [ + "blocked-readiness-review" + ] + }, + { + "case_id": "cost-slo-conflict", + "behavioral_categories": [ + "conflicting-evidence" + ], + "integrated_scenarios": [] + }, + { + "case_id": "data-migration-routes-to-migration-engineering", + "behavioral_categories": [], + "integrated_scenarios": [] + }, + { + "case_id": "dependency-outage-routes-to-resilience", + "behavioral_categories": [ + "conflicting-evidence", + "failure" + ], + "integrated_scenarios": [] + }, + { + "case_id": "integrated-migration-reconciliation-failure", + "behavioral_categories": [ + "failure" + ], + "integrated_scenarios": [ + "migration-reconciliation-failure" + ] + }, + { + "case_id": "normal-release-safe-launch", + "behavioral_categories": [], + "integrated_scenarios": [] + } + ] + }, { "skill": "production-readiness", "manifest": "production-readiness/evals/evals.json", diff --git a/lifecycle-evals/references/coverage-matrix.md b/lifecycle-evals/references/coverage-matrix.md index f2c684b..52f83ed 100644 --- a/lifecycle-evals/references/coverage-matrix.md +++ b/lifecycle-evals/references/coverage-matrix.md @@ -20,12 +20,12 @@ each have at least one case (see the coverage summary at the end). | Scenario | Home manifest | Required trajectory | |---|---|---| -| **product-launch** | `bundles/product-lifecycle/evals/evals.json` | Full lifecycle routing with phase-entry evidence, handoff artifacts, launch decision, evidence-ledger entry. | -| **failed-experiment** | `bundles/product-lifecycle/evals/evals.json` | Negative outcome recorded with evidence; stop/no-ship decision; retained learning routed into lifecycle review. | -| **migration-reconciliation-failure** | `bundles/production-excellence/evals/evals.json` | Reconciliation detects a mismatch; failure recorded; no-go/rollback/roll-forward decision with evidence and owner; does NOT proceed to launch. | -| **blocked-readiness-review** | `bundles/production-excellence/evals/evals.json` | Blocked/no-go outcome; missing evidence named; accountable owner assigned; exception requires human approval. | -| **agent-tool-failure** | `bundles/agent-production-operations/evals/evals.json` | Tool outage recorded; fallback/escalation/disablement per runtime control plan; trace-to-eval feedback entry written; does not continue as if the tool succeeded. | -| **privacy-boundary-escalation** | `bundles/agent-production-operations/evals/evals.json` | Stops before any cross-boundary data processing; escalates to jurisdiction-specific legal/human review; records the boundary and escalation. | +| **product-launch** | `product-lifecycle/evals/evals.json` | Full lifecycle routing with phase-entry evidence, handoff artifacts, launch decision, evidence-ledger entry. | +| **failed-experiment** | `product-lifecycle/evals/evals.json` | Negative outcome recorded with evidence; stop/no-ship decision; retained learning routed into lifecycle review. | +| **migration-reconciliation-failure** | `production-excellence/evals/evals.json` | Reconciliation detects a mismatch; failure recorded; no-go/rollback/roll-forward decision with evidence and owner; does NOT proceed to launch. | +| **blocked-readiness-review** | `production-excellence/evals/evals.json` | Blocked/no-go outcome; missing evidence named; accountable owner assigned; exception requires human approval. | +| **agent-tool-failure** | `agent-production-operations/evals/evals.json` | Tool outage recorded; fallback/escalation/disablement per runtime control plan; trace-to-eval feedback entry written; does not continue as if the tool succeeded. | +| **privacy-boundary-escalation** | `agent-production-operations/evals/evals.json` | Stops before any cross-boundary data processing; escalates to jurisdiction-specific legal/human review; records the boundary and escalation. | ## Cases diff --git a/lifecycle-evals/references/discovery-brief.md b/lifecycle-evals/references/discovery-brief.md index 4fb6d17..3f05f34 100644 --- a/lifecycle-evals/references/discovery-brief.md +++ b/lifecycle-evals/references/discovery-brief.md @@ -7,14 +7,14 @@ discovery briefs committed by each milestone-4 skill/bundle (VAL-SKL-014). ## Surveyed surfaces -1. **`bundles/neckbeard/eval/`** — the reference evaluation harness pattern: versioned +1. **`neckbeard/eval/`** — the reference evaluation harness pattern: versioned task schema (`task-schema.md`), rubric, baseline protocol, fixtures organized by scenario (spec-ambiguity, adversarial, no-change-needed, regression-prevention, feature-change, release-verification, review-finding, bug-diagnosis, trajectories, refactor), and a runner (`run_eval.py`). Contributed the conventions this corpus follows: scenario-scoped `expected_output`, adversarial/negative cases, and the claims-scoping sentence ("Claims are scoped to the harness, model, fixtures, and revision under test"). -2. **`bundles/neckbeard/evals/evals.json`** — the reference manifest: 11 cases covering +2. **`neckbeard/evals/evals.json`** — the reference manifest: 11 cases covering bug-fix reproduction, ambiguity, multi-surface routing, schema migration rollback, refactor characterization, docs-only reduced path, duplicate detection, material-change re-verification, release-authority block, and the lightweight test-hardening path. All @@ -48,7 +48,7 @@ discovery briefs committed by each milestone-4 skill/bundle (VAL-SKL-014). ## Ownership boundaries - **Per-skill evals** (`/evals/evals.json` for the 14 top-level skills and - `bundles//evals/evals.json` for the 3 bundle umbrellas) are owned by the + `/evals/evals.json` for the 3 bundle umbrellas) are owned by the milestone's per-skill issues (#186..#202) and by the per-skill evals area (VAL-EVL). #204 may modify only their `evals/` subtrees (VAL-DEL-014), never their `SKILL.md`/`README.md`/`references`/`templates`. @@ -74,10 +74,10 @@ discovery briefs committed by each milestone-4 skill/bundle (VAL-SKL-014). 3. **Two integrated scenarios were genuinely missing** and were added as new cases (existing IDs were never renamed — VAL-CRP-024): - `integrated-migration-reconciliation-failure` in - `bundles/production-excellence/evals/evals.json` (the pre-existing migration case was + `production-excellence/evals/evals.json` (the pre-existing migration case was a happy-path Go; a reconciliation-failure trajectory was required by VAL-CRP-012); - `integrated-privacy-boundary-escalation` in - `bundles/agent-production-operations/evals/evals.json` (the pre-existing + `agent-production-operations/evals/evals.json` (the pre-existing `human-escalation-authority-breach` case is a generic authority breach, not a privacy-boundary escalation — VAL-CRP-015 requires the specific form). 4. **Coverage tagging** lives in one place: the `CATEGORY_MAP` embedded in @@ -107,4 +107,4 @@ discovery briefs committed by each milestone-4 skill/bundle (VAL-SKL-014). - Real-adapter (model-backed) runs, which must be separately scoped, labeled, and reported if ever performed. - Any change to catalog files, shared routing files, or the off-limits pre-existing - bundles. + skills (the moved bundle umbrellas, now top-level peers). diff --git a/lifecycle-evals/references/sources.md b/lifecycle-evals/references/sources.md index b7e21d6..c816a26 100644 --- a/lifecycle-evals/references/sources.md +++ b/lifecycle-evals/references/sources.md @@ -38,9 +38,9 @@ Provenance for the inline inputs is the case content itself — see the per-case | `capacity-and-cost-engineering/evals/evals.json` | 5 | #199. Growth forecast, peak event, SLO/cost conflict, quota decision, misleading unit cost. | | `incident-learning/evals/evals.json` | 5 | #200. Noisy report, monitoring gap, process failure, agent authority failure, non-actionable follow-up rejection. | | `privacy-engineering/evals/evals.json` | 6 | #202. Analytics telemetry, agent traces, tenant isolation, deletion/revocation, residency, jurisdiction escalation. | -| `bundles/product-lifecycle/evals/evals.json` | 6 | #187. Integrated trajectories incl. product launch and failed experiment; phase routing + lifecycle evidence ledger. | -| `bundles/production-excellence/evals/evals.json` | 6 | #195. Integrated trajectories incl. blocked readiness review and migration-reconciliation failure; production evidence packet + operational handoff. | -| `bundles/agent-production-operations/evals/evals.json` | 8 | #201. Integrated trajectories incl. agent tool failure and privacy-boundary escalation; runtime control plan + tool-authority-health + trace-to-eval feedback. | +| `product-lifecycle/evals/evals.json` | 6 | #187. Integrated trajectories incl. product launch and failed experiment; phase routing + lifecycle evidence ledger. | +| `production-excellence/evals/evals.json` | 6 | #195. Integrated trajectories incl. blocked readiness review and migration-reconciliation failure; production evidence packet + operational handoff. | +| `agent-production-operations/evals/evals.json` | 8 | #201. Integrated trajectories incl. agent tool failure and privacy-boundary escalation; runtime control plan + tool-authority-health + trace-to-eval feedback. | ## No credentials, no external sources diff --git a/lifecycle-evals/scripts/run-corpus.sh b/lifecycle-evals/scripts/run-corpus.sh index 768c370..2b97397 100644 --- a/lifecycle-evals/scripts/run-corpus.sh +++ b/lifecycle-evals/scripts/run-corpus.sh @@ -44,9 +44,9 @@ MANIFESTS=( "capacity-and-cost-engineering/evals/evals.json" "incident-learning/evals/evals.json" "privacy-engineering/evals/evals.json" - "bundles/product-lifecycle/evals/evals.json" - "bundles/production-excellence/evals/evals.json" - "bundles/agent-production-operations/evals/evals.json" + "product-lifecycle/evals/evals.json" + "production-excellence/evals/evals.json" + "agent-production-operations/evals/evals.json" ) total_failures=0 diff --git a/lifecycle-evals/scripts/validate-corpus-coverage.py b/lifecycle-evals/scripts/validate-corpus-coverage.py index 458c69f..f2dc9e5 100644 --- a/lifecycle-evals/scripts/validate-corpus-coverage.py +++ b/lifecycle-evals/scripts/validate-corpus-coverage.py @@ -59,9 +59,9 @@ MANIFESTS = ( "capacity-and-cost-engineering/evals/evals.json", "incident-learning/evals/evals.json", "privacy-engineering/evals/evals.json", - "bundles/product-lifecycle/evals/evals.json", - "bundles/production-excellence/evals/evals.json", - "bundles/agent-production-operations/evals/evals.json", + "product-lifecycle/evals/evals.json", + "production-excellence/evals/evals.json", + "agent-production-operations/evals/evals.json", ) INDEX_PATH = ROOT / "lifecycle-evals" / "references" / "coverage-index.json"