mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-11 19:47:12 +03:00
fix(evals): re-point lifecycle corpus tooling at flat layout
Update the three hardcoded bundle manifest paths in run-corpus.sh and validate-corpus-coverage.py from bundles/<name>/evals/evals.json to <name>/evals/evals.json, refresh the coverage-index.json via --write-index, and update the corpus prose (README, coverage-matrix, sources, discovery-brief) to drop the bundles/ prefix. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
Magnus Hedemark
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
parent
3480b7ff5a
commit
b7aa174b84
@@ -81,7 +81,7 @@ After changing case content or tags, refresh the committed index with
|
||||
| `run-artifacts/manifests/` | One committed snapshot of fake-adapter run output (per-trial manifests), refreshed at merge time |
|
||||
|
||||
The 17 eval manifests themselves live in their owning skills:
|
||||
`<skill>/evals/evals.json` for the 14 top-level skills and `bundles/<bundle>/evals/evals.json`
|
||||
`<skill>/evals/evals.json` for the 14 top-level skills and `<bundle>/evals/evals.json`
|
||||
for the 3 bundle umbrellas.
|
||||
|
||||
## Per-case status semantics
|
||||
@@ -126,8 +126,8 @@ block / decline) assert exactly that negative outcome.
|
||||
|
||||
## Claims policy (non-claim statement)
|
||||
|
||||
This corpus is a **small, fixed output-quality corpus** (14 skills × ≥5 cases + 3 bundles
|
||||
of integrated cases). **Fake-adapter runs prove pipeline reproducibility and case
|
||||
This corpus is a **small, fixed output-quality corpus** (14 skills × ≥5 cases + 3 bundle
|
||||
umbrellas of integrated cases). **Fake-adapter runs prove pipeline reproducibility and case
|
||||
executability only.** They produce **no pass-rate, accuracy, or capability claims about any
|
||||
model** — no "10x", no "best", no universal performance claims. Any future **real-adapter**
|
||||
run (a model-backed harness) must be separately scoped, labeled (adapter + model + model
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
"manifests": [
|
||||
{
|
||||
"skill": "agent-production-operations",
|
||||
"manifest": "bundles/agent-production-operations/evals/evals.json",
|
||||
"manifest": "agent-production-operations/evals/evals.json",
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "cost-budget-breach-disablement",
|
||||
@@ -81,103 +81,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"skill": "product-lifecycle",
|
||||
"manifest": "bundles/product-lifecycle/evals/evals.json",
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "ambiguous-stakeholder-request",
|
||||
"behavioral_categories": [
|
||||
"ambiguity"
|
||||
],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "cross-phase-evidence-handoff",
|
||||
"behavioral_categories": [],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "failed-experiment-stop-path",
|
||||
"behavioral_categories": [
|
||||
"failure",
|
||||
"stop-retire"
|
||||
],
|
||||
"integrated_scenarios": [
|
||||
"failed-experiment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "justified-retirement-decision",
|
||||
"behavioral_categories": [
|
||||
"stop-retire"
|
||||
],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "new-product-complete-lifecycle",
|
||||
"behavioral_categories": [],
|
||||
"integrated_scenarios": [
|
||||
"product-launch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "non-adoption-outcome",
|
||||
"behavioral_categories": [],
|
||||
"integrated_scenarios": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"skill": "production-excellence",
|
||||
"manifest": "bundles/production-excellence/evals/evals.json",
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "blocked-launch-untested-rollback",
|
||||
"behavioral_categories": [
|
||||
"failure",
|
||||
"unsafe-authority"
|
||||
],
|
||||
"integrated_scenarios": [
|
||||
"blocked-readiness-review"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "cost-slo-conflict",
|
||||
"behavioral_categories": [
|
||||
"conflicting-evidence"
|
||||
],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "data-migration-routes-to-migration-engineering",
|
||||
"behavioral_categories": [],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "dependency-outage-routes-to-resilience",
|
||||
"behavioral_categories": [
|
||||
"conflicting-evidence",
|
||||
"failure"
|
||||
],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "integrated-migration-reconciliation-failure",
|
||||
"behavioral_categories": [
|
||||
"failure"
|
||||
],
|
||||
"integrated_scenarios": [
|
||||
"migration-reconciliation-failure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "normal-release-safe-launch",
|
||||
"behavioral_categories": [],
|
||||
"integrated_scenarios": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"skill": "capacity-and-cost-engineering",
|
||||
"manifest": "capacity-and-cost-engineering/evals/evals.json",
|
||||
@@ -576,6 +479,53 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"skill": "product-lifecycle",
|
||||
"manifest": "product-lifecycle/evals/evals.json",
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "ambiguous-stakeholder-request",
|
||||
"behavioral_categories": [
|
||||
"ambiguity"
|
||||
],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "cross-phase-evidence-handoff",
|
||||
"behavioral_categories": [],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "failed-experiment-stop-path",
|
||||
"behavioral_categories": [
|
||||
"failure",
|
||||
"stop-retire"
|
||||
],
|
||||
"integrated_scenarios": [
|
||||
"failed-experiment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "justified-retirement-decision",
|
||||
"behavioral_categories": [
|
||||
"stop-retire"
|
||||
],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "new-product-complete-lifecycle",
|
||||
"behavioral_categories": [],
|
||||
"integrated_scenarios": [
|
||||
"product-launch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "non-adoption-outcome",
|
||||
"behavioral_categories": [],
|
||||
"integrated_scenarios": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"skill": "product-operations-and-governance",
|
||||
"manifest": "product-operations-and-governance/evals/evals.json",
|
||||
@@ -656,6 +606,56 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"skill": "production-excellence",
|
||||
"manifest": "production-excellence/evals/evals.json",
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "blocked-launch-untested-rollback",
|
||||
"behavioral_categories": [
|
||||
"failure",
|
||||
"unsafe-authority"
|
||||
],
|
||||
"integrated_scenarios": [
|
||||
"blocked-readiness-review"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "cost-slo-conflict",
|
||||
"behavioral_categories": [
|
||||
"conflicting-evidence"
|
||||
],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "data-migration-routes-to-migration-engineering",
|
||||
"behavioral_categories": [],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "dependency-outage-routes-to-resilience",
|
||||
"behavioral_categories": [
|
||||
"conflicting-evidence",
|
||||
"failure"
|
||||
],
|
||||
"integrated_scenarios": []
|
||||
},
|
||||
{
|
||||
"case_id": "integrated-migration-reconciliation-failure",
|
||||
"behavioral_categories": [
|
||||
"failure"
|
||||
],
|
||||
"integrated_scenarios": [
|
||||
"migration-reconciliation-failure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "normal-release-safe-launch",
|
||||
"behavioral_categories": [],
|
||||
"integrated_scenarios": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"skill": "production-readiness",
|
||||
"manifest": "production-readiness/evals/evals.json",
|
||||
|
||||
@@ -20,12 +20,12 @@ each have at least one case (see the coverage summary at the end).
|
||||
|
||||
| Scenario | Home manifest | Required trajectory |
|
||||
|---|---|---|
|
||||
| **product-launch** | `bundles/product-lifecycle/evals/evals.json` | Full lifecycle routing with phase-entry evidence, handoff artifacts, launch decision, evidence-ledger entry. |
|
||||
| **failed-experiment** | `bundles/product-lifecycle/evals/evals.json` | Negative outcome recorded with evidence; stop/no-ship decision; retained learning routed into lifecycle review. |
|
||||
| **migration-reconciliation-failure** | `bundles/production-excellence/evals/evals.json` | Reconciliation detects a mismatch; failure recorded; no-go/rollback/roll-forward decision with evidence and owner; does NOT proceed to launch. |
|
||||
| **blocked-readiness-review** | `bundles/production-excellence/evals/evals.json` | Blocked/no-go outcome; missing evidence named; accountable owner assigned; exception requires human approval. |
|
||||
| **agent-tool-failure** | `bundles/agent-production-operations/evals/evals.json` | Tool outage recorded; fallback/escalation/disablement per runtime control plan; trace-to-eval feedback entry written; does not continue as if the tool succeeded. |
|
||||
| **privacy-boundary-escalation** | `bundles/agent-production-operations/evals/evals.json` | Stops before any cross-boundary data processing; escalates to jurisdiction-specific legal/human review; records the boundary and escalation. |
|
||||
| **product-launch** | `product-lifecycle/evals/evals.json` | Full lifecycle routing with phase-entry evidence, handoff artifacts, launch decision, evidence-ledger entry. |
|
||||
| **failed-experiment** | `product-lifecycle/evals/evals.json` | Negative outcome recorded with evidence; stop/no-ship decision; retained learning routed into lifecycle review. |
|
||||
| **migration-reconciliation-failure** | `production-excellence/evals/evals.json` | Reconciliation detects a mismatch; failure recorded; no-go/rollback/roll-forward decision with evidence and owner; does NOT proceed to launch. |
|
||||
| **blocked-readiness-review** | `production-excellence/evals/evals.json` | Blocked/no-go outcome; missing evidence named; accountable owner assigned; exception requires human approval. |
|
||||
| **agent-tool-failure** | `agent-production-operations/evals/evals.json` | Tool outage recorded; fallback/escalation/disablement per runtime control plan; trace-to-eval feedback entry written; does not continue as if the tool succeeded. |
|
||||
| **privacy-boundary-escalation** | `agent-production-operations/evals/evals.json` | Stops before any cross-boundary data processing; escalates to jurisdiction-specific legal/human review; records the boundary and escalation. |
|
||||
|
||||
## Cases
|
||||
|
||||
|
||||
@@ -7,14 +7,14 @@ discovery briefs committed by each milestone-4 skill/bundle (VAL-SKL-014).
|
||||
|
||||
## Surveyed surfaces
|
||||
|
||||
1. **`bundles/neckbeard/eval/`** — the reference evaluation harness pattern: versioned
|
||||
1. **`neckbeard/eval/`** — the reference evaluation harness pattern: versioned
|
||||
task schema (`task-schema.md`), rubric, baseline protocol, fixtures organized by scenario
|
||||
(spec-ambiguity, adversarial, no-change-needed, regression-prevention, feature-change,
|
||||
release-verification, review-finding, bug-diagnosis, trajectories, refactor), and a
|
||||
runner (`run_eval.py`). Contributed the conventions this corpus follows: scenario-scoped
|
||||
`expected_output`, adversarial/negative cases, and the claims-scoping sentence
|
||||
("Claims are scoped to the harness, model, fixtures, and revision under test").
|
||||
2. **`bundles/neckbeard/evals/evals.json`** — the reference manifest: 11 cases covering
|
||||
2. **`neckbeard/evals/evals.json`** — the reference manifest: 11 cases covering
|
||||
bug-fix reproduction, ambiguity, multi-surface routing, schema migration rollback,
|
||||
refactor characterization, docs-only reduced path, duplicate detection, material-change
|
||||
re-verification, release-authority block, and the lightweight test-hardening path. All
|
||||
@@ -48,7 +48,7 @@ discovery briefs committed by each milestone-4 skill/bundle (VAL-SKL-014).
|
||||
## Ownership boundaries
|
||||
|
||||
- **Per-skill evals** (`<skill>/evals/evals.json` for the 14 top-level skills and
|
||||
`bundles/<bundle>/evals/evals.json` for the 3 bundle umbrellas) are owned by the
|
||||
`<bundle>/evals/evals.json` for the 3 bundle umbrellas) are owned by the
|
||||
milestone's per-skill issues (#186..#202) and by the per-skill evals area (VAL-EVL).
|
||||
#204 may modify only their `evals/` subtrees (VAL-DEL-014), never their
|
||||
`SKILL.md`/`README.md`/`references`/`templates`.
|
||||
@@ -74,10 +74,10 @@ discovery briefs committed by each milestone-4 skill/bundle (VAL-SKL-014).
|
||||
3. **Two integrated scenarios were genuinely missing** and were added as new cases
|
||||
(existing IDs were never renamed — VAL-CRP-024):
|
||||
- `integrated-migration-reconciliation-failure` in
|
||||
`bundles/production-excellence/evals/evals.json` (the pre-existing migration case was
|
||||
`production-excellence/evals/evals.json` (the pre-existing migration case was
|
||||
a happy-path Go; a reconciliation-failure trajectory was required by VAL-CRP-012);
|
||||
- `integrated-privacy-boundary-escalation` in
|
||||
`bundles/agent-production-operations/evals/evals.json` (the pre-existing
|
||||
`agent-production-operations/evals/evals.json` (the pre-existing
|
||||
`human-escalation-authority-breach` case is a generic authority breach, not a
|
||||
privacy-boundary escalation — VAL-CRP-015 requires the specific form).
|
||||
4. **Coverage tagging** lives in one place: the `CATEGORY_MAP` embedded in
|
||||
@@ -107,4 +107,4 @@ discovery briefs committed by each milestone-4 skill/bundle (VAL-SKL-014).
|
||||
- Real-adapter (model-backed) runs, which must be separately scoped, labeled, and reported
|
||||
if ever performed.
|
||||
- Any change to catalog files, shared routing files, or the off-limits pre-existing
|
||||
bundles.
|
||||
skills (the moved bundle umbrellas, now top-level peers).
|
||||
|
||||
@@ -38,9 +38,9 @@ Provenance for the inline inputs is the case content itself — see the per-case
|
||||
| `capacity-and-cost-engineering/evals/evals.json` | 5 | #199. Growth forecast, peak event, SLO/cost conflict, quota decision, misleading unit cost. |
|
||||
| `incident-learning/evals/evals.json` | 5 | #200. Noisy report, monitoring gap, process failure, agent authority failure, non-actionable follow-up rejection. |
|
||||
| `privacy-engineering/evals/evals.json` | 6 | #202. Analytics telemetry, agent traces, tenant isolation, deletion/revocation, residency, jurisdiction escalation. |
|
||||
| `bundles/product-lifecycle/evals/evals.json` | 6 | #187. Integrated trajectories incl. product launch and failed experiment; phase routing + lifecycle evidence ledger. |
|
||||
| `bundles/production-excellence/evals/evals.json` | 6 | #195. Integrated trajectories incl. blocked readiness review and migration-reconciliation failure; production evidence packet + operational handoff. |
|
||||
| `bundles/agent-production-operations/evals/evals.json` | 8 | #201. Integrated trajectories incl. agent tool failure and privacy-boundary escalation; runtime control plan + tool-authority-health + trace-to-eval feedback. |
|
||||
| `product-lifecycle/evals/evals.json` | 6 | #187. Integrated trajectories incl. product launch and failed experiment; phase routing + lifecycle evidence ledger. |
|
||||
| `production-excellence/evals/evals.json` | 6 | #195. Integrated trajectories incl. blocked readiness review and migration-reconciliation failure; production evidence packet + operational handoff. |
|
||||
| `agent-production-operations/evals/evals.json` | 8 | #201. Integrated trajectories incl. agent tool failure and privacy-boundary escalation; runtime control plan + tool-authority-health + trace-to-eval feedback. |
|
||||
|
||||
## No credentials, no external sources
|
||||
|
||||
|
||||
@@ -44,9 +44,9 @@ MANIFESTS=(
|
||||
"capacity-and-cost-engineering/evals/evals.json"
|
||||
"incident-learning/evals/evals.json"
|
||||
"privacy-engineering/evals/evals.json"
|
||||
"bundles/product-lifecycle/evals/evals.json"
|
||||
"bundles/production-excellence/evals/evals.json"
|
||||
"bundles/agent-production-operations/evals/evals.json"
|
||||
"product-lifecycle/evals/evals.json"
|
||||
"production-excellence/evals/evals.json"
|
||||
"agent-production-operations/evals/evals.json"
|
||||
)
|
||||
|
||||
total_failures=0
|
||||
|
||||
@@ -59,9 +59,9 @@ MANIFESTS = (
|
||||
"capacity-and-cost-engineering/evals/evals.json",
|
||||
"incident-learning/evals/evals.json",
|
||||
"privacy-engineering/evals/evals.json",
|
||||
"bundles/product-lifecycle/evals/evals.json",
|
||||
"bundles/production-excellence/evals/evals.json",
|
||||
"bundles/agent-production-operations/evals/evals.json",
|
||||
"product-lifecycle/evals/evals.json",
|
||||
"production-excellence/evals/evals.json",
|
||||
"agent-production-operations/evals/evals.json",
|
||||
)
|
||||
|
||||
INDEX_PATH = ROOT / "lifecycle-evals" / "references" / "coverage-index.json"
|
||||
|
||||
Reference in New Issue
Block a user