mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-15 13:36:35 +03:00
390f3e3417
* feat: run isolated paired candidate and baseline skill evaluations Build the first complete paired skill-evaluation path: stage an immutable candidate, run matched candidate and baseline trials in clean environments, execute deterministic outcome graders, and produce a case-level comparison report. - eval_runner/sandbox.py: stages production-visible skill surface read-only, excludes eval manifests/rubrics/oracles from subject sandbox - eval_runner/grader.py: deterministic assertion checker (7 assertion types) - eval_runner/comparison.py: paired comparison report generation - eval_runner/paired.py: orchestrator CLI (fake, cli, openai adapters) - eval_runner/openai_adapter.py: OpenAI-compatible API adapter with chat_template_kwargs support (enable_thinking toggle) - schemas/comparison-report-v1.schema.json: report schema - .github/workflows/skill-eval.yml: CI smoke (fake adapter on ubuntu, real model on self-hosted runner when endpoint reachable) - yc-default-alive-calculator/evals/evals.json: initial 5-case eval manifest Verified against google_gemma-4-26B-A4B-it-IQ4_XS.gguf: 5/5 candidate improvements, 0 regressions. Closes #105 * ci: make paired-eval-model job non-blocking The self-hosted runner may not always be online. Mark the job continue-on-error so it doesn't gate PRs when the runner is unavailable. * ci: isolate model evals from pull requests * fix(raleigh): test arrivals against a daily route, not weekday-only The fixture only had a WEEK (Mon-Fri) service, so test_get_arrivals_for_stop returned 0 arrivals on weekends when _today_date() fell on Saturday/Sunday. Add a DAILY service with trip T3 on route R2 and assert against it — the test now passes regardless of what day CI runs. * ci: trigger checks on amended commit --------- Co-authored-by: magnus919 <magnus919>
62 lines
3.1 KiB
JSON
62 lines
3.1 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"skill_name": "yc-default-alive-calculator",
|
|
"evals": [
|
|
{
|
|
"id": "dead-basic",
|
|
"prompt": "A startup has $10,000 monthly revenue, $15,000 monthly burn, $100,000 cash on hand, and 1% monthly growth. Using the Default Alive / Default Dead framework, determine: is this startup default alive or default dead? What is the approximate runway in months? Show your reasoning.",
|
|
"expected_output": "The startup is DEFAULT DEAD with approximately 24 months of runway.",
|
|
"assertions": [
|
|
"response_contains:DEAD",
|
|
"exit_status:completed",
|
|
"activation_evidence_contains:yc-default-alive-calculator"
|
|
],
|
|
"files": ["scripts/default-alive.py"]
|
|
},
|
|
{
|
|
"id": "alive-high-growth",
|
|
"prompt": "A startup has $80,000 monthly revenue, $90,000 monthly burn, $500,000 cash on hand, and 30% monthly growth. Using the Default Alive / Default Dead framework, is this startup default alive or default dead? Explain why.",
|
|
"expected_output": "The startup is DEFAULT ALIVE because high growth will reach breakeven before cash runs out.",
|
|
"assertions": [
|
|
"response_contains:ALIVE",
|
|
"exit_status:completed",
|
|
"activation_evidence_contains:yc-default-alive-calculator"
|
|
],
|
|
"files": ["scripts/default-alive.py"]
|
|
},
|
|
{
|
|
"id": "zero-growth-dead",
|
|
"prompt": "Using the Default Alive framework: a company has $20,000 monthly revenue, $30,000 monthly burn, $60,000 cash, and 0% monthly growth. Is it default alive or dead? How many months of runway does it have?",
|
|
"expected_output": "DEFAULT DEAD with 6 months runway (zero growth means no path to breakeven).",
|
|
"assertions": [
|
|
"response_contains:DEAD",
|
|
"exit_status:completed",
|
|
"activation_evidence_contains:yc-default-alive-calculator"
|
|
],
|
|
"files": ["scripts/default-alive.py"]
|
|
},
|
|
{
|
|
"id": "already-profitable",
|
|
"prompt": "A company has $100,000 monthly revenue, $80,000 monthly burn, $200,000 cash, and 5% monthly growth. Apply the Default Alive / Default Dead analysis. Is this company default alive?",
|
|
"expected_output": "The company is already profitable (revenue exceeds burn) so it is DEFAULT ALIVE.",
|
|
"assertions": [
|
|
"response_contains:ALIVE",
|
|
"exit_status:completed",
|
|
"activation_evidence_contains:yc-default-alive-calculator"
|
|
],
|
|
"files": ["scripts/default-alive.py"]
|
|
},
|
|
{
|
|
"id": "burn-multiple-concept",
|
|
"prompt": "Explain what burn multiple means in the context of the Default Alive framework, and calculate it for a startup with $10,000 monthly revenue and $15,000 monthly burn. Is this burn multiple efficient or inefficient?",
|
|
"expected_output": "Burn multiple is net burn divided by net new revenue. For this startup: ($15k-$10k)/$10k = 0.5x, which is efficient.",
|
|
"assertions": [
|
|
"response_contains:burn",
|
|
"exit_status:completed",
|
|
"activation_evidence_contains:yc-default-alive-calculator"
|
|
],
|
|
"files": ["scripts/default-alive.py"]
|
|
}
|
|
]
|
|
}
|