magnus919_agent-skills/product-lifecycle-learning/evals/evals.json

{
  "schema_version": 1,
  "skill_name": "product-lifecycle-learning",
  "evals": [
    {
      "id": "successful-feature-outcomes-exceed-expectations",
      "prompt": "Our new search-with-AI feature launched 90 days ago. We expected 30% of users to try it in the first month, actual adoption was 42% (95% CI: 38-46%). Expected search-to-checkout conversion from AI-assisted searches to be 2x baseline — observed was 2.3x (p<0.01). Expected support tickets about search to drop 25% — they dropped 31%. User satisfaction surveys show 4.3/5 for the feature. The feature has had zero P1 incidents and p99 latency is 180ms (target was <250ms). Engineering says the code is clean and maintainable. Conduct a lifecycle-learning outcome review and recommend a decision.",
      "expected_output": "A lifecycle-learning assessment that: (1) collects and classifies expected outcomes (30% trial, 2x conversion, -25% tickets, <250ms latency), (2) collects and classifies observed outcomes with confidence intervals where available (42% trial [CI 38-46%], 2.3x conversion, -31% tickets, 180ms p99, 4.3/5 satisfaction, zero P1 incidents), (3) identifies no significant gaps — all observed outcomes meet or exceed expectations, (4) identifies no material uncertainties (all metrics have adequate data and confidence), (5) infers that the feature is healthy and performing above expectations, (6) recommends Continue (not Improve — the feature is already exceeding expectations and does not need additional investment), and (7) produces a retained learning record capturing the reusable pattern (what drove this success). The response distinguishes expected, observed, uncertain, and inferred claims explicitly — not conflating them. It does not recommend retirement or pivot when all evidence points to success.",
      "assertions": [
        "The response classifies claims using the four epistemic categories (expected, observed, uncertain, inferred) without conflating them",
        "The response identifies that all observed outcomes meet or exceed expected outcomes with adequate confidence",
        "The response recommends Continue (not Improve, Harvest, Pivot, Pause, or Retire) with evidence-backed rationale",
        "The response does not fabricate problems or recommend unnecessary changes when the feature is healthy",
        "The response produces or references a retained learning record capturing reusable patterns from the success"
      ]
    },
    {
      "id": "feature-with-clear-non-adoption",
      "prompt": "We launched a collaborative document editing feature 6 months ago for our enterprise product. Expected outcomes: 40% of teams would create at least one collaborative doc in the first quarter, reducing email attachment volume by 30%. Observed outcomes: 8% of teams created a collaborative doc in Q1 (95% CI: 5-11%), email attachment volume decreased 3% (not statistically significant, p=0.42), and the feature has a 4.1/5 satisfaction rating from the 8% who use it. Support tickets about the feature are near zero because so few people use it. There have been no technical issues. The original assumption was that email-based collaboration was a pain point; user interviews with non-adopters reveal they do not perceive email attachments as a problem worth switching away from. Conduct a lifecycle-learning outcome review and recommend a decision.",
      "expected_output": "A lifecycle-learning assessment that: (1) classifies expected outcomes (40% team adoption, 30% email reduction) vs. observed (8% adoption [CI: 5-11%], 3% reduction [not significant]), (2) identifies a large negative gap — observed adoption is 5x below expected, (3) identifies an assumption failure: the core assumption that email collaboration is a pain point broke under evidence, (4) does NOT recommend Retirement based solely on a numeric threshold like '8% < 40% therefore retire' — instead evaluates the evidence context (users who DO use it love it, but the problem assumption was wrong), (5) recommends either Pivot (if the underlying collaboration need is real but the approach is wrong) or Harvest (keep for the 8% who value it, stop investment) or Retire (if no viable pivot exists) — and the recommendation must include human-accountability framing (the decision-maker's name/role placeholder, context considered), (6) identifies the assumption ledger update: 'email attachments are a pain point' assumption moves from assumed to contradicted, and (7) routes the learning to roadmap, analytics, adoption, experimentation, and specifications. The response does NOT apply an arbitrary threshold like 'adoption < 10% = retire' without context.",
      "assertions": [
        "The response classifies expected vs. observed outcomes with explicit confidence intervals where available",
        "The response identifies that the core assumption (email collaboration is a pain point) broke under evidence",
        "The response does NOT apply an arbitrary numeric threshold ('adoption < X% therefore retire') without context",
        "The response frames the decision as requiring human judgment — names the decision-maker role and context factors",
        "The response routes learning to at least 4 feedback destinations (roadmap, analytics, adoption, experimentation, specifications)"
      ]
    },
    {
      "id": "ambiguous-mixed-results-with-confounds",
      "prompt": "We launched a redesigned onboarding flow 90 days ago. Expected outcomes: increase new-user activation from 45% to 60%, reduce time-to-first-value from 3 days to 1 day. Observed outcomes: activation is 52% (95% CI: 47-57%) — the confidence interval spans both 'better than before' and 'not meaningfully different'. Time-to-first-value dropped to 1.8 days median. However, during the same period, we also launched a new pricing tier that lowered the entry price by 40%, and our marketing team ran a promotional campaign. We cannot separate the onboarding redesign effect from the pricing and promotion effects. User feedback on the new flow is mixed: new users find it simpler, but power users who created test accounts say it hides advanced options. Our support team reports a 15% increase in 'how do I do X' questions from existing users who were forced through the new flow. Conduct a lifecycle-learning outcome review and recommend a decision.",
      "expected_output": "A lifecycle-learning assessment that: (1) classifies expected outcomes (60% activation, 1-day TTV) vs. observed (52% activation [CI: 47-57%], 1.8-day TTV), (2) explicitly identifies the pricing and promotion confounds as making the onboarding effect uncertain — classifies this as an Uncertain claim under epistemic discipline, (3) identifies the mixed user feedback as additional uncertainty (simpler for new users, frustrating for power users), (4) identifies the support-ticket increase as a negative signal, (5) recommends Pause (not Continue, not Retire) — the results are too ambiguous to commit to a direction; more evidence is needed after isolating the confounds, (6) names specific follow-up actions to reduce uncertainty (e.g., cohort analysis separating pricing-discount users from non-discount users, A/B test with confound control), and (7) routes the learning with explicit uncertainty labels to roadmap and experimentation. The response does NOT recommend Continue based on the '52% looks better than 45%' superficial comparison without addressing confounds. The response does NOT recommend Retire based on the negative power-user signal alone without addressing the positive new-user signal.",
      "assertions": [
        "The response explicitly identifies confounds (pricing change, promotional campaign) as creating uncertainty about the onboarding effect",
        "The response classifies the attribution problem as an Uncertain claim — not dressing uncertainty as certainty",
        "The response recommends Pause (not Continue or Retire) based on the ambiguity of the evidence",
        "The response names specific follow-up actions to reduce uncertainty rather than accepting ambiguity passively",
        "The response does not cherry-pick one signal (positive or negative) while ignoring contradictory signals"
      ]
    },
    {
      "id": "feature-that-should-be-retired",
      "prompt": "We have a legacy reporting dashboard that was built 4 years ago. It has 12 daily active users out of a product with 15,000 DAU (0.08% adoption). The 12 users are all from one enterprise customer who signed a contract that mentions the dashboard. The dashboard costs $8,000/month in dedicated infrastructure and requires 2 engineering days per month for maintenance and security patches. It had a P1 incident last month that took 4 hours to resolve because the original engineers left and nobody understands the codebase. A new reporting suite launched last year that covers all the same use cases with better UX, and 11 of the 12 legacy dashboard users also use the new suite. The remaining 1 user says they need the old dashboard for a specific CSV export format that the new suite does not yet support. Our VP of Engineering wants it gone; the account manager for the enterprise customer wants a 6-month migration window. Conduct a lifecycle-learning outcome review and recommend a decision.",
      "expected_output": "A lifecycle-learning assessment that: (1) identifies the stark adoption gap (0.08% DAU, 12 users out of 15,000), (2) identifies the cost and risk profile ($8K/month + 2 engineering days/month + unmaintainable code + recent P1), (3) identifies that a replacement exists and 11/12 users have already migrated, (4) identifies the remaining blocker (1 user needs CSV export format), (5) recommends Retire — but with a CONTEXTUAL, EVIDENCE-BACKED rationale, not an arbitrary threshold: the combination of negligible adoption, high cost, operational risk, existence of a replacement, and near-complete migration makes retirement the right call, (6) names the accountable human decision-maker (role: VP of Engineering or product lead, not an automated threshold), (7) defines the sunset plan: deprecation announcement to the enterprise customer, 6-month migration window per account manager's request, CSV export format added to the new suite as the migration path, and a clear timeline with milestones (announcement, EOL, EOS, removal), (8) includes customer treatment: support commitments during sunset, data export guarantee, grace period, and (9) captures retained learning: what this feature's lifecycle taught about building features for single customers, about maintaining legacy code without succession planning, and about ensuring replacements cover all use cases before deprecation. The response does NOT say '0.08% < some threshold therefore auto-retire.' It builds the case from multiple dimensions of evidence and preserves human accountability.",
      "assertions": [
        "The response recommends Retire with a multi-dimensional evidence-backed rationale (adoption, cost, risk, replacement, migration status) — not a single numeric threshold",
        "The response names an accountable human decision-maker role (not 'the algorithm' or 'the threshold')",
        "The response defines a sunset plan with deprecation communication, migration path, customer treatment, and timeline milestones",
        "The response addresses the remaining blocker (1 user's CSV export need) with a migration solution, not by ignoring it",
        "The response captures retained learning (patterns for single-customer features, legacy maintenance, replacement completeness) in a durable record"
      ]
    },
    {
      "id": "retirement-requiring-migration-and-customer-communication",
      "prompt": "Our B2B SaaS product is retiring the legacy API (v1) that 340 enterprise customers still use, representing $2.1M in annual contract value. The v2 API has been available for 18 months and covers all v1 functionality plus additional capabilities. 72% of customers have already migrated. The remaining 340 customers cite: lack of engineering bandwidth (60%), satisfaction with v1 as-is (25%), and missing v1-specific webhook format in v2 (15%). We need to retire v1 because it runs on end-of-life infrastructure that will lose vendor support in 8 months, and maintaining it costs $45K/month in dedicated ops. The CEO has mandated retirement before the infrastructure EOL. Design the complete retirement lifecycle: deprecation communication, migration path, customer treatment plan, and internal cleanup. Pay special attention to the 15% of customers who need the webhook format and the risk of churning $2.1M in revenue.",
      "expected_output": "A complete retirement lifecycle plan that covers ALL FIVE phases: (1) Deprecation Communication — announcement with rationale (infrastructure EOL, cost, v2 coverage), timeline (8-month window tied to EOL date), affected-customer segmentation (the 340 customers broken down by their stated reason for not migrating), communication channels (account managers, email, in-product notice, documentation); (2) Migration Path — step-by-step v1-to-v2 migration guide, dedicated support channel, migration tooling if available, and a SPECIFIC solution for the 15% who need the webhook format (either add webhook format to v2, provide a compatibility shim, or offer an alternative); (3) Customer Treatment — support SLA preserved during sunset, 8-month grace period tied to the hard infrastructure EOL deadline, data export guarantee, escalation path for customers who need extensions, and proactive account-manager outreach coordinated with conditional-customer-success for the $2.1M at-risk accounts; (4) Internal Cleanup — v1 API endpoint removal, feature flag removal, code archival, documentation archival and cross-reference updates, monitoring retirement, infrastructure decommissioning after the removal date; (5) Learning Closure — retained learning record capturing: the 18-month coexistence window (was it long enough?), the webhook-format gap (why was this not identified earlier?), the customer communication strategy effectiveness, and reusable patterns for future API version retirements. The response explicitly distinguishes between expected (smooth migration within 18 months), observed (72% migrated, 28% haven't for specific reasons), uncertain (will the remaining customers churn?), and inferred (the webhook gap is the binding constraint for the last 15%). The response routes the retirement communication plan to conditional-customer-success for account-level execution. Claims are scoped to the harness, model, fixtures, and revision under test.",
      "assertions": [
        "The response covers all five retirement phases: deprecation communication, migration path, customer treatment, internal cleanup, and learning closure",
        "The response addresses the webhook-format gap for the 15% of customers with a specific solution, not a generic 'they should migrate' statement",
        "The response includes a customer treatment plan with support commitments, grace period, data export, and escalation path",
        "The response identifies the revenue-at-risk ($2.1M) and coordinates with conditional-customer-success for account-level execution",
        "The response captures retained learning with reusable patterns for future API version retirements"
      ]
    },
    {
      "id": "anti-pattern-arbitrary-threshold-rejection",
      "prompt": "I want you to create a dashboard that automatically retires any feature that drops below 100 daily active users or has an NPS below 30. Can you set up the automation rules?",
      "expected_output": "The response declines to apply product-lifecycle-learning for automatic retirement based on arbitrary thresholds. It explains that retirement decisions require human judgment and context — a feature with 95 DAU might be critical to a $5M enterprise account, and a feature with NPS 25 might be a utility feature users do not love but depend on. It states that this skill does not prescribe universal retirement thresholds and that applying numeric rules without context violates the skill's retirement-decision discipline. The response may suggest using the thresholds as monitoring signals that trigger a human review (not an automatic retirement), and routes the human review to the outcome review and retirement decision templates.",
      "assertions": [
        "The response declines to automate retirement decisions based on numeric thresholds",
        "The response explains that retirement requires human judgment and context, not a formula",
        "The response provides counterexamples showing why thresholds without context are misleading",
        "The response may suggest using thresholds as monitoring signals for human review, but not as automatic triggers",
        "The response does not produce an automated retirement rule or dashboard configuration"
      ]
    },
    {
      "id": "anti-pattern-incident-postmortem-routing",
      "prompt": "Our payment service had a 4-hour outage last week that affected 12,000 transactions. We need to do a postmortem and figure out what lifecycle learning we can extract from this incident. Can you run the incident postmortem using the lifecycle-learning skill?",
      "expected_output": "The response declines to run an incident postmortem with product-lifecycle-learning. It explains that incident postmortems, root-cause analysis, and operational incident review belong to incident-learning (prose reference, not yet landed) and site-reliability-engineering. It states that lifecycle-learning consumes incident signals as input (e.g., 'the payment service had a reliability incident that affected feature health') but does not produce postmortems. The response may suggest that after the incident postmortem is completed by the appropriate skill, lifecycle-learning can consume the output as part of a feature health assessment if the payment service is being evaluated for a lifecycle decision.",
      "assertions": [
        "The response declines to run an incident postmortem with product-lifecycle-learning",
        "The response routes incident postmortems to incident-learning and site-reliability-engineering",
        "The response explains that lifecycle-learning consumes incident signals as input but does not produce postmortems",
        "The response describes how lifecycle-learning could use postmortem output as input to a feature health assessment"
      ]
    }
  ]
}