magnus919_agent-skills/qa-methodology/evals/evals.json

{"schema_version": 1, "skill_name": "qa-methodology", "evals": [{"id": "risk-prioritization-workshop", "prompt": "We have a release coming up with limited QA capacity and need to decide which areas deserve the most testing effort based on risk. Can you help me prioritize?", "expected_output": "A risk assessment using the P×I formula (Probability 1-5 × Impact 1-5) on a 5×5 matrix, mapping each risk item to priority tiers P0-P3, with a risk register containing scored items, owners, mitigations, and reassessment triggers. Test allocation is proportional to risk scores.", "assertions": ["The response applies the P×I scoring formula with probability and impact rated on a 1-5 scale", "The response maps risk scores to priority tiers P0 through P3 using the 5×5 risk matrix", "The response includes a risk register structure with scored items, owners, and mitigations", "The response defines reassessment triggers for updating risk scores over time", "The response recommends test allocation proportional to each risk tier score"]}, {"id": "exploratory-charter-design", "prompt": "I need to design an exploratory testing session for our new payment integration but I want it structured and accountable, not just random clicking around.", "expected_output": "A structured SBTM charter in the format 'Explore <target> with <resources> to discover <information>', with a 60-120 minute timebox, T/B/B metrics tracking, and a debrief plan. Heuristics such as SFDIPOT or HICCUPPS are referenced for coverage guidance.", "assertions": ["The response produces a charter in the 'Explore target with resources to discover information' format", "The response includes a timebox duration of 60 to 120 minutes for the session", "The response specifies T/B/B metrics for tracking test time allocation during the session", "The response references SBTM heuristics such as SFDIPOT or HICCUPPS for coverage guidance", "The response includes a debrief structure for reviewing session findings and follow-up actions"]}, {"id": "sdd-gate-ac-testability", "prompt": "An agent just wrote code from a spec and claims it passes all acceptance criteria, but I need an independent QA review before it merges. How should I verify?", "expected_output": "An independent verification plan executed in a separate agent session, where the implementing agent does not self-verify. The plan maps each acceptance criterion to a verification method with observable outcomes, flags untestable ACs, and attaches evidence per gate.", "assertions": ["The response requires independent verification in a separate session, not by the implementing agent", "The response maps each acceptance criterion to a specific verification method with observable outcomes", "The response flags untestable acceptance criteria that lack observable outcomes", "The response requires evidence to be attached for each gate verdict", "The response warns against self-verification by the implementing agent due to overfitting risk"]}, {"id": "agentic-eval-dataset-design", "prompt": "We are building an evaluation suite for our customer support agent and I need help designing the dataset with proper class balance and judge calibration.", "expected_output": "An eval dataset design with class balance (positive, negative, adversarial, boundary cases), N=5-10 trials per task with majority-vote aggregation, a pinned judge contract tuple, pass@k or pass^k metric selection based on product requirements, and a no-retry-until-green policy.", "assertions": ["The response specifies class balance with positive, negative, adversarial, and boundary case proportions", "The response recommends N=5-10 trials per task with majority-vote or weighted aggregation", "The response requires a pinned judge contract tuple of model ID, rubric version, and prompt hash", "The response distinguishes pass@k from pass^k and selects based on product retry requirements", "The response enforces a no-retry-until-green policy for handling flaky evaluation results"]}, {"id": "sdet-career-scope-mapping", "prompt": "I am a senior QA engineer wanting to understand what scope and evidence I need to demonstrate for a staff-level SDET promotion packet.", "expected_output": "A scope-progression analysis mapping Senior to Staff transition, identifying product-scope evidence needed (multi-team standards, shared infrastructure, guilds). The response references the gTAA architecture for SDET competency and staff-level archetypes (Architect, Solver, Team Lead, Right Hand).", "assertions": ["The response maps the Senior to Staff transition using scope progression from project to product level", "The response identifies product-scope evidence such as multi-team standards and shared infrastructure", "The response references gTAA layered architecture as part of SDET competency expectations", "The response names staff-level archetypes such as Architect, Solver, Team Lead, or Right Hand", "The response distinguishes promotion evidence from current-level excellence at the next scope tier"]}, {"id": "test-design-technique-selection", "prompt": "I have a form with twelve optional fields and three numeric range inputs and I need to figure out which test design techniques will give me the best coverage efficiently.", "expected_output": "A technique selection that applies pairwise testing (PICT) for the multi-field form interactions, boundary value analysis (BVA) with 3-value approach for the numeric ranges, and equivalence partitioning for categorical inputs. The response includes a when-to-use-which rationale.", "assertions": ["The response recommends pairwise testing with PICT for the form field interactions", "The response applies boundary value analysis with the 3-value approach for numeric range inputs", "The response uses equivalence partitioning to reduce test count for categorical inputs", "The response provides a when-to-use-which rationale for each selected technique", "The response avoids exhaustive combinatorial enumeration in favor of pairwise reduction"]}, {"id": "anti-trigger-production-debugging", "prompt": "Our production checkout service is throwing intermittent 500 errors and I need to do root-cause analysis to figure out what is causing the failures.", "expected_output": "The agent declines to apply qa-methodology for this request, recognizing that root-cause debugging of production incidents falls outside its scope. It routes the user to the systematic-debugging skill for fault localization and incident analysis.", "assertions": ["The response declines qa-methodology as the appropriate skill for root-cause debugging of production incidents", "The response names systematic-debugging as the correct sibling skill for this request", "The response does not attempt to apply QA test strategy or risk-based testing methodology to the debugging task", "The response explains that production incident root-cause analysis is outside the qa-methodology negative boundary"]}, {"id": "mutation-useful-survivor-hardening", "prompt": "A changed conditional has a surviving operator mutant. Show how to harden the tests without running an unlimited mutation campaign.", "expected_output": "The response selects a diff-aware or justified risk-bounded scope, proposes a behavior-level test, and requires independent baseline-versus-candidate-versus-exact-mutant verification with reproducible evidence.", "assertions": ["The response chooses changed lines/files or justifies a broader risk slice", "The response treats the survivor as a triage input and proposes a behavior-level oracle", "The response requires independent reruns of baseline, candidate test, and exact mutant", "The response records tool/version, budget, timeout, seed, exclusions, command, status, and raw evidence"]}, {"id": "mutation-false-confidence-rejected-test", "prompt": "Mutation analysis reports a high score, but one mutant is equivalent and the proposed test only asserts an internal helper call. Decide whether this is success.", "expected_output": "The response rejects automatic success, classifies equivalent-mutant and implementation-coupling uncertainty, and asks for a better behavioral oracle or fresh human verification.", "assertions": ["The response does not treat a high score as proof of quality", "The response identifies equivalent-mutant and denominator uncertainty", "The response rejects tautological or implementation-coupled tests", "The response requires a behavioral oracle and independent review"]}, {"id": "mutation-incomplete-unreliable-run", "prompt": "A bounded mutation run has no-coverage mutants, timeouts, a flaky test, and an infrastructure error. Can we report a clean mutation result?", "expected_output": "The response preserves unreliable outcomes outside a silently inflated denominator, refuses a clean verdict, and specifies a bounded rerun/reproduction path.", "assertions": ["The response separately classifies no coverage, timeout, flaky, and infrastructure/tooling failure", "The response defines the denominator and accounts for unknown or incomplete mutants explicitly", "The response refuses a clean pass or universal threshold", "The response gives a bounded reproduction command with tool/version and environment"]}]}