{
  "schema_version": 1,
  "skill_name": "training-data-annotation",
  "evals": [
    {"id":"label-schema-boundaries","prompt":"Design labels for classifying support messages where some requests are ambiguous and some contain multiple intents.","expected_output":"Defines the unit, mutually clear labels, multi-intent handling, ambiguity abstention, boundary examples, evidence rules, and versioned guide ownership.","assertions":["Defines the annotation unit and label meanings","Separates ambiguity from a negative label","Specifies multi-intent handling","Requires boundary examples and guide versioning"]},
    {"id":"active-selection-heldout","prompt":"We want active learning to select the most uncertain items for labeling. How do we avoid losing population coverage?","expected_output":"Combines uncertainty with diversity, novelty, and risk strata, while maintaining an independently sampled representative held-out audit set and recording selector details.","assertions":["Does not rely on uncertainty alone","Maintains a representative held-out audit","Records selector, model version, scores, and exclusions","Compares targeted batches with the audit population"]},
    {"id":"provenance-rights-release","prompt":"A vendor supplied chat logs for labeling. Create the release checklist for training use.","expected_output":"Requires purpose-appropriate license or consent, source and snapshot identity, transformations, sensitivity/access controls, retention and deletion propagation, immutable revision, and a downstream handoff record.","assertions":["Verifies rights or consent for the named purpose","Records source, snapshot, transformations, and immutable revision","Defines sensitive-data access and retention/deletion","Requires deletion propagation to derived labels"]},
    {"id":"blinded-calibration-disagreement","prompt":"Three annotators disagree on images of people carrying bicycles. One minority label is plausible. What should the team do?","expected_output":"Uses blinded calibration and domain-appropriate agreement analysis, classifies ambiguity versus mistake, preserves original labels, and adjudicates with an explicit rule and authority rather than majority vote by default.","assertions":["Uses blinded calibration","Treats meaningful ambiguity separately from annotator error","Preserves original labels and disagreement","Records adjudication rule, authority, and evidence"]},
    {"id":"preannotation-anchoring","prompt":"Annotators accept 98% of model-suggested labels. Can we conclude the model improved labeling quality?","expected_output":"Rejects that conclusion without a blinded control or independent review, tests anchoring and error patterns, and reports quality and workload separately from acceptance rate.","assertions":["Does not equate acceptance rate with quality","Requires a blinded control or independent labels","Tests anchoring and error patterns","Reports quality separately from throughput or acceptance"]},
    {"id":"cost-quality-stopping","prompt":"We have budget for 10,000 more labels but recent batches show little downstream improvement. Decide whether to continue.","expected_output":"Uses a predeclared marginal-value rule incorporating slice coverage, blind-recheck quality, expected downstream gain, cost, and time; stops when marginal value is below cost and names reopen triggers.","assertions":["Uses marginal downstream value rather than a fixed quota","Checks slice coverage and blind-recheck stability","Includes cost and time in the decision","Defines reopen triggers for distribution or guide changes"]}
  ]
}