magnus919_agent-skills/data-cleaning/evals/evals.json

{
  "schema_version": 1,
  "skill_name": "data-cleaning",
  "evals": [
    {"id":"messy-csv-audit","case_set":"dev","prompt":"I have a CSV with blank strings, -999 sentinels, duplicate customer IDs, mixed date formats, and an amount column that may use European decimals. Tell me what to do before cleaning it.","expected_output":"An evidence-first plan that profiles first, distinguishes missing states, confirms locale and units, checks key grain and duplicate policy, preserves raw data, and quarantines ambiguous rows.","assertions":["Profiles structure, missingness/sentinels, key uniqueness, date parsing, and numeric locale before mutation.","Requires explicit locale confirmation and does not silently coerce malformed values to null.","Preserves raw input and records a quarantine or reject path for ambiguous rows.","Defines intended row grain and a duplicate survivorship or escalation rule."]},
    {"id":"missingness-decision","case_set":"dev","prompt":"Thirty percent of income values are missing. Fill them with the median so the model can run.","expected_output":"A refusal to apply median imputation blindly, with missingness diagnosis, leakage-safe fitting boundary, indicator/provenance, sensitivity analysis, and escalation if consequential.","assertions":["Explains why missingness rate alone does not justify median imputation.","Requires diagnosing missingness reasons and preserving an imputation indicator.","Requires fitting only on the permitted training/reference partition.","Offers quarantine, domain review, or sensitivity analysis."]},
    {"id":"join-integrity","case_set":"dev","prompt":"Join orders to customers and clean up whatever duplicate rows appear afterward.","expected_output":"A pre-join cardinality audit checking uniqueness, expected join type, unmatched keys, and row multiplication, refusing deletion without a business rule.","assertions":["Checks key uniqueness and expected cardinality before the join.","Measures unmatched keys and row multiplication after the join.","Refuses to delete multiplied rows without identifying cause and survivorship rule.","Reconciles counts or totals and records the join contract."]},
    {"id":"entity-resolution-review","case_set":"regression","prompt":"Use fuzzy matching to merge two customer exports and automatically pick the highest score for every pair.","expected_output":"A cautious entity-resolution plan using blocking, candidate scores, thresholds, ambiguity review, original-value preservation, and an auditable decision log.","assertions":["Treats fuzzy matching as candidate generation rather than truth.","Requires blocking and an explicit threshold/ambiguity policy.","Preserves original records and match evidence, confidence, and decisions.","Includes quarantine or manual review for low-confidence/conflicting matches."]},
    {"id":"tool-selection","case_set":"dev","prompt":"Which tools should I use for a 20 GB warehouse table, a messy CSV edited by nontechnical staff, and a Python DataFrame pipeline with a schema contract?","expected_output":"A context-sensitive comparison selecting warehouse/dbt or scalable validation for the table, OpenRefine for interactive review, and Pandera plus a transformation library for the DataFrame, with caveats.","assertions":["Separates transformation tools from validation/contract tools.","Recommends warehouse-native or scalable validation for 20 GB rather than assuming in-memory pandas.","Recognizes OpenRefine's human-review strength.","Recommends a DataFrame schema validator and says convenience cleaning is not a quality contract."]},
    {"id":"safe-completion","case_set":"regression","prompt":"Clean this data and overwrite the original file. I only need the final CSV, not logs or a report.","expected_output":"A safe redirect keeping raw input intact, writing a new output, and requiring minimum provenance, decisions, validation evidence, and rejected-row accounting.","assertions":["Does not overwrite raw input by default.","Requires source identity, decisions, validation results, and output provenance.","Accounts for rejected/quarantined records rather than silently dropping them.","Defines completion beyond merely producing a CSV."]},
    {"id":"unicode-and-identifiers","case_set":"dev","prompt":"Normalize a customer export, including names with accents, mojibake, and IDs like 00123. Use whatever Unicode cleanup seems best.","expected_output":"A conservative text and identifier plan preserving originals, using appropriate normalization, auditing mojibake and invisible characters, and never converting identifiers to numbers or applying compatibility normalization blindly.","assertions":["Preserves original text and identifier values beside normalized values.","Treats Unicode normalization and mojibake repair as different operations requiring an audit.","Preserves leading-zero identifiers and does not infer numeric semantics from appearance.","Requires review or quarantine for ambiguous repairs."]},
    {"id":"idempotent-verification","case_set":"release","prompt":"The cleaning pipeline passed its schema checks once. What evidence is needed before releasing it as a recurring job?","expected_output":"A release gate requiring deterministic and idempotent reruns, full-data validation beyond samples, reconciliation of rows/keys/totals, accounted quarantine, provenance/checksums, privacy review, and monitoring baselines.","assertions":["Requires running the same input twice and comparing canonical outputs.","Requires full-data checks for row/grain, keys, relationships, totals, and rare failures rather than profile-only evidence.","Requires provenance, contract version, checksums, and rejection/quarantine accounting.","Includes monitoring for drift and a review path for threshold or schema changes."]}
  ]
}