magnus919_agent-skills/data-architect/evals/evals.json

{
  "schema_version": 1,
  "skill_name": "data-architect",
  "evals": [
    {
      "id": "quickscan-assessment",
      "prompt": "My team's data pipelines keep breaking, the cloud bill is climbing without explanation, and nobody agrees on what 'customer' means across our reports. I do not know where to start. Run a quick scan of our data organization and tell me what the top gaps are.",
      "expected_output": "A quick-scan assessment that turns the symptoms into a structured gap list: the response walks the five-minute scan over the common failure areas — pipeline reliability (how data is loaded, where failures happen and whether they are detected), cost (where spend concentrates, whether compute is runaway or rightsized), definitions (whether 'customer' and other core entities are defined once or redefined per report), ownership (who owns each pipeline and what happens when it breaks), and trust (whether anyone can trace a number on a dashboard to its source). It maps each symptom to the likely root gap: breaking pipelines to missing ownership and observability, climbing bills to ungoverned compute, and the customer disagreement to a missing shared semantic layer. It ends with the prioritized gaps and the first concrete action for each.",
      "assertions": [
        "The scan covers pipeline reliability, cost, definitions, ownership, and trust",
        "Each symptom is mapped to a likely root gap rather than a superficial fix",
        "The 'customer' disagreement is tied to a missing shared definition layer",
        "The output is a prioritized gap list with a first concrete action per gap",
        "The response reflects a quick scan and names the deeper review each gap needs"
      ]
    },
    {
      "id": "pipeline-architecture-review",
      "prompt": "We ingest events from our app into a warehouse through a chain of scripts, transform them in the database, and export dashboards. The pipeline works but takes 14 hours and fails often. Review this architecture and tell me what should change.",
      "expected_output": "An architecture review that evaluates the pipeline against the properties that matter: the response assesses the current state (script-based orchestration, in-database transforms, batch frequency) and identifies the structural weaknesses — fragile orchestration without retries and monitoring, transformations that run in the serving database and scale poorly, no incremental processing so the 14-hour runtime grows with data volume, and no data-quality checks between stages so failures surface downstream. It proposes the target shape: orchestration with retries and observability, staging and transform layers that separate raw, cleaned, and modeled data, incremental or partitioned processing to bound runtime, and quality gates at each stage. The review is prioritized: the changes that reduce failure and runtime land first, and it identifies which parts of the current architecture can stay (the serving layer, the dashboards) while the plumbing is reworked.",
      "assertions": [
        "The review identifies the structural weaknesses: fragile orchestration, in-DB transforms, no incremental processing",
        "It proposes a layered target with raw, cleaned, and modeled stages and quality gates",
        "Incremental or partitioned processing is prescribed to bound the runtime",
        "Changes are prioritized by failure-and-runtime reduction",
        "Working parts such as the serving layer are retained rather than rewritten wholesale"
      ]
    },
    {
      "id": "platform-decision-framework",
      "prompt": "We need a data platform and are torn between using our existing Postgres for everything, adopting a cloud warehouse, and a newer lakehouse stack. The team has different opinions and the vendors are pushing hard. How do I make this decision properly?",
      "expected_output": "A decision framework that defers the platform choice until the requirements are understood: the response identifies the decisions the platform must serve — the workloads (analytics, ML feature access, real-time versus batch), the data volumes and concurrency, the team's skills and operating capacity, and the future direction (lakehouse expansion, streaming). It frames the comparison across the named options on those requirements, including the total cost of ownership (licensing, compute, storage, and the people cost of operating each), and it exposes the vendor-pressure dynamic by grounding the choice in the workload evidence rather than platform enthusiasm. The framework produces a recommendation with the conditions under which the other options would win, and a pilot or proof-of-value step before commitment. It explicitly warns against picking the platform to avoid a later decision.",
      "assertions": [
        "The framework defines the workloads, volumes, concurrency, and team capacity before comparing platforms",
        "Options are compared on total cost of ownership including operating people-cost",
        "The choice is grounded in workload evidence rather than vendor momentum",
        "The recommendation includes the conditions under which each alternative would win",
        "A pilot or proof-of-value step precedes the commitment"
      ]
    },
    {
      "id": "semantic-layer-governance",
      "prompt": "Marketing reports revenue one way, finance reports it another, and the two numbers are different by 12%. I need to fix the definitions and stop the argument. How do I set up a governed semantic layer without freezing all data work?",
      "expected_output": "A semantic-layer governance design that treats definitions as owned artifacts: the response establishes a single source of truth for core metrics (revenue, customer, active user) with a written definition, the calculation, and the owner for each, and it explains the governance model — definitions change through a review process with recorded rationale rather than per-report improvisation. The migration path keeps work moving: the semantic layer is introduced for the disputed metrics first, reports are migrated one at a time with a comparison period showing the old and new numbers side by side, and the legacy report is retired only when it matches. The response explains the 12% difference by identifying the likely divergence points (inclusion of refunds, definition of the reporting period, deduplication rules) and prescribes documenting those as part of the definition.",
      "assertions": [
        "Core metrics get a written definition, calculation, and named owner",
        "Definitions change through a review process, not per-report improvisation",
        "Migration is incremental with side-by-side comparison before retiring the legacy report",
        "The 12% divergence is diagnosed against likely divergence points such as refunds and period definitions",
        "The governance model prevents the argument from restarting without freezing data work"
      ]
    },
    {
      "id": "strategy-roadmap",
      "prompt": "Our data team spends all its time firefighting broken pipelines and has no time to build the analytics the business is asking for. Leadership wants a data strategy. What should the strategy and roadmap contain, and how do we get out of firefighting?",
      "expected_output": "A data strategy that addresses the firefighting trap structurally: the response frames the strategy around the outcomes the business needs and the capabilities required, then sequences a roadmap that first stabilizes the foundation — ownership for the broken pipelines, observability so failures are detected and repaired fast, and the quick wins that stop the most frequent incidents — before adding net-new analytics. The roadmap is phased with explicit criteria for moving from one phase to the next: the firefighting load must fall below a threshold, not just a calendar date. It covers the governance and staffing implications (who owns the platform, how new requests are triaged), and it names the metrics that show the strategy working: incident rate, time-to-repair, on-time analytics delivery. The response resists a roadmap that schedules the new analytics first while the foundation stays broken.",
      "assertions": [
        "The strategy is organized around business outcomes and required capabilities",
        "The roadmap sequences foundation stabilization before net-new analytics",
        "Phase transitions are gated on firefighting-load criteria, not calendar dates",
        "Ownership, triage, and governance implications are addressed",
        "The metrics showing the strategy works are named, including incident rate and delivery"
      ]
    },
    {
      "id": "mesh-readiness-decision",
      "prompt": "Our central data team is overloaded, and leadership wants us to adopt data mesh. Three product teams own useful source data, but none has a funded support rotation, our catalog is incomplete, and the platform team is already committed to a warehouse migration. Assess whether we should adopt mesh now and give us a safe next step.",
      "expected_output": "A readiness-based recommendation that separates the central bottleneck from mesh prerequisites. It assesses domain authority, product accountability, consumer demand, self-service platform capacity, federated governance, and sponsorship; identifies missing evidence and operating load; and recommends a bounded pilot or centralized/hybrid foundation rather than treating mesh as the default. The plan has entry and exit evidence, named owners, and a stop condition.",
      "assertions": [
        "The recommendation assesses domain ownership, product accountability, platform capacity, governance, and team funding",
        "Missing support rotations, incomplete catalog coverage, and platform overload are treated as readiness gaps",
        "The response does not present data mesh as the universal or inevitable target",
        "The next step is bounded and includes measurable exit evidence and a stop condition",
        "The response distinguishes architecture decisions from platform implementation work"
      ]
    },
    {
      "id": "event-driven-product-recovery",
      "prompt": "Orders are published as events for fraud analytics and a customer-facing order timeline. We want to replace our nightly export with a streaming data product. Design the architecture decisions we must make before implementation, including snapshots, replay, duplicate and late events, ordering, compatibility, and consumer recovery.",
      "expected_output": "An architecture design that distinguishes the event stream from a current snapshot and historical analytical view, records event and publication time and ordering scope, defines producer and consumer responsibilities, and specifies duplicate, late, delayed, unavailable, poison, replay, backfill, compatibility, and recovery behavior. It routes formal event contract syntax to api-design-and-evolution and implementation to data-engineering, while still making the product's required semantics and service expectations explicit.",
      "assertions": [
        "The design distinguishes events, snapshots, and historical analytical data",
        "Duplicate, late, delayed, unavailable, poison, replay, and backfill behavior is addressed",
        "Ordering scope and event-time versus publication-time semantics are made explicit",
        "Compatibility and deprecation decisions are included without pretending schema additivity guarantees consumer safety",
        "Interface contract semantics and pipeline implementation are routed to the neighboring skills"
      ]
    },
    {
      "id": "pattern-landscape-choice",
      "prompt": "We have a regulated finance workload, a growing ML team, several operational systems, and analysts who need governed SQL. One group proposes a lakehouse, another says data mesh, and a vendor is selling a data fabric. Compare centralized warehouse, lakehouse, fabric capabilities, mesh, and a hybrid design and recommend how to choose.",
      "expected_output": "A workload- and organization-grounded comparison that explains what each pattern solves, the obligations it introduces, and where it can fail. It distinguishes fabric capabilities from the mesh ownership model, recognizes that a hybrid can be deliberate, accounts for regulation, ML, SQL serving, skills, governance, latency, portability, and operating cost, and recommends evidence or a pilot before commitment.",
      "assertions": [
        "All five candidate shapes are compared by workload, ownership, governance, skills, and operating obligations",
        "Data fabric is distinguished from data mesh rather than treated as a synonym",
        "A hybrid design is evaluated as a deliberate option with explicit boundaries",
        "The recommendation accounts for regulated serving, ML, governed SQL, and portability",
        "The decision includes a pilot or evidence plan instead of relying on vendor positioning"
      ]
    },
    {
      "id": "architecture-design-session",
      "prompt": "Facilitate a design session for a company deciding whether customer behavior should be a domain-owned data product. We need current and target flows, workloads, candidate patterns, decisions, assumptions, experiments, owners, and handoffs.",
      "expected_output": "A completed or clearly structured design-session artifact with a session brief, current source-to-consumer flow and ownership, workload table, candidate comparison, chosen scope and rationale, governance and quality expectations, explicit assumptions and evidence gaps, experiments with owners and exit evidence, transition phases, open risks, and handoffs to API, data-engineering, and platform-engineering owners.",
      "assertions": [
        "The artifact includes current flows, ownership, consumers, and workload freshness or latency needs",
        "Candidate patterns include centralized, hybrid, fabric, mesh, and event-driven options where relevant",
        "Decisions, assumptions, evidence gaps, experiments, owners, and exit criteria are separately captured",
        "The artifact names interface, implementation, and platform-operations handoffs",
        "The session does not skip unresolved risks by forcing premature consensus"
      ]
    },
    {
      "id": "boundary-event-contract",
      "prompt": "Write the Avro schema and compatibility policy for our customer-events Kafka topic, including serializer settings and consumer versioning.",
      "expected_output": "A routing response that explains this is an interface-contract and implementation task rather than a data-architecture strategy review. It routes schema, serialization, compatibility, and consumer versioning to api-design-and-evolution and Kafka or pipeline implementation to data-engineering, while optionally stating the architecture context that would be needed before making broader product or platform decisions.",
      "assertions": [
        "The response does not attempt to author the requested schema or serializer configuration as a data-architect output",
        "Event contract and compatibility semantics route to api-design-and-evolution",
        "Kafka or pipeline implementation routes to data-engineering",
        "The boundary is explained in terms of architecture strategy versus interface and implementation ownership"
      ]
    },
    {
      "id": "federated-computational-governance",
      "prompt": "Design an operating method for federated computational governance in a data mesh. It must distinguish enterprise interoperability and policy rules from domain-local rules, explain how rules are encoded and enforced, preserve execution evidence, handle temporary exceptions, assign remediation ownership, and turn recurring exceptions into better standards.",
      "expected_output": "A compact original operating method that uses a two-layer rule model and an evidence feedback loop. It defines global versus domain-local rule authority, gives machine-readable rule fields and execution points, records per-run evidence, makes exceptions scoped and time-bounded with expiry handling, assigns remediation to domain, platform, or federated governance owners, and describes a recurring review that converts patterns into standards or an explicit decision not to standardize. It remains an architecture operating model rather than a vendor-specific implementation guide.",
      "assertions": [
        "Global rules explicitly cover shared interoperability and enterprise policy while domain rules cover local meaning and operating choices",
        "Rules have versioned machine-readable encoding and named execution or enforcement points",
        "Evidence records link the rule version and product or release to the evaluation result, time, and remediation reference",
        "Exceptions are scoped, approved, compensating-control aware, and time-bounded with expiry or renewal handling",
        "Remediation ownership is assigned across domain product, platform enforcement, and federated governance responsibilities",
        "Recurring failures or exceptions feed a review that proposes a global rule, domain rule, platform control, or explicit non-standardization decision"
      ]
    }
  ]
}