magnus919_agent-skills/api-design-and-evolution/evals/evals.json

{
  "schema_version": 1,
  "skill_name": "api-design-and-evolution",
  "evals": [
    {
      "id": "rest-api-contract-design",
      "prompt": "We are building a public REST API for our invoicing product. I need the contract for the invoices resource: endpoints, pagination, error handling, and filtering. What should the contract specify and what decisions matter most for consumers?",
      "expected_output": "A REST contract design for the invoices resource that makes the consumer-facing decisions explicit: the resource URL structure with consistent plural nouns and stable identifiers, the HTTP methods and their semantics (list, get, create, update, delete) with proper status codes, pagination as a stable mechanism (cursor or offset with a stated default and maximum page size), consistent error responses with a machine-readable error code, message, and correlation ID, idempotency for creation via a client-supplied key, and filtering and sorting parameters that are documented and bounded. The response explains the compatibility discipline: fields and parameters are additive, response shapes are stable, and the OpenAPI document is the source of truth that consumers build against.",
      "assertions": [
        "The contract defines the full invoice resource surface: methods, status codes, and URL structure",
        "Pagination is specified as a stable mechanism with defaults and limits",
        "Error responses have a consistent machine-readable shape with codes and correlation IDs",
        "Creation is idempotent via a client-supplied key",
        "Filtering and sorting are documented and bounded, with OpenAPI as the source of truth"
      ]
    },
    {
      "id": "versioning-deprecation",
      "prompt": "We need to change the response of our customers endpoint from a flat structure to a nested one, which will break current consumers. The API is used by dozens of partners. How do I version this change and manage the deprecation lifecycle responsibly?",
      "expected_output": "A versioning and deprecation plan that avoids breaking consumers: the response evaluates versioning options (URL path versioning versus content negotiation versus additive-only evolution) and selects one for the change, then defines the deprecation lifecycle: ship the new version alongside the old, announce the deprecation with a concrete timeline, add Sunset headers and deprecation notices in responses so consumers see it programmatically, migrate the known partners with support, and remove the old version only after the deadline with the usage metrics confirming no remaining traffic. The response explains that a breaking change that could be done additively (new field, old field kept) should not force a version bump, and it sets the policy for when a major version is genuinely warranted.",
      "assertions": [
        "The response evaluates versioning strategies and picks one for the change",
        "The change is assessed for additive compatibility before forcing a major version",
        "A deprecation lifecycle is defined: parallel versions, announcement, Sunset headers, timeline",
        "Partner migration is supported and removal is gated on usage metrics",
        "Deprecation notices are surfaced programmatically to consumers"
      ]
    },
    {
      "id": "event-interface-asyncapi",
      "prompt": "We are adding an events interface so internal services and external partners can subscribe to invoice.created and invoice.paid events. I have never designed an event interface. What does the contract look like and what decisions do I need to make?",
      "expected_output": "An event-interface design with an AsyncAPI contract as the source of truth: the event names and their payload schemas (what fields each event carries and the guarantees about them), the delivery semantics (at-least-once with a message ID, deduplication keys, and ordering caveats), the channel or topic naming scheme, and the compatibility rules for evolving payloads (additive fields only, versioned schema for breaking changes). The response explains the core consumer-contract decisions: exactly-once is not provided so consumers must deduplicate, ordering is per-partition not global, and retries need a dead-letter policy. It specifies what the producer guarantees versus what consumers must handle.",
      "assertions": [
        "The design is documented as an AsyncAPI contract with named events and payload schemas",
        "Delivery semantics are explicit: at-least-once, message IDs, deduplication, ordering caveats",
        "Channel or topic naming and payload evolution rules are specified",
        "The response states that consumers must handle deduplication and that ordering is per-partition",
        "Retry and dead-letter handling are part of the consumer contract"
      ]
    },
    {
      "id": "api-review-existing-contract",
      "prompt": "A teammate wrote an OpenAPI spec for a new bookings API and asked for a review before publishing it to partners. What should I look for in a contract review beyond syntax correctness?",
      "expected_output": "An API contract review that checks the decisions that create or avoid future breaking changes: naming consistency and URL structure, response envelope consistency, error schema uniformity, pagination on list endpoints, whether create/update are idempotent or need to be, whether required fields are truly required or just asserted, parameter validation and bounds, and whether the spec matches the documented behavior in examples. The review prioritizes findings by consumer impact: anything that forces a breaking change later or that partners will mis-implement gets flagged first, followed by inconsistencies and documentation gaps. The response frames the review output as actionable findings with severity rather than a general comment thread.",
      "assertions": [
        "The review checks future-compatibility decisions: idempotency, pagination, error uniformity, required-field truthfulness",
        "Findings are prioritized by consumer impact and breaking-change risk",
        "The review compares the spec against its own examples for consistency",
        "Parameter validation and bounds are checked",
        "The review produces actionable, severity-ranked findings"
      ]
    },
    {
      "id": "error-handling-idempotency",
      "prompt": "Our mobile app sometimes retries a payment API call and ends up charging customers twice. The API returns 500 on timeouts, and the client retries blindly. How should the API and client coordinate so retries are safe?",
      "expected_output": "A retry-safety design centered on idempotency: the API accepts an idempotency key from the client, stores the key with the result of the first attempt, and returns the stored result on any retry with the same key instead of processing again. The response specifies the client contract: generate a key per logical operation, reuse it on retries, and treat 5xx and network timeouts as retryable while 4xx are not. It also covers the API-side decisions: idempotency-key validity window, uniqueness enforcement under concurrency, and what happens when a key is replayed with a different payload, plus the timeout-error shape (409 or a dedicated response that lets the client know the outcome is being determined) so the client does not double-submit.",
      "assertions": [
        "The design uses client-supplied idempotency keys stored with the first attempt's result",
        "Replayed keys return the stored result rather than re-processing",
        "The client contract distinguishes retryable (5xx, network) from non-retryable (4xx) failures",
        "Key validity, uniqueness under concurrency, and key-replay-with-different-payload are addressed",
        "The timeout response shape prevents the double-submit race"
      ]
    },
    {
      "id": "api-landscape-governance",
      "prompt": "Our company has 40 APIs and several teams expose customer and order data through overlapping REST endpoints. We need a landscape assessment and a governance recommendation without creating a heavyweight central review board. What should we inventory, decide, and measure?",
      "expected_output": "A landscape assessment that inventories surfaces, authority, owners, audiences, consumers, contracts, lifecycle state, evidence confidence, and topology. It identifies duplication by consumer job and domain authority rather than URL similarity, improves discoverability, assigns lifecycle and retirement ownership, and proposes risk-proportional standards with visible exceptions and local decision rights. It keeps product, platform, security, and architecture responsibilities distinct.",
      "assertions": [
        "The assessment includes inventory fields for authority, ownership, consumers, contract, lifecycle, and evidence gaps",
        "Duplication is evaluated by domain meaning and consumer job, with intentional overlap allowed when justified",
        "Discoverability and lifecycle include ownership, feedback, deprecation, retirement, and residual dependency evidence",
        "Governance is proportional to exposure, blast radius, data sensitivity, and reversibility rather than a universal maturity ladder",
        "The response preserves explicit boundaries with product, platform, security, and architecture owners"
      ]
    },
    {
      "id": "gateway-mesh-topology",
      "prompt": "We are adding an ingress gateway and a service mesh while moving internal order traffic between clusters. Explain the north-south and east-west responsibilities, where routing and policy belong, what observability is required, and how to avoid hiding failure or multiplying retries.",
      "expected_output": "A topology assessment that maps the request path and separates gateway or ingress responsibilities from mesh transport responsibilities and service/domain authorization. It identifies north-south versus east-west traffic, owner and unavailable-component behavior for each policy, timeout and retry ownership, telemetry across hops, and failure boundaries including ambiguous completion. It calls out contract-visible changes during migration and requires deployed-boundary verification.",
      "assertions": [
        "North-south and east-west traffic are distinguished with responsibilities for each",
        "Gateway/ingress and service-mesh concerns are separated without treating either as a universal product boundary",
        "Routing, identity, policy placement, owner, and fail-open/closed behavior are explicit",
        "Timeouts, retries, budgets, observability, and failure attribution address retry multiplication and hidden failures",
        "Topology migration checks contract-visible behavior and deployed-boundary evidence"
      ]
    },
    {
      "id": "landscape-retirement-boundary",
      "prompt": "A team wants to delete an old partner API because its dashboard shows almost no traffic. Give me a safe decision process and state which parts belong to API design versus product, platform, security, and architecture owners.",
      "expected_output": "A retirement decision process that verifies inventory completeness, undocumented and indirect consumers, owner and successor, consumer-specific migration support, telemetry limitations, communication, pause or rollback conditions, residual routes and credentials, and post-retirement monitoring. It distinguishes API contract and lifecycle evidence from product value decisions, platform route removal, security credential or abuse review, and architecture-level cross-domain decisions.",
      "assertions": [
        "Low observed traffic is treated as insufficient evidence without checking inventory confidence and hidden consumers",
        "Retirement requires an owner, successor or explicit no-successor decision, migration support, criteria, communication, and pause/recovery path",
        "Residual routes, credentials, jobs, caches, SDKs, and indirect dependencies are considered",
        "Product, platform, security, architecture, and API ownership boundaries are explicit",
        "The process ends with evidence-based retirement and post-retirement monitoring rather than deletion alone"
      ]
    }
  ]
}