diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index e892771..aff3ae5 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -129,7 +129,7 @@ "./backend-engineering" ], "strict": false, - "description": "Design and implement backend services and APIs — REST, gRPC, GraphQL endpoint patterns, service architecture (clean/hexagonal/layered), database access patterns, integration and middleware design, error handling, and service-level testing. Language and framework agnostic. Do not use for frontend, data engineering, or platform infrastructure provisioning." + "description": "Design and implement backend services and APIs — REST, gRPC, GraphQL, event-driven handlers, transaction boundaries, outbox/inbox delivery, migration coexistence, database access, integration, error handling, and service-level testing. Use for application/domain/infrastructure implementation decisions. Language and framework agnostic. Do not use for frontend, data engineering, platform provisioning, API contract ownership, service decomposition strategy, or cross-system migration planning." }, { "name": "binary-analysis", diff --git a/README.md b/README.md index 33880a2..f84d281 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ An expert-level skill for building **conversational multi-agent systems** with M ### [backend-engineering](backend-engineering/SKILL.md) -Design and implement backend services and APIs — REST, gRPC, GraphQL endpoint patterns, service architecture (clean/hexagonal/layered), database access patterns, integration and middleware design, error handling, and service-level testing. Language and framework agnostic. Do not use for frontend, data engineering, or platform infrastructure provisioning. +Design and implement backend services and APIs — REST, gRPC, GraphQL endpoint patterns, clean/hexagonal/layered services, event-driven handlers, outbox/inbox delivery, migration coexistence, database access, integrations, errors, and service-level testing. Language agnostic; routes API contracts, decomposition strategy, cross-system migration, data engineering, and platform provisioning to their owners. ### [binary-analysis](binary-analysis/SKILL.md) diff --git a/backend-engineering/README.md b/backend-engineering/README.md index 1b43625..6cb74fa 100644 --- a/backend-engineering/README.md +++ b/backend-engineering/README.md @@ -1,6 +1,6 @@ # Backend Engineering -Backend engineering methodology — API implementation patterns (REST, gRPC, GraphQL), service architecture (clean/hexagonal/layered), database access patterns, integration and middleware design, error handling, and service-level testing. Language and framework agnostic. +Backend engineering methodology — API implementation patterns (REST, gRPC, GraphQL), service architecture (clean/hexagonal/layered), event-driven application flows, outbox/inbox coordination, migration coexistence seams, database access patterns, integration and middleware design, error handling, and service-level testing. Language and framework agnostic. ## Why Install This Skill @@ -12,13 +12,15 @@ Your agent gains structured patterns for API design, service architecture, datab |-----------|---------| | `SKILL.md` | Core methodology, trigger conditions, reference index | | `references/` | Deep-dive reference files loaded on demand | -| `templates/` | Fillable records: service design record, error-handling taxonomy | +| `templates/` | Fillable records: service design, event flow/coexistence, and error-handling taxonomy | | `scripts/` | `n1-query-spotter.py` — scans Python source for potential N+1 query patterns | | `evals/` | Output-quality eval manifest for the skill's methodology cases | ## Triggers -Building or reviewing APIs, designing service layers, implementing database access patterns, adding error handling, or integrating external services. +Building or reviewing APIs, designing service layers, implementing database access patterns, adding error handling, integrating external services, publishing or consuming domain events, implementing outbox/inbox delivery, or keeping old and new service paths safe during a migration. + +Do not load this skill as the owner of API/event contracts, service decomposition strategy, schema/pipeline operations, or cross-system migration lifecycle; route those decisions to the linked specialist skills. ## Requirements diff --git a/backend-engineering/SKILL.md b/backend-engineering/SKILL.md index d54af06..3948a8e 100644 --- a/backend-engineering/SKILL.md +++ b/backend-engineering/SKILL.md @@ -1,31 +1,35 @@ --- name: backend-engineering -description: Design and implement backend services and APIs — REST, gRPC, GraphQL - endpoint patterns, service architecture (clean/hexagonal/layered), database access - patterns, integration and middleware design, error handling, and service-level - testing. Language and framework agnostic. Do not use for frontend, data engineering, - or platform infrastructure provisioning. +description: Design and implement backend services and APIs — REST, gRPC, GraphQL, + event-driven handlers, transaction boundaries, outbox/inbox delivery, migration + coexistence, database access, integration, error handling, and service-level testing. + Use for application/domain/infrastructure implementation decisions. Language and + framework agnostic. Do not use for frontend, data engineering, platform provisioning, + API contract ownership, service decomposition strategy, or cross-system migration + planning. license: MIT metadata: - tags: backend, api, services, server, database, integration, middleware, query-optimization, - testing + tags: backend, api, services, server, database, integration, middleware, events, outbox, + inbox, idempotency, coexistence, query-optimization, testing source_repo: https://github.com/magnus919/hermes-profiles --- # Backend Engineering Methodology -Backend engineering is the craft of building the server-side systems that power applications — APIs, services, data access, integrations, and the runtime behavior that makes the architecture real. This methodology covers the implementation patterns between architecture design (software-architecture-analysis) and quality validation (qa-methodology). +Backend engineering is the craft of building the server-side systems that power applications — APIs, services, data access, integrations, and the runtime behavior that makes the architecture real. This methodology covers implementation after target design in `software-architecture` and before quality validation in `qa-methodology`; use `software-architecture-analysis` when the current system must first be reverse-engineered. It makes runtime boundaries, transaction behavior, message handling, and coexistence seams executable without taking ownership of the surrounding architecture or migration decision. ## The Backend Engineer's Domain | You own | You don't own | |---------|--------------| -| API implementation — REST/gRPC/GraphQL endpoints, request validation, response formatting, error handling, middleware chains | API contract and service boundary design — that's the api-design-and-evolution | -| Service logic — business rules, workflow orchestration, state management, background job processing | Deployment pipeline and infrastructure — that's the platform-engineer | -| Database access patterns — query design, connection management, transaction boundaries, N+1 detection, pagination | Schema design and migrations — that's the data-architect / data-engineer | -| Integration code — third-party API clients, webhook handlers, message queue consumers/producers | Code review and quality gates — that's the qa-methodology | -| Observability instrumentation at the service level — structured logging, metrics, tracing hooks | Observability infrastructure — that's the SRE / platform-engineer | -| Service-level tests — unit tests for business logic, integration tests for API contracts | Test strategy and automation — that's the QA-engineer | +| API implementation — REST/gRPC/GraphQL endpoints, request validation, response formatting, error handling, middleware chains | API contracts belong to `api-design-and-evolution`; service decomposition and target boundaries belong to `software-architecture` | +| Service logic — business rules, workflow orchestration, state management, background job processing | Deployment pipeline and infrastructure — that's `platform-engineering` | +| Event-driven implementation — domain-event publication, outbox/inbox coordination, handler idempotency, replay and failure paths | Event contract ownership and delivery semantics — that's the api-design-and-evolution | +| Migration seams inside a service — adapters, selectable paths, authority checks, and implementation handoffs | Cross-system migration lifecycle and cutover authority — that's the migration-engineering | +| Database access patterns — query design, connection management, transaction boundaries, N+1 detection, pagination | Data-platform and model strategy belong to `data-architect`; schema and pipeline operations belong to `data-engineering` | +| Integration code — third-party API clients, webhook handlers, message queue consumers/producers | Code review and quality gates — that's `qa-methodology` | +| Observability instrumentation at the service level — structured logging, metrics, tracing hooks | Observability infrastructure and reliability policy belong to `platform-engineering` and `site-reliability-engineering` | +| Service-level tests — unit tests for business logic, integration tests for API contracts | Test strategy and automation — that's `qa-methodology` | ## Reference Files @@ -33,9 +37,12 @@ Backend engineering is the craft of building the server-side systems that power |-----------|-------------| | `references/api-patterns.md` | Designing or implementing API endpoints — resource modeling, versioning, pagination, error response formats, request validation | | `references/service-patterns.md` | Structuring service logic — clean/hexagonal/layered architecture, dependency injection, middleware composition, request lifecycle, background jobs | +| `references/event-driven-service-implementation.md` | Implementing event-driven application flows — domain events, unit of work, transactional outbox/inbox, idempotent handlers, retry/replay, observability, and failure handling | +| `references/migration-coexistence-patterns.md` | Keeping old and new implementations safe to run together — adapters, strangler handoffs, anti-corruption boundaries, dual paths, authority, and removal conditions | | `references/database-testing.md` | Database access patterns (connection pooling, query optimization, N+1 detection, pagination strategies, transaction boundaries, read/write splitting, replication lag) and service-level testing (unit testing business logic, integration testing API contracts with test containers/WireMock, contract testing with Pact, test fixtures, CI integration) | | `references/integration-patterns.md` | Integrating with external systems — retry with backoff, circuit breakers, idempotency keys, webhook verification, message queue consumers | | `references/error-handling.md` | Handling errors systematically — classification (client vs server), structured responses, exception handling patterns, observability correlation | +| `references/source-index.md` | Provenance and ownership notes for this original synthesis; load when reviewing scope or source boundaries | ## Templates @@ -52,6 +59,13 @@ Backend engineering is the craft of building the server-side systems that power ## Related Skills +- [programming-principles](../programming-principles/SKILL.md) — DDD owns bounded contexts, aggregates, domain language, repositories, and domain-modeling guidance. This skill applies those decisions at implementation seams rather than duplicating that catalog. +- [api-design-and-evolution](../api-design-and-evolution/SKILL.md) — owns event/message contracts, delivery semantics, compatibility, and consumer-facing API decisions. +- [migration-engineering](../migration-engineering/SKILL.md) — owns cross-system migration classification, compatibility windows, reconciliation, cutover, recovery, deprecation, and cleanup. This skill only implements service-local coexistence seams. +- [software-architecture](../software-architecture/SKILL.md) — owns service decomposition and target-boundary strategy; backend engineering implements an approved boundary. +- [data-engineering](../data-engineering/SKILL.md) — owns schema migration and pipeline operations; application code may expose the repository or transaction interfaces those operations use. +- [secure-software-engineering](../secure-software-engineering/SKILL.md) — owns threat modeling, authorization, secrets, untrusted inputs, and security acceptance evidence. +- [release-engineering](../release-engineering/SKILL.md) — owns progressive delivery, artifact promotion, release gates, and rollback mechanics. - [postgres](../postgres/SKILL.md) — diagnosing the PostgreSQL side of a database problem: configuration review, index and query-plan issues, vacuum/bloat, backups/PITR, replication and failover. Application-level data access patterns stay here; engine-level operations route there. - [supabase](../supabase/SKILL.md) — building on Supabase: migrations, RLS, Auth, Storage, and Edge Functions. To measure an agent's Supabase task competence, use its [agent evals harness reference](../supabase/references/agent-evals.md). @@ -66,3 +80,29 @@ Backend engineering is the craft of building the server-side systems that power **Design for failure, not just success** — Every external call can fail. Every database connection can drop. Every message can be duplicated. Idempotency, retry, and graceful degradation are not optimizations — they're requirements. **Test at the right level** — Business logic gets unit tests. API contracts get integration tests. Service boundaries get contract tests. Each level catches a different class of failure. + +## Implementation Decision Path + +1. Name the bounded context, aggregate/invariant boundary, and source of truth. Use + [programming-principles](../programming-principles/SKILL.md) for DDD choices rather + than rebuilding its catalog here. +2. Put transport, broker, database, clock, and vendor concerns behind ports owned by + the application or domain-facing code. Let infrastructure implement those ports. +3. For a command that changes durable state and emits a fact, load the aggregate, + invoke domain behavior, and commit state plus outbox records in one unit of work. + Do not hold that transaction open across network calls. +4. For an incoming message, validate the envelope at the edge, deduplicate within the + consumer's authority, apply the handler, and acknowledge only after its durable + effects commit. Load the event reference for replay and poison-message decisions. +5. If old and new paths coexist, record which path is authoritative for each operation, + how outputs are compared, and what evidence permits handoff or removal. Load the + migration reference for the implementation seam; route the migration lifecycle out. +6. Add unit tests for domain/application behavior and boundary integration tests for + transaction, outbox, inbox, duplicate, retry, replay, and recovery behavior. + +## Exit Criteria + +This skill is complete when the implementation has explicit dependency direction, +transaction and authority boundaries, classified failure/retry behavior, observable +message or coexistence paths, focused tests for duplicate and failure cases, and clear +links to the neighboring owner for every out-of-scope decision. diff --git a/backend-engineering/evals/evals.json b/backend-engineering/evals/evals.json index f843a04..bc3e6f4 100644 --- a/backend-engineering/evals/evals.json +++ b/backend-engineering/evals/evals.json @@ -73,6 +73,54 @@ "The response resolves timeout ambiguity by querying the hold status before retrying the mutation", "The response defines failure behavior after retries are exhausted rather than hanging" ] + }, + { + "id": "event-driven-command-boundary", + "prompt": "An order command updates the orders table and then publishes OrderConfirmed directly to the broker before the database transaction commits. A broker timeout causes the handler to retry, and some consumers send duplicate emails. Design the implementation flow and tests.", + "expected_output": "A service implementation plan that separates domain, application, and infrastructure responsibilities; uses a unit of work to commit aggregate state and an outbox record atomically; publishes only after commit; assumes duplicate delivery; chooses one atomic inbox transaction design with a uniqueness constraint and acknowledges completed duplicates without claiming an unpersisted outcome; uses idempotent side-effect handling; classifies retryable versus permanent failures; and tests crash, duplicate, retry, quarantine, and replay behavior with lifecycle observability.", + "assertions": [ + "The response places domain behavior, application coordination, and broker/database adapters behind explicit boundaries", + "The response commits state and an outbox record in one local transaction and does not publish inside the transaction", + "The response uses stable event identity plus consumer-scoped inbox deduplication, chooses one coherent atomic inbox transaction design, and acknowledges duplicates without claiming an unpersisted outcome", + "The response distinguishes retryable, permanent, poison, and replay cases and prevents duplicate email side effects", + "The response includes tests and metrics/logs/traces for outbox age, duplicates, retries, lag, and dead-letter handling" + ] + }, + { + "id": "event-replay-and-failure", + "prompt": "A payment consumer has a backlog after a deployment. Some events are from an older schema version, one handler bug creates poison messages, and the relay may have published before crashing. Give an implementation and recovery checklist, but do not redesign the public event contract.", + "expected_output": "A bounded handler/replay checklist that treats delivery as at-least-once, deduplicates stable event identities, uses versioned translation or a repair path for old schemas, quarantines poison messages, records replay scope and handler version, retries only transient failures, and verifies side effects and observability. It uses the same consumer identity by default and warns that fresh-consumer replay defeats inbox deduplication unless the handler is side-effect-safe or the replay uses repair/compensation. It explicitly routes public contract semantics to api-design-and-evolution.", + "assertions": [ + "The checklist assumes duplicate publication and requires consumer deduplication", + "The response handles old schema versions through a versioned translator or repair path rather than silently applying changed rules", + "The response quarantines poison messages and defines bounded retry and replay stop conditions", + "The response records replay selection, handler version, operator/evidence, verifies side effects, and warns that a fresh consumer scope defeats deduplication unless side-effect-safe or repair/compensation handling is used", + "The response does not redesign the event contract and routes contract semantics to api-design-and-evolution" + ] + }, + { + "id": "migration-coexistence-handoff", + "prompt": "We are moving invoice calculation from a monolith module to an approved service boundary. For two releases both paths must exist, the monolith currently owns writes, and a legacy status vocabulary differs from the new service's model. What should the backend implementation team build and measure?", + "expected_output": "An implementation seam plan using an adapter and explicit anti-corruption translation, a selectable strangler handoff, one declared authority for each operation, comparison or shadow execution without duplicated irreversible side effects, and observable late-old-write detection. It proves the new path authoritative and accepting writes before disabling old writers, then names evidence-based removal conditions while routing decomposition and migration lifecycle decisions to their owners.", + "assertions": [ + "The response uses an adapter and explicit translation boundary to keep legacy vocabulary out of the new domain/application model", + "The response makes old and new paths selectable and keeps the monolith authoritative until comparison evidence supports handoff", + "The response declares authority per operation/data field and addresses duplicate side effects, lag, disagreement, and late old writes", + "The response sequences new-path authority and verified write acceptance before disabling old writers, then names measurable handoff and removal conditions including callers, queues/writes, flags, credentials, and recovery evidence", + "The response routes target decomposition to software-architecture and cross-system migration lifecycle to migration-engineering" + ] + }, + { + "id": "event-consumer-security-boundary", + "prompt": "A service consumes signed order events from a broker. The handler currently deserializes the payload with a general-purpose object loader, trusts the producer_id and event_id fields, interpolates a payload field into a SQL query, and stores rejected messages by writing the raw payload into a shared quarantine table. Design the implementation changes and tests without redesigning the public event contract.", + "expected_output": "An implementation plan that verifies producer authenticity and authorization before trusting event identity, uses safe data-only deserialization and treats fields as untrusted at every boundary, parameterizes database access and protects command/template/path sinks, stores quarantine material as bounded opaque or encrypted bytes with sanitized metadata and restricted access, and routes detailed threat modeling and security-control design to secure-software-engineering while keeping handler and boundary tests in backend-engineering.", + "assertions": [ + "The response verifies producer authenticity and authorization before trusting producer-supplied event identity and routes the control design to secure-software-engineering", + "The response requires safe data-only deserialization and treats deserialized fields as untrusted before domain mapping or use in SQL, commands, templates, or paths", + "The response requires parameterized or otherwise safe handling at injection sinks", + "The response stores quarantined content as bounded opaque or encrypted data with sanitized metadata and restricted access rather than blindly reparsing raw payloads", + "The response includes backend boundary tests for forged identity, unsafe payloads, injection attempts, and quarantine handling without redesigning the public event contract" + ] } ] } diff --git a/backend-engineering/references/event-driven-service-implementation.md b/backend-engineering/references/event-driven-service-implementation.md new file mode 100644 index 0000000..656f867 --- /dev/null +++ b/backend-engineering/references/event-driven-service-implementation.md @@ -0,0 +1,122 @@ +# Event-Driven Service Implementation + +Load this reference when a service publishes domain facts, consumes messages, or +coordinates durable state with asynchronous work. It covers implementation seams; +use `api-design-and-evolution` for the event contract and delivery agreement. + +## Boundaries And Flow + +Keep three responsibilities visible, even when the project uses different names: + +| Boundary | Owns | Must not know | +|---|---|---| +| Domain | Invariants, state transitions, and meaningful past-tense business facts | ORM types, broker clients, HTTP, serialization, retry policy | +| Application | Use-case coordination, unit-of-work scope, ports, and event dispatch intent | Vendor-specific connection or broker details | +| Infrastructure | Database mappings, transaction implementation, outbox relay, broker adapter, inbox store | Business decisions and domain invariants | + +The normal command path is: + +1. Translate the request into an application command at the transport boundary. +2. Open one unit of work, load the aggregate through a domain-facing repository, and + invoke behavior that protects its invariants. +3. Collect domain events produced by the successful state transition. Events are facts, + not instructions to make setters run; do not emit one for every field change. +4. Persist aggregate changes and an outbox row in the same local transaction. The row + contains a stable event identity, type, schema version, aggregate identity/version, + occurred-at value, payload, and trace/correlation context appropriate for the + service's data policy. +5. Commit before performing broker I/O. A relay publishes committed rows and records a + delivery result or retry state. It must tolerate a crash after publish and before + marking the row complete, so consumers still need deduplication. + +The unit of work is a logical application operation, normally one aggregate's +immediate invariants plus its outbox records. A broader transaction needs a concrete +invariant justification. Do not turn it into a distributed transaction by calling an +external service while the local transaction is open. + +## Inbox And Idempotent Handlers + +Assume at-least-once delivery unless the contract proves a narrower guarantee. An +inbox record or equivalent deduplication mechanism should be keyed by the consumer's +identity and the producer event identity, not merely by a business field that may be +reused. This reference uses a single atomic transaction design: the inbox insert, +handler effects, and completed state commit together. It does not use leases or +separate claim/completion transactions. + +1. Validate the envelope and schema version before domain mapping. Treat event input + as untrusted: verify producer authenticity and authorization before trusting the + producer identity or event identity. Route the protocol, key, credential, and + authorization design to `secure-software-engineering`. +2. Parse with a safe, data-only deserializer; never use pickle, eval, unsafe YAML, or + gadget-prone formats. Treat every deserialized field as untrusted at downstream + boundaries and use parameterized queries plus safe command, template, and path + handling. The quarantine path must use bounded, access-controlled storage for + opaque or encrypted raw bytes and sanitized metadata, not blindly persist and + reparse executable or injection-bearing content. Route detailed injection and + deserialization controls to `secure-software-engineering`. +3. Insert the inbox identity with a uniqueness constraint in the same transaction as + the handler effects. +4. If the identity already completed, acknowledge without repeating side effects or + claiming to return an outcome that was not persisted. A competing transaction + observes the unique-conflict result after the owner commits; it does not wait on a + lease or stale claim. +5. Apply a handler whose state transition is safe to repeat. Use a natural idempotency + key or a version/precondition check for effects outside the local store. +6. Persist the handler's changes and completed-inbox state atomically, then acknowledge. + +Do not call a handler "idempotent" because the final row looks unchanged. Check +emails, payments, downstream commands, counters, notifications, and external writes +for duplicate effects. Where an external effect cannot be made idempotent, persist a +durable intent and reconcile its status rather than guessing after a timeout. + +## Retry, Replay, And Failure + +Classify failures before choosing the response: + +| Failure | Default action | Evidence or stop condition | +|---|---|---| +| Temporary broker/database/network issue | Bounded retry with backoff and jitter | Attempt count, age, and queue lag remain within budget | +| Concurrency or serialization conflict | Roll back the unit of work and retry the whole operation | Finite attempts; preserve the original correlation context | +| Invalid or unauthorized event | Do not retry; quarantine and alert the owner | Payload, producer, and reason are available without leaking secrets | +| Poison message or deterministic handler bug | Move to a dead-letter/quarantine path after policy limits | Repair or replacement is tested before replay | +| Unknown schema version or gap | Pause or route to compatibility handling | Contract owner resolves version/ordering decision | +| Publish succeeded but acknowledgement was lost | Republish safely; consumer deduplication resolves the duplicate | Stable event identity and duplicate metrics | + +Replay is a controlled operation, not a blind retry loop. Record the source position +or event IDs, selection criteria, handler version, destination, operator, and expected +side effects. Use the same consumer identity and inbox semantics by default. A fresh +consumer scope deliberately defeats inbox deduplication and must be restricted to +handlers proven side-effect-safe or routed through a repair/compensation path; it is +not safe for payments, refunds, or other irreversible effects. Make the stop condition +explicit. If handlers are not deterministic across versions, +use a versioned translator or a new repair command instead of replaying old payloads +through changed rules. + +## Observability And Tests + +Instrument the lifecycle with stable dimensions such as service, event type, producer, +consumer, schema version, result class, and deployment version. Useful signals include +outbox age and backlog, publish attempts and failures, inbox duplicate rate, handler +latency, retry counts, dead-letter volume, replay volume, and consumer lag. Propagate +trace/correlation identifiers, but redact payloads, credentials, and sensitive fields. + +Test the smallest useful unit at each boundary: + +- Unit-test aggregate invariants and the exact domain facts emitted for successful + transitions; no broker or database is needed. +- Integration-test that state and outbox rows commit together, and that a rollback + leaves neither durable effect. +- Test relay crashes and duplicate publication, then verify consumer deduplication. +- Test handler retry, permanent failure, quarantine, schema mismatch, replay, and + acknowledgement ordering against a real or production-compatible broker/store. +- Assert observability fields and alert inputs for lag, age, duplicate, and poison paths. + +## Ownership Boundary + +`programming-principles` owns bounded contexts, aggregates, domain language, domain +events as DDD concepts, repositories, and anti-corruption guidance. This reference +only explains how those decisions become application and infrastructure behavior. +`api-design-and-evolution` owns event names, envelopes, compatibility, ordering scope, +retention, and consumer-facing delivery semantics. `site-reliability-engineering` +owns SLOs and paging policy; `secure-software-engineering` owns threat modeling, +authorization, and secure logging. diff --git a/backend-engineering/references/migration-coexistence-patterns.md b/backend-engineering/references/migration-coexistence-patterns.md new file mode 100644 index 0000000..70d9e95 --- /dev/null +++ b/backend-engineering/references/migration-coexistence-patterns.md @@ -0,0 +1,101 @@ +# Migration Coexistence Patterns + +Load this reference when an implementation must run beside an older path during an +authorized service or interface transition. It does not decide whether to decompose +the system or own the cross-system migration lifecycle. Use `migration-engineering` +for classification, compatibility windows, reconciliation, cutover, recovery, +deprecation, and cleanup; use `software-architecture` for the target boundary. + +## Make The Seam Explicit + +Before changing behavior, record the capability, current path, candidate path, +callers, data touched, invariant, authority, and interruption point. Choose a seam +that can be observed and selected without making both implementations the permanent +source of truth. + +| Pattern | Implementation responsibility | Evidence to collect | +|---|---|---| +| Adapter | Translate the old request/response shape into the new port or the reverse | Translation rules, lost information, error mapping, and contract tests | +| Strangler handoff | Route a bounded operation or cohort to the new path while the old path remains selectable | Traffic, outcome, latency, and failure comparison by route/cohort | +| Anti-corruption boundary | Keep legacy or foreign vocabulary out of the local domain/application model | Explicit mapping, ownership of translation, and tests for unknown/obsolete states | +| Dual path | Execute or read through both paths only when side effects can be isolated or deduplicated | Authority, comparison tolerance, mismatch categories, and cost guardrail | + +These patterns are different. An adapter translates; strangler routing selects; an +anti-corruption boundary protects a model; a dual path creates comparison evidence. +Combine them only when each role and failure behavior is named. + +## Authority And Handoff + +For every operation and important field, state one authority during each coexistence +phase. A read switch is not an ownership transfer. The handoff record should answer: + +- Which path accepts writes, and how are old writes detected or rejected? +- Which store is authoritative, and are other values derived, cached, or shadow-only? +- Can the invariant be maintained without a distributed transaction? +- Are duplicate, out-of-order, missing, and deleted records represented? +- What does the non-authoritative path do on disagreement: compare, alert, serve, or + remain unused? +- What customer-visible staleness, ordering, latency, or error change is accepted? +- Who can change the selector, and what evidence permits the next increment? + +Prefer one writer. If a second representation must be kept current, use an explicit +outbox/CDC or other owned synchronization mechanism and reconcile it. Avoid a +permanent synchronous dual-write in request code when neither side is clearly +authoritative; it creates two failure-prone commits without removing the ownership +ambiguity. + +## Safe Handoff Sequence + +1. Add the port, adapter, selector, and characterization/contract tests while the old + path remains the default. +2. Establish the new path's input/output contract and translation boundary. Do not + hide a breaking consumer contract in an adapter; route that decision to + `api-design-and-evolution`. +3. Populate or synchronize data using the migration owner's chosen backfill/CDC + method. Track lag, duplicates, deletes, and reconciliation failures. +4. Compare shadow or parallel outcomes while the old path remains authoritative. Do + not duplicate irreversible side effects merely to obtain a comparison. +5. Shift a bounded cohort or operation. Define a precondition, abort signal, + customer-impact check, and recovery classification for the increment. Route + rollout and flag mechanics to `release-engineering`. +6. Transfer authority explicitly: update the source-of-truth record, enable the new + path, and verify that it accepts writes authoritatively before disabling old + writers. Confirm old-path accesses are observable and denied or harmless in the + same cutover step; abort and restore the prior selector if the new write path is + not proven authoritative. +7. Remove adapters, selectors, old reads/writes, synchronization feeds, flags, and + credentials only when their individual removal conditions pass. Keep a tested + forward repair or restore path where rollback is no longer possible. + +## Removal Conditions + +Removal is justified by evidence, not elapsed time alone. Record conditions such as: + +- no registered or observed callers use the old route for a defined observation window; +- new and old outcomes reconcile within an agreed tolerance across the required + population and time range; +- the new path owns all writes and old writes are blocked or detected; +- queued, delayed, and replayed work is handled by the new consumer; +- dashboards, alerts, runbooks, support procedures, and security controls use the new + path; +- the old data, contract, adapter, flag, and credentials have an owner-approved + recovery classification before removal. + +If any condition is unknown, keep the old path selectable and stop the handoff rather +than calling the coexistence complete. Once the old store or contract is removed, +do not promise rollback by default; classify roll-forward, restore, or irreversible +recovery with `migration-engineering` and `release-engineering`. + +## Failure Cases To Exercise + +- Selector points to a path whose dependency is unavailable. +- Adapter receives an unknown legacy enum or a response with missing fields. +- Both paths observe the same command and an external side effect is attempted twice. +- Old and new reads disagree because of lag or a mapping defect. +- A late old write arrives after authority transfer. +- A replayed event reaches both old and new consumers. +- Removal hides a caller that was not in the inventory. + +For each case, specify whether to reject, queue, serve the authoritative result, alert, +or halt promotion. Include correlation IDs and path/version labels in the evidence, +without logging sensitive payloads. diff --git a/backend-engineering/references/service-patterns.md b/backend-engineering/references/service-patterns.md index 4d537dd..fc17f93 100644 --- a/backend-engineering/references/service-patterns.md +++ b/backend-engineering/references/service-patterns.md @@ -26,6 +26,25 @@ Each layer has a distinct responsibility: | Service | Business rules, workflow orchestration, state mgmt | HTTP concerns, direct database access | | Repository | Data access, query construction, result mapping | Business rules, request parsing | +## Domain, Application, And Infrastructure + +Use the boundary that makes policy independent from delivery and storage details: + +| Boundary | Responsibility | Dependency rule | +|---|---|---| +| Domain | Invariants, state transitions, value semantics, and meaningful business facts | No framework, database, broker, or vendor imports | +| Application | Use-case coordination, ports, unit-of-work scope, authorization handoff, and transaction intent | Depends on domain and interfaces it owns; does not construct infrastructure | +| Infrastructure | ORM/data mapping, transaction implementation, broker relay, HTTP clients, and framework wiring | Implements application ports; does not decide domain policy | + +Organize by business capability or use case within the bounded context before falling +back to technical layers. Keep one logical commit boundary visible in the application +service. For state changes that publish facts, use the focused +[`event-driven-service-implementation.md`](event-driven-service-implementation.md) +reference for outbox/inbox coordination, replay, and failure behavior. The existing +DDD catalog in [`programming-principles`](../../programming-principles/SKILL.md) +owns bounded-context, aggregate, repository, and domain-modeling decisions; do not +duplicate it here. + ## Background Job Processing | Pattern | When to use | Concerns | diff --git a/backend-engineering/references/source-index.md b/backend-engineering/references/source-index.md new file mode 100644 index 0000000..53d407f --- /dev/null +++ b/backend-engineering/references/source-index.md @@ -0,0 +1,29 @@ +# Source Index + +This skill is an original, task-centered synthesis. Sources inform concepts and +terminology; the repository does not reproduce protected prose, tables, diagrams, +examples, taxonomies, or chapter structures. + +| Source | Use in this skill | URL | +|---|---|---| +| Agent Skills specification | Progressive disclosure, relative references, and skill structure | https://agentskills.io/specification | +| CloudEvents | Event identity and envelope boundary considerations | https://cloudevents.io/ | +| Martin Fowler, Event Sourcing | Distinguishing event history from ordinary event publication | https://martinfowler.com/eaaDev/EventSourcing.html | +| Martin Fowler, Strangler Fig Application | Incremental coexistence vocabulary | https://martinfowler.com/bliki/StranglerFigApplication.html | +| Martin Fowler, Parallel Change | Compatibility-first change sequencing | https://martinfowler.com/bliki/ParallelChange.html | +| PostgreSQL documentation | Transaction isolation and failure behavior | https://www.postgresql.org/docs/current/transaction-iso.html | +| OpenTelemetry documentation | Trace and metric instrumentation concepts | https://opentelemetry.io/docs/ | + +## Ownership Notes + +- DDD concepts and anti-corruption boundaries route to `programming-principles`. +- Event contract and delivery semantics route to `api-design-and-evolution`. +- Cross-system transition and recovery route to `migration-engineering`. +- Schema and pipeline operations route to `data-engineering`. +- Security controls and sensitive logging route to `secure-software-engineering`. +- Release promotion and rollback mechanics route to `release-engineering`. + +The private architecture comparison report supplied for this task was used only as +safe synthesized research input. Purchased ebooks were not read or quoted. Future +edits should be rewritten from user tasks and public sources if wording or structure +resembles a source too closely. diff --git a/backend-engineering/templates/service-design-record.md b/backend-engineering/templates/service-design-record.md index eda91c1..70fda1f 100644 --- a/backend-engineering/templates/service-design-record.md +++ b/backend-engineering/templates/service-design-record.md @@ -46,6 +46,23 @@ architecture. |---|---|---|---|---| | `[fill: system]` | `[fill: sync call, webhook, queue]` | `[fill: retry/circuit breaker policy]` | `[fill: how duplicates are prevented]` | `[fill: queue limit, rate limit, load shedding]` | +## Event Flow + +- Domain facts emitted: `[fill: meaningful completed business facts, not property changes]` +- Unit-of-work rule: `[fill: what state and outbox records commit atomically]` +- Outbox relay: `[fill: event identity, publish retry, crash-after-publish handling]` +- Incoming message deduplication: `[fill: consumer identity + event identity, inbox lease/state]` +- Handler idempotency: `[fill: repeated side effects and external-effect strategy]` +- Retry, quarantine, and replay: `[fill: classifications, limits, operator evidence, and stop rules]` + +## Coexistence And Authority + +- Old and new paths: `[fill: adapters, selector, strangler route, or dual path]` +- Authority by operation/data field: `[fill: exactly one authoritative writer or explain the exception]` +- Comparison evidence: `[fill: shadow/parallel comparison, tolerance, and mismatch action]` +- Handoff condition: `[fill: measurable evidence and owner who can transfer authority]` +- Removal condition: `[fill: callers, queues, old writes, flags, credentials, and recovery evidence]` + ## Observability - Structured logging fields: `[fill: request id, trace id, service, environment]` diff --git a/llms.txt b/llms.txt index 1017341..8ac9b49 100644 --- a/llms.txt +++ b/llms.txt @@ -15,7 +15,7 @@ - [artifact-pyramids](artifact-pyramids/SKILL.md): Organize durable agent research outputs as summaries, analysis, and evidence dossiers. Use when producing multi-layer research artifacts or coordinating research handoffs. - [ascii-city-engine](ascii-city-engine/SKILL.md): Build portable, first-person colored ASCII city engines and small GIS-derived city packs. Use when designing terrain-following walking, raycast character rendering, city-provider schemas, or reproducible public-GIS ingestion. Do not use for conventional 3D/WebGL games, multi-level interiors, general GIS analysis, or committing full-resolution GIS archives. - [autogen](autogen/SKILL.md): Expert skill for conversational multi-agent AI with Microsoft AutoGen. AssistantAgent, UserProxyAgent, GroupChat, code execution, nested chats, cancellation tokens, tool integration, and MCP support. Use when building conversation-driven multi-agent systems or comparing agent frameworks. -- [backend-engineering](backend-engineering/SKILL.md): Design and implement backend services and APIs — REST, gRPC, GraphQL endpoint patterns, service architecture (clean/hexagonal/layered), database access patterns, integration and middleware design, error handling, and service-level testing. Language and framework agnostic. Do not use for frontend, data engineering, or platform infrastructure provisioning. +- [backend-engineering](backend-engineering/SKILL.md): Design and implement backend services and APIs — REST, gRPC, GraphQL, event-driven handlers, transaction boundaries, outbox/inbox delivery, migration coexistence, database access, integration, error handling, and service-level testing. Use for application/domain/infrastructure implementation decisions. Language and framework agnostic. Do not use for frontend, data engineering, platform provisioning, API contract ownership, service decomposition strategy, or cross-system migration planning. - [binary-analysis](binary-analysis/SKILL.md): Analyze unknown binary files through a deterministic CLI that wraps Ghidra's static-analysis engine. Use when you need to inspect a PE, ELF, or Mach-O file — triage suspicious binaries, map imported APIs, decompile functions, trace call paths, or produce structured evidence reports. Do not use for runtime analysis (debugging, dynamic tracing, sandbox execution), for modifying or patching binaries, or for binaries you already know everything about. The skill owns planning, hypothesis formation, and evidence synthesis; the CLI owns all deterministic operations. - [brand-designer](brand-designer/SKILL.md): Create comprehensive brand identity documentation for any brand. Guides you through documenting strategy, visual identity (logo, color, typography, imagery), voice and tone, application guidelines, governance, and asset inventory. Produces markdown specs, compiled brand books, and brand-compliant images via reference-image-aware generation. Use when you need to capture a brand's identity in structured, durable form — for vault storage, agency handoff, or press kit distribution. - [c4-diagramming](c4-diagramming/SKILL.md): Create C4 software-architecture diagrams using Mermaid or Structurizr. Use when teams need clear system context, container, component, or code-level views. diff --git a/references/skill-triggers.md b/references/skill-triggers.md index 625a47a..1c5971b 100644 --- a/references/skill-triggers.md +++ b/references/skill-triggers.md @@ -22,6 +22,7 @@ Each skill's `description` field is the canonical routing contract. This conveni | "weather", "forecast", "temperature", "is it raining", "Tempest" | [tempest-cli](../tempest-cli/SKILL.md) | | "traefik", "reverse proxy", "load balancer", "API gateway", "Let's Encrypt", "ACME", "Docker routing", "traefik.yml", "entry point", "middleware", "TLS termination", "forward auth", "rate limit" | [traefik](../traefik/SKILL.md) | | "reverse-engineer", "understand this codebase", "PRD from code", "architecture document", "architecture health", "coupling analysis", "modularity", "decomposition readiness", "data ownership map", "distributed workflow analysis", "reconciliation path" | [software-architecture-analysis](../software-architecture-analysis/SKILL.md) | +| "backend service", "service layer", "domain/application/infrastructure", "unit of work", "domain event implementation", "transactional outbox", "inbox deduplication", "idempotent handler", "event replay handler", "message consumer implementation", "service coexistence", "strangler handoff", "service adapter", "anti-corruption adapter", "dual path authority" | [backend-engineering](../backend-engineering/SKILL.md) | | "software architecture design", "greenfield architecture", "target architecture", "architecture tradeoff", "modular monolith", "service decomposition decision", "distributed workflow design", "architecture fitness function", "architecture drift", "architecture review" | [software-architecture](../software-architecture/SKILL.md) | | "enterprise architecture", "enterprise capability map", "capability mapping", "value-stream architecture", "application portfolio", "information portfolio", "business technology alignment", "current-state architecture", "target-state enterprise architecture", "transition architecture", "enterprise architecture operating model", "cross-domain architecture decision rights", "enterprise architecture governance", "architecture stakeholder engagement" | [enterprise-architecture](../enterprise-architecture/SKILL.md) | | "data architecture", "data platform", "data strategy", "data mesh", "governance" | [data-architect](../data-architect/SKILL.md) |