diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 09fdd93..25695d8 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1013,6 +1013,15 @@ "strict": false, "description": "Build and maintain technology radars for adoption, trial, assessment, and hold decisions. Use when governing technology choices, build-versus-buy decisions, or engineering portfolio risk." }, + { + "name": "telemetry", + "source": "./", + "skills": [ + "./telemetry" + ], + "strict": false, + "description": "Operate the observability stack that deploys as one unit: Prometheus scrape configuration, recording and alerting rules, relabeling, retention, and high availability; OpenTelemetry Collector pipelines (receivers, processors, exporters, sampling, trace/span correlation); and Loki ingest, LogQL, retention, and label design — with a bundled read-only telemetry-check script for Prometheus rule sanity and scrape-target reachability. Use when running, tuning, or troubleshooting a Prometheus, OpenTelemetry Collector, or Loki deployment, or reviewing the collection/ingest/retention layer. Do not use for observability strategy, SLI/SLO design, or paging policy (that is platform-engineering) or Grafana dashboards, panels, and Grafana-side alerting (that is grafana)." + }, { "name": "tempest-cli", "source": "./", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index ed6b03f..f50be0a 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -131,6 +131,7 @@ "./systematic-debugging", "./technical-documentation", "./technology-radar", + "./telemetry", "./tempest-cli", "./terraform", "./three", diff --git a/README.md b/README.md index a18e748..0a71d54 100644 --- a/README.md +++ b/README.md @@ -465,6 +465,10 @@ Make documentation useful at the moment someone needs to install, operate, exten Turn scattered technology preferences into explicit, reviewable decisions with owners, evidence, and a clear adoption posture. +### [telemetry](telemetry/SKILL.md) + +Operate the Prometheus + OpenTelemetry + Loki observability stack as one unit: scrape config, recording and alerting rules, relabeling, retention, and HA; OpenTelemetry Collector pipelines (receivers, processors, exporters, sampling, trace/span correlation); and Loki ingest, LogQL, retention, and label design. Ships the read-only `telemetry-check` script (rule sanity + scrape-target reachability, `--json`), fixtures, tests, dated references, and 6 evals. Routes strategy to platform-engineering and dashboards to grafana. + ### [tempest-cli](tempest-cli/SKILL.md) Hyper-local weather from a WeatherFlow Tempest station. Query current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. A complete reference implementation of the cli-builder patterns in a working, testable project — including the CLI binary and full API field layout reference. diff --git a/llms.txt b/llms.txt index 5355463..2320632 100644 --- a/llms.txt +++ b/llms.txt @@ -114,6 +114,7 @@ - [tailscale](bundles/tailscale/SKILL.md): Self-hosted Tailscale/Headscale ecosystem: deploy and manage a Headscale control server, configure tailscale clients, manage ACL policies, node lifecycle, subnet routing, DERP relays, and backup/migration. Use when the user mentions Tailscale, Headscale, tailnet, mesh VPN, WireGuard mesh, or self-hosted VPN infrastructure. - [technical-documentation](technical-documentation/SKILL.md): Create and review technical documentation, including READMEs, agent-facing instructions, API references, and CLI help. Use when documentation must help someone complete real work. Do not use for marketing copy, brand messaging, or long-form editorial content. - [technology-radar](technology-radar/SKILL.md): Build and maintain technology radars for adoption, trial, assessment, and hold decisions. Use when governing technology choices, build-versus-buy decisions, or engineering portfolio risk. +- [telemetry](telemetry/SKILL.md): Operate the observability stack that deploys as one unit: Prometheus scrape configuration, recording and alerting rules, relabeling, retention, and high availability; OpenTelemetry Collector pipelines (receivers, processors, exporters, sampling, trace/span correlation); and Loki ingest, LogQL, retention, and label design — with a bundled read-only telemetry-check script for Prometheus rule sanity and scrape-target reachability. Use when running, tuning, or troubleshooting a Prometheus, OpenTelemetry Collector, or Loki deployment, or reviewing the collection/ingest/retention layer. Do not use for observability strategy, SLI/SLO design, or paging policy (that is platform-engineering) or Grafana dashboards, panels, and Grafana-side alerting (that is grafana). - [tempest-cli](tempest-cli/SKILL.md): Query hyper-local weather from a WeatherFlow Tempest station: current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. Use when the user asks about the weather, temperature, rain, wind, humidity, forecast, or wants conditions from their own station rather than a generic weather service. - [terraform](terraform/SKILL.md): Operate Terraform and OpenTofu across the whole infrastructure lifecycle: module structure, state backends and locking, plan/apply workflow, drift detection, remote state, upgrade and refactor flows, and evidence-based diagnostics. Use when running or inspecting terraform plans, applies, state files, imports, or state surgery, or when the bundled tfops script should handle the task. Do not use for IaC methodology or cloud design decisions - those route up to platform-engineering. - [three](three/SKILL.md): Build browser-based Three.js and WebGL scenes, animations, and interactive 3D visualizations with a small vanilla JavaScript starting point. diff --git a/telemetry/README.md b/telemetry/README.md new file mode 100644 index 0000000..0618c92 --- /dev/null +++ b/telemetry/README.md @@ -0,0 +1,50 @@ +# Telemetry — Operational Skill for the Prometheus + OpenTelemetry + Loki Stack + +Operate the observability stack that deploys as one unit: Prometheus scrape configuration, recording and alerting rules, relabeling, retention, and high availability; OpenTelemetry Collector pipelines (receivers, processors, exporters, sampling, trace/span correlation); and Loki ingest, LogQL, retention, and label design. + +## Why Install This Skill + +Your agent can run the collection/ingest/retention layer of observability instead of guessing: review and fix Prometheus scrape configs, author and sanity-check recording and alerting rules, design OpenTelemetry Collector pipelines with deliberate sampling, tune Loki ingest and retention, and diagnose the classic failure modes — missing series, silent ingest loss, and exploding label cardinality — in a fixed evidence order. + +It ships a read-only checker (`telemetry-check`) that parses Prometheus rules files with a bundled stdlib YAML reader and runs sanity checks mirroring `promtool check rules`, then probes scrape-target reachability with TCP connects. It cannot mutate anything: no config writes, no reloads, no data sent anywhere. That makes it safe for an agent to run during discovery. + +The references are distilled from the official Prometheus, OpenTelemetry Collector, and Loki documentation with dated sources and verification-first guidance. Observability strategy deliberately routes to `platform-engineering` and dashboards/alerting to `grafana`; this skill owns the layer those two skills query. + +## What You Get + +| Directory | Purpose | +|---|---| +| `SKILL.md` | Agent-facing operating loop, mutation gates, and verification boundaries | +| `references/` | Five dated references: source index, Prometheus operations, OpenTelemetry Collector, Loki operations, stack integration and retention | +| `scripts/telemetry-check` | Read-only rule sanity + scrape-target reachability checker: stdlib-only Python, `--json`, `--rules`/`--scrape`/`--targets`, `--help` with no server | +| `fixtures/` | Valid `prometheus-rules.yml` and `scrape-config.yml` used by the tests and as starting points | +| `tests/` | Deterministic tests against the fixture configs, including the read-only contract | +| `evals/evals.json` | Six output-quality evaluation cases for agent runs | + +## Quick Start + +```bash +# Help works with no Prometheus server installed +telemetry/scripts/telemetry-check --help + +# Sanity-check a Prometheus rules file, machine-readable +telemetry/scripts/telemetry-check --rules telemetry/fixtures/prometheus-rules.yml --json + +# Probe the static targets of a scrape config +telemetry/scripts/telemetry-check --scrape telemetry/fixtures/scrape-config.yml --json + +# Probe a plain host:port list with a per-target timeout +telemetry/scripts/telemetry-check --targets targets.txt --timeout 5 +``` + +Exit codes: 0 all checks passed, 1 issues found or a fatal error, 2 usage error. The rule checks mirror `promtool check rules` for structure (exactly one of `record`/`alert`, non-empty `expr` with balanced delimiters, valid durations, valid names and label values); use `promtool check rules` when you need full PromQL parsing. + +## Triggers + +Load this skill for `prometheus`, `otel`/`opentelemetry`, `loki`, or general telemetry/observability operations: scrape config and `prometheus.yml` review, recording and alerting rule authoring or debugging, `relabel_configs` problems, TSDB retention and compaction, Prometheus HA pairs, OpenTelemetry Collector pipeline design or troubleshooting (receivers, processors, exporters, sampling), trace/span correlation with logs and metrics, Loki ingest health, LogQL query cost, Loki retention and compactor, and label cardinality. Do not load it for observability strategy or SLOs (that is `platform-engineering`), Grafana dashboards or Grafana-side alerting (that is `grafana`), application instrumentation code (that is `backend-engineering`), or deploying the stack itself on Kubernetes/Docker (that is `kubernetes`/`docker-compose`). + +## Requirements + +- Python 3.9+ for the `telemetry-check` script (`--help` and rule/scrape parsing need nothing else). +- Network access to scrape targets for `--scrape`/`--targets` reachability probes; rule checks are purely local. +- For live verification beyond the checker: access to the Prometheus `/api/v1/*` endpoints and the OTel Collector and Loki health endpoints, and `promtool` if you want full PromQL rule validation. diff --git a/telemetry/SKILL.md b/telemetry/SKILL.md new file mode 100644 index 0000000..fb2fcea --- /dev/null +++ b/telemetry/SKILL.md @@ -0,0 +1,130 @@ +--- +name: telemetry +description: >- + Operate the observability stack that deploys as one unit: Prometheus scrape + configuration, recording and alerting rules, relabeling, retention, and high + availability; OpenTelemetry Collector pipelines (receivers, processors, + exporters, sampling, trace/span correlation); and Loki ingest, LogQL, + retention, and label design — with a bundled read-only telemetry-check script + for Prometheus rule sanity and scrape-target reachability. Use when running, + tuning, or troubleshooting a Prometheus, OpenTelemetry Collector, or Loki + deployment, or reviewing the collection/ingest/retention layer. Do not use + for observability strategy, SLI/SLO design, or paging policy (that is + platform-engineering) or Grafana dashboards, panels, and Grafana-side + alerting (that is grafana). +license: MIT +compatibility: >- + The bundled telemetry-check script runs on Python 3.9+ and needs no + Prometheus server for --help. Rule and scrape-config checks read local + YAML/JSON files; scrape-target reachability probes use TCP connects only and + require network access to the targets. +metadata: + source: https://prometheus.io/docs/introduction/overview/ + source_index: references/00-source-index.md + research_checked: "2026-08-03" +--- + +# Telemetry Operations + +Use this skill to operate the **telemetry stack** — Prometheus, the OpenTelemetry Collector, and Loki — as the one deployment unit it ships as: collection and scraping, ingestion, retention, and the rules that turn raw signals into alerts. This is a **tool skill** for one stack of named tools. Observability *strategy* — SLIs, SLOs, error budgets, and what to instrument — belongs to [platform-engineering](../platform-engineering/SKILL.md); dashboards, panels, and Grafana-side alert rules, contact points, and notification policies belong to [grafana](../grafana/SKILL.md). This skill owns the collection/ingest/retention layer and the Prometheus rules files that both of those skills consume. + +## Operating contract + +1. **Read-only discovery before any mutation.** Inspect scrape configs, rules files, collector pipelines, and retention settings first. The bundled `telemetry-check` script runs rule sanity and scrape-target reachability checks without changing anything. +2. **Confirm the target, scope, and rollback path before acting.** Read-only discovery may proceed without confirmation. Mutations — a config reload, a `promtool` rules push, a collector restart, a retention-policy change — require an explicit human directive naming the instance. +3. **A config that parses is not a config that works.** Rule sanity catches structure; it does not prove the expression is meaningful or that the target is scrapable. Verify at the delivery boundary (scrape succeeded, rule evaluated, alert fired) before claiming health. +4. **Keep evidence bounded.** Summarize config diffs and query results; never dump full `prometheus.yml`, collector pipelines, or credentials into chat. +5. **Own the retention decision.** Retention is a capacity and compliance decision made deliberately per component — Prometheus block retention, OTel exporter buffering, Loki retention per tenant — and reviewed on a schedule, not left at defaults. + +## The telemetry-check script + +`scripts/telemetry-check` is an agent-first, read-only checker. It parses Prometheus rules files with a bundled stdlib YAML reader and runs sanity checks mirroring `promtool check rules`; it extracts static targets from scrape configs and probes TCP reachability; and it emits bounded JSON. It never writes files and never sends data anywhere. + +```bash +scripts/telemetry-check --help # no server needed +scripts/telemetry-check --rules rules.yml --json # rule sanity, machine-readable +scripts/telemetry-check --scrape prometheus.yml --json # probe static targets +scripts/telemetry-check --targets targets.txt --timeout 5 +``` + +Exit codes: 0 all checks passed, 1 issues found or a fatal error, 2 usage error. The rule checks mirror promtool: exactly one of `record`/`alert` per rule, a non-empty expression with balanced delimiters, valid durations, recording-rule and label names, and string-only label values. Use `promtool check rules` for full PromQL parsing. + +## Operating loop + +1. **Identify the deployment**: which components are in scope (Prometheus, OTel Collector, Loki), how they are deployed (binary, container, operator), where configs live, and who owns them. +2. **Collect evidence**: run `telemetry-check --rules` and `--scrape` on the configs, then check the live status endpoints (`/-/healthy`, `/api/v1/targets`, collector health, Loki ready) where access exists. +3. **Triage against the symptom**: map the reported problem to the evidence (missing series → scrape or relabeling; alert not firing → rule or retention; logs missing → ingest or label cardinality). +4. **Act with confirmation**: bounded, scoped mutations after a human directive, with a rollback path named first. +5. **Verify**: re-run the relevant check and confirm the observable at the delivery boundary. + +## Prometheus: scrape, rules, relabeling, retention, HA + +- **Scrape config** (`scrape_configs`): one job per scrape group with a deliberate `scrape_interval`, `scrape_timeout` below it, and `metrics_path`. Prefer `static_configs` for known endpoints and service discovery (`*_sd_configs`) for dynamic ones. Verify the running config with `/api/v1/status/config` and targets with `/api/v1/targets?state=active`. +- **Recording and alerting rules**: rules files are `groups` of `record` or `alert` rules with a PromQL `expr`, optional `for`/`keep_firing_for` durations, and `labels`/`annotations`. Validate every change with `promtool check rules` and with the bundled `telemetry-check --rules` before reload. Rules must be small, well-named, and reviewable — a 100-line expression is a debugging liability, not a rule. +- **Relabeling**: `relabel_configs` and `metric_relabel_configs` rewrite labels before ingestion. Use them to enforce label naming, drop high-cardinality or internal labels, and attach scrape metadata. Relabeling mistakes silently change series identity — verify with a targeted `curl` of `/metrics` and the target's `scrapeUrl` in `/api/v1/targets`. +- **Retention**: `--storage.tsdb.retention.time` and `--storage.tsdb.retention.size` bound local block retention; blocks are 2h by default. Retention is a capacity decision (see `references/04-stack-integration-and-retention.md`), not a default to leave alone. Watch `prometheus_tsdb_head_series` and `prometheus_tsdb_compaction` for cardinality and compaction pressure. +- **High availability (HA)**: two identically configured Prometheus instances with `--query.max-concurrency` headroom and consistent external labels let you shard or deduplicate at the query layer (Thanos, Mimir, or Grafana data sources). Alerting rules must not double-fire: HA pairs need a dedup layer or consistent labeling, and rule evaluation must stay consistent across replicas. Rule evaluation state (`for` counters) is local to each instance. + +## OpenTelemetry Collector: pipeline, sampling, correlation + +- **Collector pipeline**: a pipeline is a directed acyclic chain of `receivers` → `processors` → `exporters` per signal type (metrics, logs, traces). Keep pipelines narrow and per-signal; a pipeline that mixes signals becomes un-debuggable. Each pipeline must have at least one exporter; unused receivers/exporters are dead configuration. +- **Receivers, processors, exporters**: receivers accept data (OTLP, Prometheus, filelog, hostmetrics); processors transform, batch, filter, sample, and attach resource attributes; exporters send data onward (OTLP, Prometheus remote write, Loki, logging). Order matters — batching and the `memory_limiter` processor belong before exporters; `tail_sampling` belongs on trace pipelines only. +- **Sampling**: `tail_sampling` on traces decides at the batch level; `probabilistic_sampler` is stateless and cheaper. Sample deliberately: full traces for errors and slow paths, tail sampling for high-volume success traffic, and never sample away the error signal. Sampling must be coordinated with retention — a sampled trace is gone forever, so the decision belongs in the pipeline design, not in an emergency. +- **Trace/span correlation**: carry `trace_id` and `span_id` in log lines and metric exemplars so LogQL and PromQL can pivot back to the trace. The collector's `spanmetrics` processor derives RED metrics from spans, and OTLP logs with trace context land in Loki with `trace_id` as a structured label for correlation. Trace context propagation is an application-level concern that [backend-engineering](../backend-engineering/SKILL.md) owns; the collector side is here. + +## Loki: ingest, LogQL, retention, labels + +- **Ingest**: Loki ingests over the push API (`/loki/api/v1/push`) from Promtail, the OTel Collector's `loki` exporter, or the Grafana Agent/Alloy. Verify ingest with `loki_distributor_bytes_received_total` and the ready endpoint; an ingest that silently drops (rate limits, `too many outstanding requests`) hides outages. +- **LogQL**: `{label="value"} |= "filter" | json` selects streams and filters lines; label matchers are the primary cost driver. LogQL analytics (`sum by (...) (rate({app="x"} |~ "error"[5m]))`) work on the label index plus line filtering — design labels so the matchers you actually use are cheap. +- **Retention**: `retention_period` and `retention_size` apply per tenant; the compactor enforces them and merges index shards. Log volume is unbounded if ungoverned — set retention before rollout, track it with `loki_compactor` metrics, and treat log retention as a compliance decision with an owner. +- **Labels**: Loki labels are inverted indexes — high-cardinality labels (request IDs, user IDs, trace IDs) explode index size and streaming cost. Keep labels to tenant, app, environment, and job; put high-cardinality fields in the log line and extract them with LogQL `| json`/`| regexp` or OTel structured metadata. Cardinality guidance: a label whose values change with every log line does not belong in the index. + +## Retention across the stack + +Retention is a stack-wide decision: Prometheus blocks (raw samples), OTel Collector buffering (in-memory queue, exporter retries), and Loki (indexed logs) each have independent retention, and the *combined* storage footprint is what the team pays for. Decide per component based on the question the data answers (hot metrics for alerting, samples for trends, logs for debugging and audit), set it in config, and review it on a schedule. See `references/04-stack-integration-and-retention.md` for the trade-off tables and the alerting rule that watches retention. + +## Reference routing + +| Load when | Reference | +|---|---| +| Sources, version observations, refresh procedure | `references/00-source-index.md` | +| Scrape config, recording/alerting rules, relabeling, retention, HA | `references/01-prometheus-operations.md` | +| Collector pipelines, receivers/processors/exporters, sampling, correlation | `references/02-opentelemetry-collector.md` | +| Ingest, LogQL, retention, label design | `references/03-loki-operations.md` | +| Cross-component retention decisions and stack integration | `references/04-stack-integration-and-retention.md` | + +## Included artifacts + +- `scripts/telemetry-check`: read-only rule sanity + scrape-target reachability checker (stdlib-only, `--json`, `--rules`/`--scrape`/`--targets`, `--help` without a server). +- `tests/test_telemetry_check.py`: deterministic tests against fixture configs, including the read-only contract. +- `fixtures/`: `prometheus-rules.yml` (valid rules) and `scrape-config.yml` (valid scrape config) used by the tests and as starting points. +- `references/`: four dated, source-indexed references plus the source index. +- `evals/evals.json`: six output-quality evaluation cases for agent runs. + +## Verification boundary + +| Claim | Minimum evidence | +|---|---| +| A rules file is structurally sound | `telemetry-check --rules FILE --json` exits 0 with no errors | +| A rules file is semantically valid | `promtool check rules FILE` exits 0 | +| A target is scrapable | `telemetry-check --scrape CONFIG --json` reports it reachable, and `/api/v1/targets` shows `state="up"` | +| A pipeline is live | Collector health endpoint responds and per-signal metrics (`otelcol_receiver_*`, `otelcol_exporter_*`) advance | +| Ingest is healthy | Distributor metrics advance and the ready endpoint returns 200 | +| Retention is governed | `retention_period`/`retention_size` are set explicitly, and compactor/TSDB metrics confirm the policy | + +## Hard boundaries + +- Never mutate a scrape config, rules file, collector pipeline, or retention policy without an explicit human directive naming the target and a stated rollback path. Read-only discovery may proceed freely. +- Never claim a rule or target works without delivery-boundary evidence: a scrape that succeeded, a rule that evaluated, an alert that fired. +- Never expose full configs, credentials, or raw logs in chat; summarize evidence instead. +- Never run `telemetry-check` as anything but what it is — read-only. It has no mutation surface. +- Dashboards, Grafana alert rules, contact points, and notification policies are [grafana](../grafana/SKILL.md) territory; SLI/SLO design and observability strategy are [platform-engineering](../platform-engineering/SKILL.md) territory. Do not duplicate their content here. + +## When not to use + +- **Observability strategy and SLOs** (what to instrument, SLI/SLO design, error budgets, paging policy) — that is [platform-engineering](../platform-engineering/SKILL.md). +- **Grafana product work** (dashboards, panels, data sources, Grafana alert rules, contact points, notification policies, RBAC) — that is [grafana](../grafana/SKILL.md); it queries Prometheus and Loki but owns the Grafana side. +- **Application instrumentation code** (OTel SDKs in services, trace context propagation, custom exporters in application code) — that is application development; see [backend-engineering](../backend-engineering/SKILL.md). +- **Reverse proxy and edge observability** (Traefik metrics/tracing/access-log config) — that is [traefik](../traefik/SKILL.md), whose observability reference treats this stack as its backend. +- **Infrastructure deployment of the stack** (Helm charts, Kubernetes operators, Docker Compose for the stack itself) — that is [kubernetes](../kubernetes/SKILL.md) and [docker-compose](../docker-compose/SKILL.md). +- **Other backends** (Tempo, Mimir, Thanos, Datadog, InfluxDB) — those stay with their owners; this skill covers Prometheus, the OTel Collector, and Loki as a unit. diff --git a/telemetry/evals/evals.json b/telemetry/evals/evals.json new file mode 100644 index 0000000..86631e6 --- /dev/null +++ b/telemetry/evals/evals.json @@ -0,0 +1,72 @@ +{ + "schema_version": 1, + "skill_name": "telemetry", + "evals": [ + { + "id": "prometheus-rule-authoring-review", + "prompt": "Our SRE wants to add two rules to a Prometheus rules file: a recording rule that computes the 5-minute request rate per endpoint and an alert that pages when the API error rate exceeds 5% for ten minutes. What should the rule file look like, what sanity checks should run before it is loaded, and what are the common authoring mistakes to avoid?", + "expected_output": "A rules file with one group containing two rules: a recording rule named with the level:metric:operation convention (for example job:http_requests:rate5m) with expr sum by (job, endpoint) (rate(http_requests_total[5m])), and an alerting rule (for example ApiHighErrorRate) with the error-rate expression, a for: 10m clause, labels such as severity and team, and annotations with a summary and a runbook link. The response states that each rule must set exactly one of record or alert, expr must be present and parse, durations must be valid Prometheus durations, and label values must be strings. It validates with promtool check rules for full PromQL parsing and the bundled telemetry-check --rules for structural sanity before reload, then verifies with the /api/v1/rules endpoint. Common mistakes called out: putting both record and alert on one rule, writing an expression that is too large to review, misusing relabeling so the labels the rule queries do not exist, and forgetting for on alerts that should require sustained conditions.", + "assertions": [ + "The response produces a rules file with a recording rule using the level:metric:operation naming convention and an alerting rule with expr, for, labels, and annotations", + "Exactly one of record or alert per rule, a non-empty expr, and valid Prometheus durations are stated as requirements", + "Validation with promtool check rules and telemetry-check --rules before reload is prescribed, with verification via the rules API", + "At least three authoring mistakes are named (both record and alert set, oversized expressions, relabeling/label mismatches, missing for)" + ] + }, + { + "id": "otel-collector-pipeline-design", + "prompt": "We are standing up an OpenTelemetry Collector that receives OTLP traces and metrics from a few services and sends them to a backend. We want to sample 10% of success traces but keep all error traces, and we are worried about the collector using too much memory. How should the pipelines and processors be designed?", + "expected_output": "A pipeline design with separate traces and metrics pipelines: traces through otlp receiver, memory_limiter, batch, tail_sampling, then the otlp exporter; metrics through otlp receiver, memory_limiter, batch, then the metrics exporter. The response explains that tail_sampling decides per trace at the batch level, so it belongs on the trace pipeline after batching, with policies that keep spans whose status.code or http.status_code indicates an error and a probabilistic policy for the rest, and that the sampling decision should be recorded as a span attribute. It prescribes one memory_limiter processor before exporters with a limit sized against the container memory budget, batching to amortize exporter cost, and warns not to apply trace samplers to the metrics pipeline. It verifies with the collector health endpoint and receiver/exporter metrics advancing, and notes that a debug exporter is for temporary troubleshooting only.", + "assertions": [ + "Separate traces and metrics pipelines with receivers, processors, and exporters are specified", + "tail_sampling is placed on the trace pipeline after batching, with error-keeping and probabilistic policies", + "memory_limiter runs before exporters and is sized against the container memory budget", + "The response warns that trace samplers must not be applied to the metrics pipeline and verification uses collector health and per-signal metrics" + ] + }, + { + "id": "loki-label-and-retention-review", + "prompt": "Our team is about to ship a new service and wants to push its logs to Loki with labels for app, environment, tenant, user_id, and request_id so they can filter per user. They also have not set any retention. What is wrong with this plan and what should the label and retention design be?", + "expected_output": "A review that app, environment, and tenant are reasonable Loki labels because they are low-cardinality and used as index matchers, but user_id and request_id are high-cardinality per-line fields that explode the inverted index and stream count if indexed. The response puts user_id and request_id in the log line and extracts them with LogQL | json or | regexp when needed, and sets retention deliberately per tenant: retention_enabled true with a retention_period and retention_size, enforced by the compactor, with an owner and a review schedule. It explains the cost model: label matchers run against the inverted index (cheap), line filters and parsing run per line (expensive), and a label whose values change with every log line does not belong in the index. Verification uses loki_ingester_streams to confirm stream cardinality stays bounded and compactor metrics to confirm retention is running.", + "assertions": [ + "app/environment/tenant are accepted as low-cardinality labels while user_id and request_id are called out as high-cardinality index hazards", + "High-cardinality fields are moved into the log line and extracted with LogQL json or regexp parsing", + "Retention is set per tenant with retention_enabled, retention_period, retention_size, compactor enforcement, and an owner", + "The inverted-index versus per-line-filter cost model is explained and loki_ingester_streams is used as the cardinality signal" + ] + }, + { + "id": "prometheus-retention-and-ha-decision", + "prompt": "Our single Prometheus instance keeps growing: queries are fine, but the disk fills every few months and someone keeps raising the retention flag to keep more history. Management now wants a second instance so we are 'highly available'. How should we reason about retention and HA before adding machines?", + "expected_output": "A decision process that separates the questions: retention is a deliberate capacity and compliance choice (how long the data must answer which questions), not a flag to raise reactively; and HA is a redundancy choice (two identically configured instances scraping the same targets with consistent external labels so a query layer can deduplicate), which does not increase storage capacity or history length. The response prescribes setting retention by the question the data answers (alerting windows, trend analysis, audit), bounding it with retention.time and retention.size, watching prometheus_tsdb_head_series and compaction metrics for the actual cost, and routing long-term history to a separate store with its own owner instead of stretching the hot instance. It states that two replicas do not double history, do not share rule state, and need a dedup layer or consistent labeling so alerts do not double-fire, and that rule evaluation consistency across replicas matters more than uptime.", + "assertions": [ + "Retention is framed as a capacity and compliance decision driven by the questions the data must answer, not a reactive flag", + "HA is framed as redundant identical instances with consistent external labels and a dedup layer, explicitly not a storage or history increase", + "Retention flags and TSDB metrics for bounding cost are named, with long-term history routed to a separate store", + "The double-fire risk and rule-state locality of HA pairs are stated" + ] + }, + { + "id": "trace-span-correlation-setup", + "prompt": "We run the OpenTelemetry Collector and send OTLP logs, metrics, and traces to the backend. When an alert fires on a metric, the on-call engineer has to search logs by timestamp and guess which request was slow. What should we configure so a metric alert can pivot to the exact trace and its log lines?", + "expected_output": "A correlation setup: the spanmetrics processor derives RED metrics from spans with trace_id exemplars so PromQL histograms carry the trace ID of slow requests; OTLP log records carry trace_id and span_id and the collector's Loki exporter maps them to structured metadata so LogQL can filter {app=\"x\"} | trace_id=\"...\"; and resource attributes such as service.name and deployment.environment flow through all three signals as the join keys. The response explains that correlation depends on context propagation from the application SDKs, so a missing trace_id in logs usually means propagation is not wired, and prescribes verifying the pivot end-to-end with one query: fire a test request, find its trace ID in the metrics exemplar, and confirm the same ID appears in Loki. It notes that instrumenting application code and propagation are backend-engineering territory while the collector-side join is this skill's scope, and that changing the pipeline requires re-verifying the pivot query.", + "assertions": [ + "spanmetrics with trace_id exemplars and the Loki exporter mapping trace_id/span_id to structured metadata are specified", + "Resource attributes are identified as the cross-signal join keys", + "Dependency on application-side context propagation is stated, with a missing trace_id diagnosed as a propagation problem", + "Verification is an end-to-end pivot query from metric exemplar to trace to log lines" + ] + }, + { + "id": "stack-ingest-outage-diagnosis", + "prompt": "A dashboard panel shows no data for the last hour for one service, while other services are fine. The Prometheus scrape targets list shows the job as up, the OTel Collector is healthy, and Loki shows the service's logs. What is the evidence-ordered diagnosis, and what should we check at each layer before changing anything?", + "expected_output": "An evidence-ordered diagnosis that works from the symptom down: first confirm which layer lost data by checking each component's own signals — the Prometheus targets API for scrape health and the actual metric series (the job can be up while the metric is empty, which points at metric_relabel_configs dropping the series), collector receiver and exporter metrics for delivery, and the rules API for evaluation state; then check config-level causes: relabeling that renamed or dropped labels, a scrape config change that changed the series identity, or a recording rule whose expression no longer matches. The response keeps the diagnosis read-only (telemetry-check --rules and --scrape plus read-only API queries) and treats any config change as a mutation requiring confirmation with a rollback path. It explicitly avoids assuming correlation is causation, e.g. a slow query and missing data are separate evidence, and verifies any fix by re-running the checks and confirming the series appears at the delivery boundary.", + "assertions": [ + "The diagnosis works layer by layer from the symptom with each component's own signals (targets API, series existence, collector receiver/exporter metrics, rules API)", + "Config-level causes such as relabeling drops, series-identity changes, and rule-expression mismatches are considered", + "The diagnosis is kept read-only with the bundled checker and read-only API queries, and changes require confirmation with a rollback path", + "Correlation is not presented as causation and fixes are verified by re-running the checks" + ] + } + ] +} diff --git a/telemetry/fixtures/prometheus-rules.yml b/telemetry/fixtures/prometheus-rules.yml new file mode 100644 index 0000000..20d1693 --- /dev/null +++ b/telemetry/fixtures/prometheus-rules.yml @@ -0,0 +1,29 @@ +# Sample Prometheus recording and alerting rules used by telemetry-check. +# Mirrors the structure promtool check rules accepts: groups of recording and +# alerting rules with labels, annotations, and durations. +groups: + - name: api-slo + interval: 1m + rules: + - record: job:http_requests:rate5m + expr: sum by (job) (rate(http_requests_total{job="api"}[5m])) + - record: job:http_errors:rate5m + expr: sum by (job) (rate(http_requests_total{job="api",status=~"5.."}[5m])) + - alert: ApiHighErrorRate + expr: job:http_errors:rate5m / job:http_requests:rate5m > 0.05 + for: 10m + labels: + severity: page + team: platform + annotations: + summary: "API error rate above 5% for 10 minutes" + runbook: "https://example.com/runbooks/api-high-error-rate" + - name: node-health + rules: + - alert: InstanceDown + expr: up == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Instance {{ $labels.instance }} is down" diff --git a/telemetry/fixtures/scrape-config.yml b/telemetry/fixtures/scrape-config.yml new file mode 100644 index 0000000..183ff0d --- /dev/null +++ b/telemetry/fixtures/scrape-config.yml @@ -0,0 +1,12 @@ +# Sample Prometheus scrape configuration used by telemetry-check tests. +scrape_configs: + - job_name: node-exporter + scrape_interval: 30s + static_configs: + - targets: ["127.0.0.1:9100", "127.0.0.1:9111"] + labels: + env: dev + - job_name: api + static_configs: + - targets: + - "127.0.0.1:8080" diff --git a/telemetry/references/00-source-index.md b/telemetry/references/00-source-index.md new file mode 100644 index 0000000..8191068 --- /dev/null +++ b/telemetry/references/00-source-index.md @@ -0,0 +1,66 @@ +# Telemetry Operations — Source Index + +> **Last Updated:** 2026-08-03 + +This index tracks the authoritative sources behind the telemetry skill +(Prometheus + OpenTelemetry Collector + Loki as one stack) and the refresh +procedure for keeping it current. + +## Canonical sources + +| Component | Source | +|---|---| +| Prometheus documentation (current) | https://prometheus.io/docs/introduction/overview/ | +| Prometheus configuration (scrape config, relabeling) | https://prometheus.io/docs/prometheus/latest/configuration/configuration/ | +| Prometheus recording rules | https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/ | +| Prometheus alerting rules | https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/ | +| Prometheus storage and retention | https://prometheus.io/docs/prometheus/latest/storage/ | +| `promtool` rule checking | https://prometheus.io/docs/prometheus/latest/command-line/promtool/ | +| Prometheus rule format reference (rulefmt) | https://github.com/prometheus/prometheus/blob/main/model/rulefmt/rulefmt.go | +| OpenTelemetry Collector | https://opentelemetry.io/docs/collector/ | +| Collector components (receivers/processors/exporters) | https://opentelemetry.io/docs/collector/configuration/ | +| Collector sampling | https://opentelemetry.io/docs/collector/sampling/ | +| OpenTelemetry traces and spans | https://opentelemetry.io/docs/concepts/signals/traces/ | +| Loki documentation (current) | https://grafana.com/docs/loki/latest/ | +| Loki storage and retention | https://grafana.com/docs/loki/latest/operations/storage/retention/ | +| LogQL | https://grafana.com/docs/loki/latest/query/ | +| Loki label design guidance | https://grafana.com/docs/loki/latest/get-started/labels/ | + +## Version observations (as of this refresh) + +- Prometheus 3.x defaults to UTF-8 metric-name validation; the legacy metric + name pattern `[a-zA-Z_:][a-zA-Z0-9_:]*` remains the documented pattern for + recording rule names and is what `promtool` enforces under legacy validation. +- `promtool check rules` validates rule files structurally and parses every + expression with the full PromQL parser; `telemetry-check` deliberately + covers the structural subset so it can run with no Prometheus tooling. +- The OpenTelemetry Collector's `memory_limiter` processor is recommended + before every exporter; `tail_sampling` and `probabilistic_sampler` are the + two supported sampler processors for traces. +- Loki retention is enforced by the compactor on a per-tenant basis; + `retention_period` and `retention_size` are per-tenant limits, and + `retention_enabled` must be `true` for period-based retention. +- OTLP is the current stable protocol; the Collector can also receive/export + Prometheus exposition format, and its Loki exporter translates structured + log records into LogQL-compatible streams. + +## Refresh procedure + +1. Re-check the sources above for new minor or major releases. +2. Update the version observations that changed (defaults, renamed components, + new processors, changed retention semantics). +3. Re-run the bundled checker against a rules fixture and a scrape config and + confirm every check still parses: `telemetry/scripts/telemetry-check + --rules telemetry/fixtures/prometheus-rules.yml --json`. +4. Re-verify the SKILL.md keyword sweep from the validation contract and the + routing links to `platform-engineering` and `grafana`. + +## Related skill sources + +- `platform-engineering` owns observability strategy (SLIs, SLOs, what to + instrument) and its observability reference treats Prometheus, OTel, and + Loki as one stack; this skill owns operating that stack. +- `grafana` owns dashboards, panels, and Grafana-side alerting; it queries + Prometheus (PromQL) and Loki (LogQL) but does not operate the backends. +- `traefik` ships Prometheus and OTel configuration for the edge; its + observability reference documents the metric names this stack ingests. diff --git a/telemetry/references/01-prometheus-operations.md b/telemetry/references/01-prometheus-operations.md new file mode 100644 index 0000000..750a268 --- /dev/null +++ b/telemetry/references/01-prometheus-operations.md @@ -0,0 +1,112 @@ +# Prometheus Operations + +> **Last Updated:** 2026-08-03 + +Operational patterns for the Prometheus half of the telemetry stack: scrape +configuration, recording and alerting rules, relabeling, retention, and high +availability. Sources: Prometheus documentation (prometheus.io/docs, +accessed 2026-08-03) and the rule format reference in the Prometheus source +(reviewed 2026-08-03). + +## Scrape configuration + +One scrape job is one scrape group: a `job_name`, a `scrape_interval`, a +`scrape_timeout` strictly below the interval, a `metrics_path`, and a target +source. The default `metrics_path` is `/metrics`; TLS and auth go in +`scheme`, `tls_config`, and `basic_authorization`/`authorization`. + +```yaml +scrape_configs: + - job_name: node + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + static_configs: + - targets: ["node1:9100", "node2:9100"] +``` + +Verify the live state, not the file: `/api/v1/status/config` returns the +effective config and `/api/v1/targets?state=active` returns per-target scrape +state. A target that never appears in the target list is usually a relabeling +or discovery problem, not a Prometheus outage. + +## Recording and alerting rules + +Rules files are `groups`, each with a `name`, an optional `interval`, and a +`rules` list. Every rule has exactly one of `record` or `alert` plus an +`expr`; alerting rules may add `for`, `keep_firing_for`, `labels`, and +`annotations`. Group names must be unique within a file; label and annotation +names must be valid label names; label values must be strings. + +```yaml +groups: + - name: api-slo + interval: 1m + rules: + - record: job:http_requests:rate5m + expr: sum by (job) (rate(http_requests_total{job="api"}[5m])) + - alert: ApiHighErrorRate + expr: job:http_errors:rate5m / job:http_requests:rate5m > 0.05 + for: 10m + labels: + severity: page + annotations: + summary: "API error rate above 5%" +``` + +Rules that touch the same series belong in one group because groups evaluate +sequentially; cross-group timing is undefined. Validate with +`promtool check rules` (full PromQL parsing) and `telemetry-check --rules` +(structural sanity, no external tools) before every reload, then +`promtool reload` via `POST /-/reload` and confirm with +`/api/v1/rules?type=alert`. + +Recording rules are caching, not aggregation religion: name them with the +conventional `level:metric:operation` style, keep them idempotent, and prefer +`sum`/`rate` over `count`-style ratios that need division in every query. +Alerting rules should be small and reviewable; a rule whose expression needs +a comment to explain is a candidate for a recording rule instead. + +## Relabeling + +`relabel_configs` run at target discovery time (before scraping) and +`metric_relabel_configs` run after scraping (per metric). Use them to: + +- enforce label naming and drop forbidden labels (`__meta_*`, `job`); +- add scrape metadata (`__address__`, `__scheme__`, `__metrics_path__`); +- drop high-cardinality labels from `metric_relabel_configs` before the TSDB. + +Relabeling is the classic silent-breakage point: a dropped or renamed label +changes series identity without an error. Verify with the target's effective +labels in `/api/v1/targets` and a spot-check of `/metrics` on the endpoint. +`keep`/`drop`/`replace`/`labelmap` are the operators you will actually use; +`regex` capture groups feed `replacement` with `$1`-style references. + +## Retention + +Local retention is `--storage.tsdb.retention.time` (age) and +`--storage.tsdb.retention.size` (bytes); blocks are ~2h and compaction merges +them. Set retention as a deliberate capacity decision (see +`references/04-stack-integration-and-retention.md`), never leave the defaults +for a long-running server. Watch `prometheus_tsdb_head_series` (cardinality), +`prometheus_tsdb_compactions_total` and `prometheus_tsdb_blocks_loaded` for +compaction health, and `prometheus_tsdb_storage_blocks_bytes` for the +footprint. Retention is enforced lazily by compaction — a server under +compaction pressure can exceed its retention window temporarily. + +## High availability (HA) + +HA for Prometheus means two identical instances scraping the same targets, +with the same rules, and consistent external labels, so a query layer can +deduplicate or shard. The instances do not share state: each has its own TSDB, +its own `for`-counter state, and its own alert evaluation. Practical rules: + +- run both replicas with `--web.external-url` stable and identical rule files; +- give replicas distinct `replica` external labels so dedup can pick one; +- never add alert-specific noise that makes the two replicas fire different + alert instances — dedup is by label sets; +- consider Thanos or Mimir for query federation and long-term retention, but + only after the two-replica story is correct. + +Rule evaluation correctness across replicas matters more than uptime: a +failover that changes when alerts fire is worse than a brief scrape gap. diff --git a/telemetry/references/02-opentelemetry-collector.md b/telemetry/references/02-opentelemetry-collector.md new file mode 100644 index 0000000..d0f013c --- /dev/null +++ b/telemetry/references/02-opentelemetry-collector.md @@ -0,0 +1,91 @@ +# OpenTelemetry Collector Operations + +> **Last Updated:** 2026-08-03 + +Operational patterns for the OpenTelemetry Collector in the telemetry stack: +pipeline design, receivers/processors/exporters, sampling, and trace/span +correlation. Sources: OpenTelemetry Collector documentation +(opentelemetry.io/docs/collector, accessed 2026-08-03). + +## Pipeline design + +A pipeline is a named, directed acyclic chain per signal type (metrics, logs, +traces): one or more `receivers`, zero or more `processors`, one or more +`exporters`. Signals flow through every processor in order, so pipeline length +is a cost and a debugging surface. + +```yaml +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch, tail_sampling] + exporters: [otlp/backend] + metrics: + receivers: [otlp, prometheus] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [loki] +``` + +Design rules: + +- keep pipelines per signal — a pipeline that mixes traces and logs becomes + un-debuggable and couples sampling decisions; +- run one `memory_limiter` processor before every exporter to bound memory + (`ballast_size_mib` is deprecated; size `check_interval`/`limit_mib` against + the container limit); +- `batch` after sampling and before exporting amortizes exporter cost; +- an unused receiver or exporter is dead configuration — remove it. + +## Receivers, processors, exporters + +| Role | Examples | Notes | +|---|---|---| +| Receiver | `otlp` (default port 4317 gRPC / 4318 HTTP), `prometheus`, `filelog`, `hostmetrics` | Receivers own the ingest surface; auth and TLS live here | +| Processor | `memory_limiter`, `batch`, `tail_sampling`, `probabilistic_sampler`, `resource`, `attributes`, `filter`, `transform`, `spanmetrics` | Order matters: sampling before batch changes semantics; `resource` should run early | +| Exporter | `otlp`, `prometheusremotewrite`, `loki`, `logging`/`debug`, `kafka` | Exporters own delivery and retry; failing exporters backpressure the pipeline | + +The `logging`/`debug` exporter is the troubleshooting tool: attach it to a +pipeline temporarily to see what actually leaves the collector, then remove +it. Never ship a debug exporter in production config. + +## Sampling + +Two trace samplers ship with the collector: + +- `probabilistic_sampler` — stateless, per-span hash of the trace ID, cheap, + no cross-span state. Good for high-volume, low-value traffic. +- `tail_sampling` — buffers spans per trace and decides at the batch level, + so policies can depend on span attributes (status, error, duration). Stateful + and memory-hungry; belongs on a trace pipeline after `batch`. + +Sample with the alerting and debugging goal in mind: keep every error and slow +path (via `tail_sampling` policy on `status.code`, `http.status_code`, +duration), sample the long tail of success traffic, and record the sampling +decision (`sampler.type`/`sampling.score`) as a span attribute so downstream +queries can scale results. Metrics and logs are not sampled by these processors +— do not apply trace samplers to other signal pipelines. + +## Trace/span correlation + +Correlation is the payoff of the stack: a trace ID in a log line or an +exemplar lets any query pivot from "this happened" to "here is the full +request". The collector supports this by: + +- carrying `trace_id`/`span_id` through OTLP log records — the `loki` exporter + maps them to `trace_id`/`span_id` structured metadata for LogQL matching; +- the `spanmetrics` processor deriving RED (rate/errors/duration) metrics from + spans, with `trace_id` exemplars on the histogram so PromQL can jump to a + trace; +- resource attributes (`service.name`, `deployment.environment`) flowing + through to every signal so logs, metrics, and traces share the join keys. + +Correlation is only as good as propagation: if the application does not +propagate context, the collector cannot invent it. Missing trace context in +logs usually means the SDK side is not wired — that is application-level work +for [backend-engineering](../backend-engineering/SKILL.md); the collector side +of the join is this skill. diff --git a/telemetry/references/03-loki-operations.md b/telemetry/references/03-loki-operations.md new file mode 100644 index 0000000..222d1f9 --- /dev/null +++ b/telemetry/references/03-loki-operations.md @@ -0,0 +1,81 @@ +# Loki Operations + +> **Last Updated:** 2026-08-03 + +Operational patterns for the Loki half of the telemetry stack: ingest, LogQL, +retention, and label design. Sources: Grafana Loki documentation +(grafana.com/docs/loki, accessed 2026-08-03). + +## Ingest + +Loki ingests via the push API (`POST /loki/api/v1/push`) from Promtail, the +OpenTelemetry Collector `loki` exporter, Grafana Alloy/Agent, or the SDKs. +Ingest health is distributor-side: + +- `loki_distributor_bytes_received_total` and + `loki_distributor_lines_received_total` must advance per tenant; +- `loki_ingester_streams` shows active streams — a flat line here while + producers push means a config or network problem; +- the ready endpoint (`/ready`) must return 200; `429 Too Many Requests` from + the distributor means rate limits — the producer retries, but a sustained + backlog is a capacity signal. + +The OTel Collector `loki` exporter maps log records to streams: the +`loki.tenant` and `loki.format` attributes plus configured labels control +stream cardinality (see Labels below). Promtail is the file-tailer option; +choose one producer per source and document it. + +## LogQL + +LogQL has two layers: stream selectors and pipeline expressions. + +- Selectors choose streams by label: `{app="api", env="prod"}`. Label + matchers are evaluated against the inverted index — this is the cheap part, + and it is why label design matters. +- Pipeline expressions filter and transform lines: `|= "error"`, + `|~ "5[0-9][0-9]"`, `| json`, `| regexp "(?P...)"`, `| line_format`. + These run per line after selection — the expensive part. + +Metric queries wrap the pipeline in `rate`, `count_over_time`, etc.: +`sum by (app) (rate({job="api"} |~ "error"[5m]))`. When a query is slow, +the cause is almost always a too-broad selector (too many streams) or +regexp/JSON parsing on every line; fix the selector or pre-extract fields at +ingest, not by writing a faster regexp. + +## Retention + +Loki retention is per-tenant, enforced by the compactor: + +- `retention_enabled: true` in the limits config enables period-based + retention; `retention_period` sets the age limit and `retention_size` the + byte limit per tenant; +- the compactor deletes expired chunks and merges index shards; watch + `loki_compactor_delete_requests_total` and + `loki_compactor_compactor_running` to confirm it is actually running; +- retention applies at query and delete time, so expired data can still be + counted until compaction finishes — size-based limits are the practical + control for runaway log volume. + +Set retention before rollout and treat it as a compliance decision with an +owner (see `references/04-stack-integration-and-retention.md`). A Loki with +default retention and no owner will silently grow until storage is the +incident. + +## Labels + +Loki labels are an inverted index — every distinct label value adds index +entries and stream overhead. The guidance is deliberately simple: + +- keep labels to tenant, app, environment, job, and a handful of + service-defined dimensions; +- never index high-cardinality fields: request IDs, user IDs, trace IDs, + IPs, timestamps, or any field whose values change per log line; +- put high-cardinality data in the log line itself and extract it with LogQL + `| json`/`| regexp`, or as OTel structured metadata, when you need it; +- a good rule of thumb: if a label's values exceed the low hundreds of + distinct values, it is a line field, not a label. + +Cardinality damage is silent and cumulative: `loki_ingester_streams` +climbing while label changes are "small" is the leading signal. Stream +sharding and `chunk_target_size` tune large-stream handling, but they do not +make a bad label design good. diff --git a/telemetry/references/04-stack-integration-and-retention.md b/telemetry/references/04-stack-integration-and-retention.md new file mode 100644 index 0000000..0d15976 --- /dev/null +++ b/telemetry/references/04-stack-integration-and-retention.md @@ -0,0 +1,97 @@ +# Stack Integration and Retention + +> **Last Updated:** 2026-08-03 + +How the three components of the telemetry stack fit together, and how to make +retention decisions across them instead of per-component by accident. Sources: +Prometheus storage documentation, OpenTelemetry Collector documentation, and +Loki retention documentation (all accessed 2026-08-03). + +## The stack as one unit + +The components share a data flow — instrumented services emit metrics, logs, +and traces; the OpenTelemetry Collector (or direct exporters) delivers them; +Prometheus stores metrics and evaluates rules; Loki stores logs; Grafana +queries both and displays the result. The stack is operated as one unit +because a change in any layer changes the behavior of every layer above it: + +| Layer | Component | Owns | +|---|---|---| +| Collection | OTel Collector receivers, Prometheus scrape jobs | What data enters the stack | +| Processing | Collector processors (batch, sample, resource) | What the data looks like when stored | +| Storage | Prometheus TSDB, Loki chunks/index | How long data lives and how fast queries are | +| Evaluation | Prometheus rules | What the data means (alerts, recording rules) | +| Presentation | Grafana | What humans see (dashboards, Grafana-side alerting) | + +Operational decisions therefore cross component boundaries: a label added in +relabeling is a label PromQL sees; a `trace_id` carried by the collector is a +LogQL matcher; a retention window chosen for Prometheus is a gap in history a +dashboards-as-code change cannot recover. + +## Retention as a stack decision + +Each component has independent retention, and the combined footprint is what +the team pays for: + +| Component | Setting | Applies to | Default behavior | +|---|---|---|---| +| Prometheus | `--storage.tsdb.retention.time` / `.size` | Raw samples (blocks) | 15 days; lazily enforced by compaction | +| OTel Collector | exporter queue + `memory_limiter` | In-flight data | Queue backs up then drops oldest on pressure | +| Loki | `retention_enabled`, `retention_period`, `retention_size` (per tenant) | Indexed log streams | No retention unless enabled; compactor enforces it | + +Decide per component by the question the data answers: + +- hot metrics for alerting: keep enough history to evaluate every rule window + plus headroom (a `for: 10m` rule needs at least that much history); +- samples for trends: retention length is a capacity trade-off, not a + correctness requirement — long histories belong in a separate store + (Thanos/Mimir) with its own owner; +- logs for debugging and audit: retention is a compliance decision with an + owner; short retention on logs destroys the only evidence an incident + post-mortem can use. + +The failure mode is defaulting every component and discovering the cost when +storage is the incident. Write the decision down: `retention_period`, +retention flags, and the RPO/RTO framing belong in the deployment config and +its review checklist, not in tribal memory. + +## Cross-component consistency checks + +- Scrape config and collector receivers must agree on the metrics endpoint: + a `metric_relabel_configs` drop on Prometheus side does not stop the + collector from exporting the metric elsewhere. +- Rule expressions and recording rules must not depend on labels the scrape + config does not produce — validate with `telemetry-check --rules` plus a + live query, not by reading the config. +- LogQL matchers must match labels the `loki` exporter actually attaches — + a matcher on `app` fails silently when the collector maps it to + `service.name` only. +- Trace correlation needs `trace_id`/`span_id` in both the OTLP log records + and the metrics exemplars; verify one pivot query end-to-end after any + collector pipeline change. + +## Alerting on the stack itself + +The stack needs its own health rules (in a file you also validate with +`telemetry-check --rules`): + +- `up{job=~"prometheus|otel-collector|loki"}` == 0 for component loss; +- `prometheus_tsdb_head_series` growth vs a budget for cardinality; +- `rate(loki_distributor_lines_received_total[5m])` vs a floor to catch + silent ingest loss; +- `otelcol_exporter_send_failed_ratio` above a threshold for delivery loss; +- retention drift: `prometheus_tsdb_storage_blocks_bytes` and compactor + delete counters vs the decided policy. + +These rules live in this skill's scope (the Prometheus rules file); the +dashboards and Grafana-side alert routes for them are `grafana` territory. + +## Routing + +- Observability strategy — what to instrument, SLI/SLO design, error budgets — + is [platform-engineering](../platform-engineering/SKILL.md). +- Dashboards, panels, Grafana alert rules, contact points, and notification + policies are [grafana](../grafana/SKILL.md). +- This skill owns the collection/ingest/retention layer and the Prometheus + rules files those layers consume. When a task crosses into those skills, + route there instead of duplicating their content. diff --git a/telemetry/scripts/telemetry-check b/telemetry/scripts/telemetry-check new file mode 100755 index 0000000..6700424 --- /dev/null +++ b/telemetry/scripts/telemetry-check @@ -0,0 +1,782 @@ +#!/usr/bin/env python3 +"""telemetry-check - read-only Prometheus rule sanity and scrape-target reachability checker. + +Checks the Prometheus side of the Prometheus + OpenTelemetry + Loki telemetry +stack before it is deployed or changed: + + * --rules FILE parses a Prometheus recording/alerting rules file with a + minimal YAML reader (stdlib only) and runs sanity checks + mirroring promtool: group name uniqueness, exactly one of + record/alert per rule, a non-empty expr, balanced + delimiters in the expression, valid durations, recording + rule names and label names, and string-only label values. + * --scrape FILE parses a Prometheus scrape configuration (scrape_configs), + extracts static targets, and probes TCP reachability of + each target. + * --targets FILE probes a plain host:port list (one per line) or a YAML/JSON + list of targets. + +The tool is strictly read-only: it only reads local files and opens short TCP +connect probes. It never mutates configuration, never writes files, and never +sends telemetry data anywhere. Full PromQL parsing belongs to promtool; this +script provides structural sanity that catches the common classes of errors +before a rules file reaches a Prometheus server. + +Exit codes: 0 all checks passed (warnings allowed), 1 issues found or a fatal +error, 2 usage error. +""" +import argparse +import datetime +import json +import re +import socket +import sys +from typing import Any, Dict, List, Optional, Tuple + +TOOL_NAME = "telemetry-check" +TOOL_VERSION = "1.0.0" + +# --- minimal YAML reader (block mappings, sequences, flow collections, --- +# --- plain/quoted scalars, block scalars, comments) --- + +BLOCK_SCALAR_MARKERS = ("|", ">", "|-", ">-", "|+", ">+") +YAML_RESERVED_STARTS = ("&", "*", "!", "%", "@", "`") + + +class YAMLError(ValueError): + """Raised when a file is not valid YAML within the supported subset.""" + + +class Scalar: + """A YAML scalar leaf; tracks whether the source value was quoted.""" + + __slots__ = ("value", "quoted") + + def __init__(self, value: str, quoted: bool = False) -> None: + self.value = value + self.quoted = quoted + + def __repr__(self) -> str: + return "Scalar(%r)" % (self.value,) + + +def _split_comment(line: str) -> str: + """Return the line with any unquoted # comment removed.""" + in_single = False + in_double = False + for index, char in enumerate(line): + if char == "'" and not in_double: + in_single = not in_single + elif char == '"' and not in_single: + in_double = not in_double + elif char == "#" and not in_single and not in_double: + return line[:index] + return line + + +def _split_key_value(content: str) -> Tuple[Optional[str], Optional[str]]: + """Split 'key: value' on the first colon outside quotes and flow nesting.""" + in_single = False + in_double = False + depth = 0 + for index, char in enumerate(content): + if char in "[{": + depth += 1 + elif char in "]}": + depth = max(0, depth - 1) + elif char == "'" and not in_double: + in_single = not in_single + elif char == '"' and not in_single: + in_double = not in_double + elif char == ":" and depth == 0 and not in_single and not in_double: + if index + 1 == len(content) or content[index + 1] in " \t": + return content[:index].strip(), content[index + 1:].strip() + return None, None + + +def _split_flow(inner: str) -> List[str]: + """Split a flow collection body on commas outside quotes and nesting.""" + parts: List[str] = [] + start = 0 + in_single = False + in_double = False + depth = 0 + for index, char in enumerate(inner): + if char == "'" and not in_double: + in_single = not in_single + elif char == '"' and not in_single: + in_double = not in_double + elif char in "[{": + depth += 1 + elif char in "]}": + depth = max(0, depth - 1) + elif char == "," and depth == 0 and not in_single and not in_double: + parts.append(inner[start:index].strip()) + start = index + 1 + parts.append(inner[start:].strip()) + return [part for part in parts if part] + + +def _parse_scalar(raw: str) -> Scalar: + value = raw.strip() + quoted = False + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + quote = value[0] + inner = value[1:-1] + if quote == '"': + inner = inner.replace('\\"', '"').replace("\\\\", "\\") + elif quote == "'": + inner = inner.replace("''", "'") + return Scalar(inner, True) + return Scalar(value, False) + + +def _parse_flow(raw: str, line_no: int) -> Any: + """Parse a single-line flow mapping or sequence.""" + if raw.startswith("{") and raw.endswith("}"): + inner = raw[1:-1].strip() + result: Dict[str, Any] = {} + if inner: + for part in _split_flow(inner): + key, value = _split_key_value(part) + if key is None: + raise YAMLError("malformed flow mapping at line %d" % line_no) + result[key] = _parse_scalar(value) + return result + if raw.startswith("[") and raw.endswith("]"): + inner = raw[1:-1].strip() + if not inner: + return [] + result = [] + for part in _split_flow(inner): + if part.startswith("{") or part.startswith("["): + result.append(_parse_flow(part, line_no)) + else: + result.append(_parse_scalar(part)) + return result + raise YAMLError("malformed flow collection at line %d" % (line_no + 1)) + + +def _tokenize(text: str) -> List[Tuple[int, str]]: + lines: List[Tuple[int, str]] = [] + first_content = True + for raw in text.splitlines(): + content = _split_comment(raw).rstrip() + if not content.strip(): + continue + indent = len(content) - len(content.lstrip(" ")) + prefix = content[:indent] + if "\t" in prefix: + raise YAMLError("tab indentation is not supported (line %d)" % (len(lines) + 1)) + body = content.strip() + if body == "---": + if first_content: + continue + raise YAMLError("multi-document YAML is not supported (line %d)" % (len(lines) + 1)) + if body == "...": + raise YAMLError("unexpected document end marker (line %d)" % (len(lines) + 1)) + if body.startswith(YAML_RESERVED_STARTS): + raise YAMLError( + "YAML tags, anchors, aliases, and directives are not supported (line %d)" + % (len(lines) + 1) + ) + lines.append((indent, body)) + first_content = False + return lines + + +def _parse_block_scalar( + lines: List[Tuple[int, str]], index: int, parent_indent: int +) -> Tuple[Scalar, int]: + marker = lines[index][1].split(":", 1)[1].strip() + fold = marker.startswith(">") + index += 1 + parts: List[str] = [] + while index < len(lines) and lines[index][0] > parent_indent: + parts.append(lines[index][1]) + index += 1 + if fold: + return Scalar(" ".join(parts)), index + return Scalar("\n".join(parts)), index + + +def _parse_value( + lines: List[Tuple[int, str]], index: int, parent_indent: int, raw: str, line_no: int +) -> Tuple[Any, int]: + """Parse the value of 'key: ' at lines[index]; returns (value, index).""" + if raw in BLOCK_SCALAR_MARKERS: + return _parse_block_scalar(lines, index, parent_indent) + if raw == "": + if index + 1 < len(lines) and lines[index + 1][0] > parent_indent: + return _parse_node(lines, index + 1) + return None, index + if raw.startswith("{") or raw.startswith("["): + return _parse_flow(raw, line_no), index + 1 + return _parse_scalar(raw), index + 1 + + +def _parse_mapping( + lines: List[Tuple[int, str]], index: int, indent: int +) -> Tuple[Dict[str, Any], int]: + mapping: Dict[str, Any] = {} + while index < len(lines): + cur_indent, content = lines[index] + if cur_indent < indent: + break + if cur_indent > indent: + raise YAMLError("bad indentation: unexpected block at line %d" % (index + 1)) + if content.startswith("- "): + raise YAMLError("expected a mapping key at line %d" % (index + 1)) + key, raw = _split_key_value(content) + if key is None: + raise YAMLError("expected 'key: value' at line %d" % (index + 1)) + if key in mapping: + raise YAMLError("duplicate key %r at line %d" % (key, index + 1)) + mapping[key], index = _parse_value(lines, index, indent, raw, index) + return mapping, index + + +def _parse_sequence( + lines: List[Tuple[int, str]], index: int, indent: int +) -> Tuple[List[Any], int]: + items: List[Any] = [] + while index < len(lines): + cur_indent, content = lines[index] + if cur_indent != indent or not content.startswith("- "): + break + rest = content[2:].strip() + if rest == "": + if index + 1 >= len(lines) or lines[index + 1][0] <= indent: + raise YAMLError("empty sequence item at line %d" % (index + 1)) + item, index = _parse_node(lines, index + 1) + items.append(item) + continue + key, raw = _split_key_value(rest) + if key is None: + items.append(_parse_scalar(rest)) + index += 1 + continue + item: Dict[str, Any] = {} + item[key], index = _parse_value(lines, index, indent + 2, raw, index) + if index < len(lines) and lines[index][0] > indent: + key_indent = lines[index][0] + if key_indent != indent + 2: + raise YAMLError( + "bad indentation of mapping key at line %d" % (index + 1) + ) + extra, index = _parse_mapping(lines, index, key_indent) + item.update(extra) + items.append(item) + return items, index + + +def _parse_node( + lines: List[Tuple[int, str]], index: int +) -> Tuple[Any, int]: + indent, content = lines[index] + if content.startswith("- "): + return _parse_sequence(lines, index, indent) + return _parse_mapping(lines, index, indent) + + +def parse_yaml(text: str) -> Any: + """Parse a YAML document from the supported subset; raises YAMLError.""" + lines = _tokenize(text) + if not lines: + return {} + node, index = _parse_node(lines, 0) + if index != len(lines): + raise YAMLError("unexpected content after the document (line %d)" % (index + 1)) + return node + + +# --- Prometheus rule sanity checks (mirror promtool check rules) --- + +RECORD_NAME_RE = re.compile(r"^[a-zA-Z_:][a-zA-Z0-9_:]*$") +LABEL_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") +DURATION_RE = re.compile( + r"^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?" + r"(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$" +) +NON_STRING_SCALAR_RE = re.compile( + r"^[-+]?(\d+\.?\d*|\.\d+)([eE][-+]?\d+)?$|^(true|false|null|~)$", re.IGNORECASE +) +GROUP_KEYS = {"name", "interval", "query_offset", "limit", "rules", "labels"} +RULE_KEYS = {"record", "alert", "expr", "for", "keep_firing_for", "labels", "annotations"} +OPEN_TO_CLOSE = {"(": ")", "[": "]", "{": "}"} +CLOSERS = set(OPEN_TO_CLOSE.values()) + + +def as_str(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, Scalar): + return value.value + return str(value) + + +def balanced_delimiters(expr: str) -> bool: + stack: List[str] = [] + in_single = False + in_double = False + for char in expr: + if char == "'" and not in_double: + in_single = not in_single + elif char == '"' and not in_single: + in_double = not in_double + elif not in_single and not in_double: + if char in OPEN_TO_CLOSE: + stack.append(OPEN_TO_CLOSE[char]) + elif char in CLOSERS: + if not stack or stack.pop() != char: + return False + return not stack + + +def check_duration(value: Optional[str], errors: List[str], context: str) -> None: + if value is None: + return + value = value.strip() + if value == "": + return + if not DURATION_RE.match(value): + errors.append( + "%s: invalid duration %r (expected a Prometheus duration such as " + "5m, 1h, 30s, 1h30m, or 24h)" % (context, value) + ) + + +def check_labels(labels: Any, errors: List[str], context: str) -> None: + if labels is None: + return + if not isinstance(labels, dict): + errors.append("%s must be a mapping" % context) + return + for key, value in labels.items(): + if key == "__name__": + errors.append("%s: label name '__name__' is reserved" % context) + elif not LABEL_NAME_RE.match(key): + errors.append( + "%s: invalid label name %r (expected ^[a-zA-Z_][a-zA-Z0-9_]*$)" + % (context, key) + ) + if isinstance(value, Scalar) and not value.quoted: + if NON_STRING_SCALAR_RE.match(value.value): + errors.append( + "%s: label %r value %r is not a YAML string (quote it)" + % (context, key, value.value) + ) + elif not isinstance(value, Scalar): + errors.append( + "%s: label %r value is not a string" % (context, key) + ) + + +def check_rule(rule: Any, group_name: str, errors: List[str], warnings: List[str]) -> None: + if not isinstance(rule, dict): + errors.append("group %r: each rule must be a mapping" % group_name) + return + for key in rule: + if key not in RULE_KEYS: + warnings.append("group %r: unknown rule field %r" % (group_name, key)) + + record = as_str(rule.get("record")) + alert = as_str(rule.get("alert")) + has_record = bool(record and record.strip()) + has_alert = bool(alert and alert.strip()) + if has_record and has_alert: + errors.append("group %r: only one of 'record' and 'alert' must be set" % group_name) + if not has_record and not has_alert: + errors.append("group %r: one of 'record' or 'alert' must be set" % group_name) + + expr = as_str(rule.get("expr")) + if expr is None or expr.strip() == "": + errors.append("group %r: field 'expr' must be set in rule" % group_name) + elif not balanced_delimiters(expr): + errors.append( + "group %r: expression has unbalanced parentheses, brackets, or braces" + % group_name + ) + + if has_record: + if "{" in record or "}" in record: + errors.append( + "group %r: braces present in recording rule name %r " + "(should it be in expr?)" % (group_name, record) + ) + elif not RECORD_NAME_RE.match(record): + warnings.append( + "group %r: recording rule name %r does not match the recommended " + "metric-name pattern [a-zA-Z_:][a-zA-Z0-9_:]*" % (group_name, record) + ) + if rule.get("annotations"): + errors.append( + "group %r: invalid field 'annotations' in recording rule" % group_name + ) + for key in ("for", "keep_firing_for"): + if key in rule and as_str(rule.get(key)): + errors.append( + "group %r: invalid field %r in recording rule" % (group_name, key) + ) + + if has_alert: + if alert.strip() == "": + errors.append("group %r: alert name must not be empty" % group_name) + for key in ("for", "keep_firing_for"): + if key in rule and rule.get(key) is not None: + check_duration( + as_str(rule.get(key)), + errors, + "group %r rule %r field %r" % (group_name, alert, key), + ) + + check_labels(rule.get("labels"), errors, "group %r rule labels" % group_name) + check_labels(rule.get("annotations"), errors, "group %r rule annotations" % group_name) + + +def load_document(path: str) -> Tuple[Any, Optional[str]]: + """Read a file and parse it as YAML; returns (document, error_message).""" + try: + with open(path, "r", encoding="utf-8") as handle: + text = handle.read() + except OSError as exc: + return None, "cannot read file: %s" % exc + try: + return parse_yaml(text), None + except YAMLError as exc: + return None, "invalid YAML: %s" % exc + + +def check_rules_file(path: str) -> Dict[str, Any]: + result: Dict[str, Any] = {"name": "rules", "file": path} + doc, error = load_document(path) + if error is not None: + result["status"] = "error" + result["error"] = error + return result + if not isinstance(doc, dict) or doc.get("groups") is None: + result["status"] = "error" + result["error"] = "rules file must be a YAML mapping with a 'groups' list" + return result + groups = doc["groups"] + if not isinstance(groups, list): + result["status"] = "error" + result["error"] = "'groups' must be a list" + return result + + errors: List[str] = [] + warnings: List[str] = [] + seen_names: Dict[str, bool] = {} + rule_count = 0 + for group in groups: + if not isinstance(group, dict): + errors.append("each group must be a mapping") + continue + name = as_str(group.get("name")) + if name is None or name.strip() == "": + errors.append("group is missing a non-empty 'name'") + elif name in seen_names: + errors.append("group name %r is repeated in the same file" % name) + else: + seen_names[name] = True + for key in group: + if key not in GROUP_KEYS: + warnings.append("unknown group field %r" % key) + check_labels(group.get("labels"), errors, "group %r labels" % (name or "?")) + for key in ("interval", "query_offset"): + if key in group and group.get(key) is not None: + check_duration( + as_str(group.get(key)), errors, "group %r field %r" % (name or "?", key) + ) + rules = group.get("rules") + if rules is None: + continue + if not isinstance(rules, list): + errors.append("group %r 'rules' must be a list" % (name or "?")) + continue + for rule in rules: + rule_count += 1 + check_rule(rule, name or "?", errors, warnings) + + result["groups"] = len(groups) + result["rules"] = rule_count + result["errors"] = errors + result["warnings"] = warnings + result["status"] = "ok" if not errors else "issues" + return result + + +# --- scrape-target reachability --- + +def split_target(target: str) -> Tuple[Optional[str], Optional[int]]: + target = target.strip() + if target.startswith("["): + end = target.find("]") + if end == -1: + return target, None + host = target[1:end] + rest = target[end + 1:] + if rest.startswith(":") and rest[1:].isdigit(): + return host, int(rest[1:]) + return host, None + if target.count(":") == 1: + host, _, port = target.partition(":") + if port.isdigit(): + return host, int(port) + return target, None + + +def probe_target(target: str, timeout: float) -> Dict[str, Any]: + host, port = split_target(target) + if port is None: + return { + "target": target, + "reachable": False, + "error": "no port specified; expected host:port", + } + try: + with socket.create_connection((host, port), timeout=timeout): + return {"target": target, "reachable": True, "error": None} + except OSError as exc: + return {"target": target, "reachable": False, "error": str(exc)} + + +def _collect_scrape_targets(doc: Any, errors: List[str], warnings: List[str]) -> List[Dict[str, str]]: + targets: List[Dict[str, str]] = [] + configs = doc.get("scrape_configs") + if not isinstance(configs, list): + errors.append("'scrape_configs' must be a list") + return targets + for cfg in configs: + if not isinstance(cfg, dict): + errors.append("each scrape_config must be a mapping") + continue + job_name = as_str(cfg.get("job_name")) or "" + static = cfg.get("static_configs") + if static is None: + warnings.append( + "job %r has no static_configs; service-discovery targets are not probed" + % job_name + ) + continue + if not isinstance(static, list): + errors.append("job %r static_configs must be a list" % job_name) + continue + for entry in static: + if not isinstance(entry, dict): + errors.append("job %r static_configs entries must be mappings" % job_name) + continue + entry_targets = entry.get("targets") + if entry_targets is None: + continue + if not isinstance(entry_targets, list): + errors.append("job %r targets must be a list" % job_name) + continue + for item in entry_targets: + value = as_str(item) + if value and value.strip(): + targets.append({"target": value.strip(), "job": job_name}) + return targets + + +def check_scrape_config(path: str, timeout: float) -> Dict[str, Any]: + result: Dict[str, Any] = {"name": "scrape_targets", "file": path} + doc, error = load_document(path) + if error is not None: + result["status"] = "error" + result["error"] = error + return result + if not isinstance(doc, dict): + result["status"] = "error" + result["error"] = "scrape config must be a YAML mapping with 'scrape_configs'" + return result + + errors: List[str] = [] + warnings: List[str] = [] + targets = _collect_scrape_targets(doc, errors, warnings) + probes = [probe_target(entry["target"], timeout) for entry in targets] + unreachable = [probe for probe in probes if not probe["reachable"]] + + result["jobs"] = len(doc.get("scrape_configs") or []) + result["targets"] = probes + result["errors"] = errors + result["warnings"] = warnings + result["status"] = "ok" if not errors and not unreachable else "issues" + return result + + +def check_targets_file(path: str, timeout: float) -> Dict[str, Any]: + result: Dict[str, Any] = {"name": "targets", "file": path} + try: + with open(path, "r", encoding="utf-8") as handle: + text = handle.read() + except OSError as exc: + result["status"] = "error" + result["error"] = "cannot read file: %s" % exc + return result + + targets: Optional[List[Any]] = None + loaded = False + try: + loaded = json.loads(text) + if isinstance(loaded, list): + targets = loaded + except (ValueError, TypeError): + loaded = None + if loaded is None: + try: + doc = parse_yaml(text) + loaded = doc + except YAMLError: + loaded = None + if isinstance(loaded, list): + targets = loaded + elif isinstance(loaded, dict): + if "targets" in loaded and loaded["targets"] is not None: + targets = loaded["targets"] + elif "scrape_configs" in loaded: + errors: List[str] = [] + warnings: List[str] = [] + collected = _collect_scrape_targets(loaded, errors, warnings) + probes = [probe_target(entry["target"], timeout) for entry in collected] + result["jobs"] = len(loaded.get("scrape_configs") or []) + result["targets"] = probes + result["errors"] = errors + result["warnings"] = warnings + unreachable = [p for p in probes if not p["reachable"]] + result["status"] = "ok" if not errors and not unreachable else "issues" + return result + else: + result["status"] = "error" + result["error"] = "unrecognized targets file format" + return result + + if targets is None: + targets = [] + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + targets.append(Scalar(line)) + + probes = [ + probe_target(as_str(item) or "", timeout) + for item in targets + if as_str(item) and as_str(item).strip() + ] + unreachable = [probe for probe in probes if not probe["reachable"]] + result["targets"] = probes + result["errors"] = [] + result["warnings"] = [] + result["status"] = "ok" if not unreachable else "issues" + return result + + +# --- output --- + +def emit(payload: Dict[str, Any], as_json: bool) -> None: + if as_json: + print(json.dumps(payload, indent=2, sort_keys=True)) + return + lines: List[str] = [] + for check in payload["checks"]: + lines.append("== %s: %s ==" % (check["name"], check.get("file", ""))) + if check["status"] == "error": + lines.append(" error: %s" % check["error"]) + continue + for warning in check.get("warnings", []): + lines.append(" warning: %s" % warning) + for error in check.get("errors", []): + lines.append(" error: %s" % error) + if "groups" in check: + lines.append(" %d group(s), %d rule(s)" % (check["groups"], check["rules"])) + for probe in check.get("targets", []): + if probe["reachable"]: + lines.append(" %s reachable" % probe["target"]) + else: + lines.append(" %s UNREACHABLE: %s" % (probe["target"], probe["error"])) + if not lines: + lines.append("no checks to report") + print("\n".join(lines)) + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser( + prog=TOOL_NAME, + description=( + "Read-only Prometheus rule sanity and scrape-target reachability checker " + "for the Prometheus + OpenTelemetry + Loki telemetry stack. The tool only " + "reads local files and opens short TCP connect probes; it never mutates " + "configuration, writes files, or sends telemetry data." + ), + epilog=( + "Exit codes: 0 all checks passed, 1 issues found or a fatal error, " + "2 usage error. Use promtool check rules for full PromQL validation." + ), + ) + parser.add_argument( + "--rules", + action="append", + default=[], + metavar="FILE", + help="check Prometheus recording/alerting rules in FILE (repeatable)", + ) + parser.add_argument( + "--scrape", + action="append", + default=[], + metavar="FILE", + help="check a Prometheus scrape config in FILE and probe its static targets " + "(repeatable)", + ) + parser.add_argument( + "--targets", + action="append", + default=[], + metavar="FILE", + help="probe targets listed in FILE: one host:port per line, or a YAML/JSON " + "list of targets (repeatable)", + ) + parser.add_argument( + "--json", + action="store_true", + help="emit machine-readable JSON on stdout", + ) + parser.add_argument( + "--timeout", + type=float, + default=3.0, + help="seconds per TCP connect probe (default: %(default)s)", + ) + parser.add_argument( + "--version", + action="version", + version="%s %s" % (TOOL_NAME, TOOL_VERSION), + ) + args = parser.parse_args(argv) + + if not (args.rules or args.scrape or args.targets): + parser.error("at least one of --rules, --scrape, or --targets is required") + + checks: List[Dict[str, Any]] = [] + for path in args.rules: + checks.append(check_rules_file(path)) + for path in args.scrape: + checks.append(check_scrape_config(path, args.timeout)) + for path in args.targets: + checks.append(check_targets_file(path, args.timeout)) + + ok = all(check["status"] == "ok" for check in checks) + payload: Dict[str, Any] = { + "tool": TOOL_NAME, + "version": TOOL_VERSION, + "ok": ok, + "checks": checks, + "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + } + emit(payload, args.json) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/telemetry/tests/test_telemetry_check.py b/telemetry/tests/test_telemetry_check.py new file mode 100644 index 0000000..a153296 --- /dev/null +++ b/telemetry/tests/test_telemetry_check.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""Deterministic tests for the telemetry/scripts/telemetry-check tool. + +Runs the script as a subprocess so the tests exercise the real CLI surface +(--help, --json, --rules, --scrape, --targets, exit codes, JSON payloads). +Rules fixtures are written to temp directories at test time; scrape-target +tests probe a real local listening socket for the reachable case and a +just-released port for the unreachable case, so no external network is needed. +Also asserts the read-only contract: the script never opens files in write mode. +""" +import json +import re +import socket +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "telemetry-check" +FIXTURES = ROOT / "fixtures" + + +def run_script(*args): + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + timeout=30, + ) + + +def load_json(proc): + return json.loads(proc.stdout) + + +def free_port(): + """Bind a socket to an ephemeral port and return (port, socket); caller closes.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + sock.listen(1) + return sock.getsockname()[1], sock + + +class HelpTests(unittest.TestCase): + def test_help_exits_zero_and_advertises_capabilities(self): + proc = run_script("--help") + self.assertEqual(proc.returncode, 0) + self.assertIn("--json", proc.stdout) + self.assertIn("rule", proc.stdout.lower()) + self.assertIn("scrape", proc.stdout.lower()) + self.assertIn("read-only", proc.stdout.lower()) + + def test_version_flag(self): + proc = run_script("--version") + self.assertEqual(proc.returncode, 0) + self.assertIn("telemetry-check", proc.stdout) + + def test_no_args_is_usage_error(self): + proc = run_script() + self.assertEqual(proc.returncode, 2) + + +class RulesFileTests(unittest.TestCase): + def test_valid_fixture_parses_to_json_and_passes(self): + proc = run_script("--rules", str(FIXTURES / "prometheus-rules.yml"), "--json") + self.assertEqual(proc.returncode, 0, proc.stderr) + payload = load_json(proc) + self.assertTrue(payload["ok"]) + self.assertEqual(payload["checks"][0]["status"], "ok") + self.assertEqual(payload["checks"][0]["groups"], 2) + self.assertEqual(payload["checks"][0]["rules"], 4) + self.assertEqual(payload["checks"][0]["errors"], []) + + def test_malformed_yaml_is_rejected(self): + bad = "groups:\n - name: bad\n rules:\n - record: x\n" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "bad.yml" + path.write_text(bad, encoding="utf-8") + proc = run_script("--rules", str(path), "--json") + self.assertNotEqual(proc.returncode, 0) + payload = load_json(proc) + self.assertFalse(payload["ok"]) + self.assertIn("invalid YAML", payload["checks"][0]["error"]) + + def test_duplicate_group_names_detected(self): + rules = ( + "groups:\n" + " - name: g1\n" + " rules:\n" + " - record: a_total\n" + " expr: sum(foo)\n" + " - name: g1\n" + " rules:\n" + " - alert: B\n" + " expr: up == 0\n" + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "dup.yml" + path.write_text(rules, encoding="utf-8") + proc = run_script("--rules", str(path), "--json") + self.assertEqual(proc.returncode, 1) + errors = load_json(proc)["checks"][0]["errors"] + self.assertTrue(any("repeated" in error for error in errors)) + + def test_rule_errors_detected(self): + rules = ( + "groups:\n" + " - name: g1\n" + " rules:\n" + " - record: bad{name}\n" + " alert: AlsoAlert\n" + " for: 5x\n" + " labels:\n" + " severity: 5\n" + " - record: ok_name\n" + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "errors.yml" + path.write_text(rules, encoding="utf-8") + proc = run_script("--rules", str(path), "--json") + self.assertEqual(proc.returncode, 1) + errors = load_json(proc)["checks"][0]["errors"] + joined = "\n".join(errors) + self.assertIn("only one of 'record' and 'alert'", joined) + self.assertIn("braces present in recording rule name", joined) + self.assertIn("invalid duration", joined) + self.assertIn("not a YAML string", joined) + self.assertIn("field 'expr' must be set", joined) + + def test_unbalanced_expression_is_detected(self): + rules = ( + "groups:\n" + " - name: g1\n" + " rules:\n" + " - record: a_total\n" + " expr: sum(foo\n" + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "unbalanced.yml" + path.write_text(rules, encoding="utf-8") + proc = run_script("--rules", str(path), "--json") + self.assertEqual(proc.returncode, 1) + errors = load_json(proc)["checks"][0]["errors"] + self.assertTrue(any("unbalanced" in error for error in errors)) + + def test_quoted_numeric_label_value_is_accepted(self): + rules = ( + "groups:\n" + " - name: g1\n" + " rules:\n" + " - alert: HighCPU\n" + " expr: cpu_usage > 0.9\n" + " labels:\n" + " severity: \"5\"\n" + " team: platform\n" + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "quoted.yml" + path.write_text(rules, encoding="utf-8") + proc = run_script("--rules", str(path), "--json") + self.assertEqual(proc.returncode, 0, proc.stdout) + self.assertEqual(load_json(proc)["checks"][0]["errors"], []) + + def test_warning_only_findings_are_not_fatal(self): + rules = ( + "groups:\n" + " - name: g1\n" + " rules:\n" + " - record: my rule\n" + " expr: sum(foo)\n" + " extra_field: 1\n" + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "warn.yml" + path.write_text(rules, encoding="utf-8") + proc = run_script("--rules", str(path), "--json") + self.assertEqual(proc.returncode, 0, proc.stdout) + check = load_json(proc)["checks"][0] + self.assertEqual(check["status"], "ok") + self.assertTrue(any("metric-name pattern" in w for w in check["warnings"])) + self.assertTrue(any("unknown rule field" in w for w in check["warnings"])) + + def test_block_scalar_expression_parses(self): + rules = ( + "groups:\n" + " - name: g1\n" + " rules:\n" + " - alert: SlowQueries\n" + " expr: |\n" + " histogram_quantile(0.99,\n" + " sum by (le) (rate(query_duration_seconds_bucket[5m])))\n" + " for: 15m\n" + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "block.yml" + path.write_text(rules, encoding="utf-8") + proc = run_script("--rules", str(path), "--json") + self.assertEqual(proc.returncode, 0, proc.stdout) + self.assertEqual(load_json(proc)["checks"][0]["errors"], []) + + def test_missing_rules_file_is_fatal(self): + proc = run_script("--rules", "/nonexistent/rules.yml", "--json") + self.assertEqual(proc.returncode, 1) + payload = load_json(proc) + self.assertEqual(payload["checks"][0]["status"], "error") + + +class ScrapeTargetTests(unittest.TestCase): + def test_scrape_config_probes_reachable_and_unreachable(self): + reachable_port, listener = free_port() + try: + with tempfile.TemporaryDirectory() as tmp: + unreachable_port, probe = free_port() + probe.close() + config = ( + "scrape_configs:\n" + " - job_name: local\n" + " static_configs:\n" + " - targets:\n" + " - '127.0.0.1:%d'\n" + " - '127.0.0.1:%d'\n" % (reachable_port, unreachable_port) + ) + path = Path(tmp) / "scrape.yml" + path.write_text(config, encoding="utf-8") + proc = run_script( + "--scrape", str(path), "--json", "--timeout", "1" + ) + finally: + listener.close() + self.assertEqual(proc.returncode, 1) + check = load_json(proc)["checks"][0] + self.assertEqual(check["status"], "issues") + by_target = {entry["target"]: entry for entry in check["targets"]} + self.assertTrue(by_target["127.0.0.1:%d" % reachable_port]["reachable"]) + self.assertFalse(by_target["127.0.0.1:%d" % unreachable_port]["reachable"]) + + def test_targets_plain_list_file(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "targets.txt" + path.write_text("127.0.0.1:1\n127.0.0.1:2\n", encoding="utf-8") + proc = run_script("--targets", str(path), "--json", "--timeout", "1") + self.assertEqual(proc.returncode, 1) + check = load_json(proc)["checks"][0] + self.assertEqual(len(check["targets"]), 2) + self.assertTrue(all(not entry["reachable"] for entry in check["targets"])) + + def test_targets_yaml_list_file(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "targets.yml" + path.write_text("- 127.0.0.1:1\n- 127.0.0.1:2\n", encoding="utf-8") + proc = run_script("--targets", str(path), "--json", "--timeout", "1") + self.assertEqual(proc.returncode, 1) + check = load_json(proc)["checks"][0] + self.assertEqual(len(check["targets"]), 2) + + +class ReadOnlyContractTests(unittest.TestCase): + def test_script_never_opens_files_for_writing(self): + source = SCRIPT.read_text(encoding="utf-8") + pattern = re.compile(r"open\([^)]*['\"][w]['\"]") + self.assertIsNone(pattern.search(source)) + self.assertIn("read-only", source.lower()) + + +if __name__ == "__main__": + unittest.main()