diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index a4e558c..b85e319 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -653,6 +653,15 @@ "strict": false, "description": "Manage Linear teams, projects, cycles, issues, comments, workflow state, and documents from a terminal through Linear's public GraphQL API. Use when a user asks to list, search, inspect, create, update, move, or comment on Linear work, or to find Linear documents. Do not use to embed a live agent inside Linear or to build an MCP integration." }, + { + "name": "litellm", + "source": "./", + "skills": [ + "./litellm" + ], + "strict": false, + "description": "Operate, configure, secure, and troubleshoot the LiteLLM AI gateway (proxy) and Python SDK: run the proxy (litellm --config), route to 100+ providers through one OpenAI-compatible API, configure model lists and routing/reliability, virtual keys, teams, budgets, rate limits, caching, guardrails, observability, and spend, and diagnose request failures. Use when deploying or running a LiteLLM proxy or gateway (config.yaml, ghcr.io/berriai/litellm), wiring the Python SDK or OpenAI SDK through it, or hardening a public-facing deployment. Do not use for operating a single inference engine (vllm, llama-cpp), for engine-selection methodology (ml-engineering), or for building applications on top of an LLM API (backend/frontend engineering)." + }, { "name": "llama-cpp", "source": "./", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 876400d..e7f0ba0 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -90,6 +90,7 @@ "./legal-strategy", "./life-coach", "./linear", + "./litellm", "./llama-cpp", "./llamaindex", "./mermaid-diagrams", diff --git a/README.md b/README.md index 0643d08..5cb6e9a 100644 --- a/README.md +++ b/README.md @@ -296,6 +296,10 @@ Guide bounded, user-led coaching for adult nonclinical goals, decisions, transit Work with Linear teams, projects, cycles, issues, comments, workflow transitions, and documents using a small, dependency-free GraphQL CLI with bounded reads, dry-run previews, and focused reference guidance. +### [litellm](litellm/SKILL.md) + +Operate, configure, secure, and troubleshoot the LiteLLM AI gateway (proxy) and Python SDK: one OpenAI-compatible config routing to 100+ providers, model lists and load-balanced groups, virtual keys/teams/budgets/rate limits, caching and guardrails, observability and spend, deployment and public-facing hardening, and evidence-led failure diagnosis. Ships a read-only `litellm-health` probe (`--json`), proxy-config and deployment record templates, 9 dated references, tests, and 6 evals. Routes engine selection to ml-engineering and single-engine operation to vllm/llama-cpp. + ### [llama-cpp](llama-cpp/SKILL.md) Operate llama.cpp from hardware-aware installation and GGUF selection through verified local inference, OpenAI-compatible serving, reproducible tuning, multi-GPU operation, and evidence-led troubleshooting. Uses dated upstream sources, operation and benchmark templates, and six output-quality eval cases without adding a wrapper CLI. diff --git a/litellm/README.md b/litellm/README.md new file mode 100644 index 0000000..9ce03f1 --- /dev/null +++ b/litellm/README.md @@ -0,0 +1,66 @@ +# LiteLLM — AI Gateway Operations Skill + +Operate, configure, secure, and troubleshoot the LiteLLM AI gateway (proxy) and Python SDK: one config that routes to 100+ LLM providers through an OpenAI-compatible API, with virtual keys, teams, budgets and rate limits, caching, guardrails, observability, spend tracking, and evidence-led failure diagnosis. + +## Why Install This Skill + +Your agent can run the gateway instead of guessing. Teams that put an LLM gateway in front of OpenAI, Anthropic, Bedrock, Azure, Vertex, and local engines need someone (or something) that knows how to write a `config.yaml` whose duplicate `model_name` entries load-balance a group, why budgets silently fail open without Postgres, which response header tells you which deployment served a request, why `AnthropicException - Overloaded` is not a gateway bug, and how to harden a public-facing proxy against the 2026 CVE wave — without leaking keys or prompt content. + +This skill ships that operating knowledge plus two fillable templates — a proxy config record (so every deployment is reproducible) and a deployment record (image digest, ports, data stores, rollback path) — and a read-only `litellm-health` probe that checks a running proxy's liveness, readiness, registered models, and model info over HTTP without changing anything. The references are distilled from the official LiteLLM documentation and verified against litellm 1.97.0 with dated sources. Engine-selection methodology deliberately routes up to `ml-engineering`; single-engine operation routes to `vllm` and `llama-cpp`; this skill owns the day-to-day operation of LiteLLM itself. + +## What You Get + +| Directory | Purpose | +|---|---| +| `SKILL.md` | Agent-facing operating contract, operating loop, verification boundaries, hard boundaries | +| `references/` | Nine dated, source-indexed references: source index, quickstart + SDK, config & routing, keys/teams/budgets/spend, caching & guardrails, observability & logging, deployment, security & public hosting, troubleshooting | +| `templates/proxy-config-record.md` | Fillable record of every model entry, routing knob, budget, and secret reference — the rollback unit | +| `templates/proxy-deployment.md` | Fillable record of the runtime: pinned image, ports, env vars, Postgres/Redis endpoints, probes, rollback path | +| `scripts/litellm-health` | Read-only probe: liveliness, readiness, `/v1/models`, `/model/info`; stdlib-only, `--json`, `--help` without a server | +| `tests/` | Deterministic tests against a local stub HTTP server, including the read-only contract | +| `evals/evals.json` | Six output-quality evaluation cases for agent runs | + +## Quick Start + +```bash +# Help works with no LiteLLM proxy running +scripts/litellm-health --help + +# Probe a running proxy, machine-readable +scripts/litellm-health --url http://127.0.0.1:4000 --json + +# Model routes need the master key or a virtual key +scripts/litellm-health --check health --check readiness \ + --check models --key "$LITELLM_MASTER_KEY" --json + +# Minimal multi-provider config, then start it +cat > config.yaml <<'YAML' +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY +YAML +litellm --config config.yaml --port 4000 + +# Verify at the delivery boundary +curl -s http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ + -H 'Content-Type: application/json' \ + -d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "ping"}]}' | head -c 400 +``` + +The `litellm-health` script uses only Python's standard library and issues GET requests only. Exit codes: 0 all checks passed, 1 issues found or a fatal error, 2 usage error, 124 timeout. Health probes (`/health/liveliness`, `/health/readiness`) are unauthenticated by design; `models` and `model_info` require a bearer key. Before changing any production setting, fill in `templates/proxy-config-record.md` — it is the rollback unit. + +## Triggers + +Load this skill for LiteLLM operations: deploying or updating a proxy (`litellm --config`, the `ghcr.io/berriai/litellm` image, Helm charts), writing or debugging `config.yaml` (`model_list`, `router_settings`, `litellm_settings`, `general_settings`), routing to multiple providers through one OpenAI-compatible endpoint, configuring virtual keys, teams, budgets, or rate limits, response caching or guardrails (Presidio PII masking), observability callbacks (Langfuse, OpenTelemetry, Prometheus `/metrics`), spend tracking, hardening a public-facing gateway, or diagnosing request failures (401 vs provider auth errors, `No deployments available`, context-window fallbacks, timeouts). Do not load it for engine selection or serving methodology (`ml-engineering`), for operating vLLM or llama.cpp themselves (`vllm`, `llama-cpp`), or for generic Docker/Kubernetes administration (`docker-compose`, `kubernetes`). + +## Requirements + +- A LiteLLM release: `pip install 'litellm[proxy]'` (the `[proxy]` extra is required for the server; Python >=3.10 since 1.84.0) or the pinned container image `ghcr.io/berriai/litellm:vX.Y.Z`. +- For keys, teams, budgets, spend, and the admin UI: PostgreSQL (`DATABASE_URL`). For more than one replica: Redis >=7. +- Public deployments must run >=1.83.7 (CVE-2026-42208/42203/42271 fix floor; Starlette >=1.0.1). +- Python 3.9+ for the `litellm-health` script (`--help` needs nothing else); live probes need HTTP(S) access to the running proxy, and model routes require the master key or a virtual key. diff --git a/litellm/SKILL.md b/litellm/SKILL.md new file mode 100644 index 0000000..0888b81 --- /dev/null +++ b/litellm/SKILL.md @@ -0,0 +1,286 @@ +--- +name: litellm +description: >- + Operate, configure, secure, and troubleshoot the LiteLLM AI gateway (proxy) and + Python SDK: run the proxy (litellm --config), route to 100+ providers through one + OpenAI-compatible API, configure model lists and routing/reliability, virtual keys, + teams, budgets, rate limits, caching, guardrails, observability, and spend, and + diagnose request failures. Use when deploying or running a LiteLLM proxy or gateway + (config.yaml, ghcr.io/berriai/litellm), wiring the Python SDK or OpenAI SDK through + it, or hardening a public-facing deployment. Do not use for operating a single + inference engine (vllm, llama-cpp), for engine-selection methodology (ml-engineering), + or for building applications on top of an LLM API (backend/frontend engineering). +license: MIT +compatibility: >- + Requires litellm (pip, Python >=3.10) or the litellm proxy image + (ghcr.io/berriai/litellm or docker.litellm.ai/berriai/litellm, pinned >=1.83.7 for + public deployments). The bundled litellm-health script runs on Python 3.9+ and needs + no proxy for --help; live probes require HTTP(S) access to a running proxy, and + model routes require the master key or a virtual key. +metadata: + source: https://docs.litellm.ai/ + source_index: references/00-source-index.md + research_checked: "2026-08-22" +--- + +# LiteLLM AI Gateway Operations + +Use this skill to operate **LiteLLM** as an organization's AI gateway: run the proxy +(`litellm --config config.yaml`), route requests to 100+ LLM providers through one +OpenAI-compatible API, manage model lists, routing and reliability, virtual keys, +teams, budgets and rate limits, caching, guardrails, observability, and spend — and +diagnose failures with evidence. LiteLLM ships two surfaces: a Python SDK +(`litellm.completion()`, in-process) and the proxy (a FastAPI service on port 4000 +with keys, budgets, and an admin UI). This is a **tool skill** for the named tool. +Engine selection and serving methodology belong to +[ml-engineering](../ml-engineering/SKILL.md); operating a single engine belongs to +[vllm](../vllm/SKILL.md) or [llama-cpp](../llama-cpp/SKILL.md). + +## Operating contract + +1. **Record the deployment before tuning it.** Capture the pinned image or pip + version, `config.yaml`, model list, routing, budgets, env-var references, and + data stores in the [proxy config record](templates/proxy-config-record.md). That + record is the rollback unit. +2. **Confirm the target, scope, and rollback path before mutating.** Read-only + discovery (health probes, `/v1/models`, logs, spend queries) may proceed without + confirmation. Mutations — config changes, key mint/revocation, restarts, image + upgrades, DB migrations — require an explicit human directive naming the deployment. +3. **A proxy that responds is not a proxy that serves.** `/health/liveliness` + returning 200 proves liveness only. Verify at the delivery boundary: a + representative `/v1/chat/completions` request returns tokens and + `x-litellm-model-id` names the deployment you expected. +4. **Keep evidence bounded.** Summarize logs and configs; never dump full logs, + `.env` contents, master keys, or provider credentials into chat. Spend logs and + debug output can contain prompt content — redact before sharing. +5. **Pin versions.** LiteLLM releases weekly and changes defaults; every claim here + was checked against 1.97.0 (2026-08-22). Re-verify version-sensitive behavior + against your installed release before relying on it. + +## The litellm-health script + +`scripts/litellm-health` is a read-only probe for a running proxy. It issues GET +requests only, never writes files, and emits bounded output. + +```bash +scripts/litellm-health --help # no proxy needed +scripts/litellm-health --url http://127.0.0.1:4000 --json +scripts/litellm-health --check health --check readiness --json +scripts/litellm-health --check models --check model_info \ + --key "$LITELLM_MASTER_KEY" --json +``` + +Exit codes: 0 all checks passed, 1 issues found or a fatal error, 2 usage error, +124 timeout. Checks: `health` (`GET /health/liveliness`, unauthenticated), `readiness` +(`GET /health/readiness`, unauthenticated; 503 when the configured DB is unreachable), +`models` (`GET /v1/models`, requires key), and `model_info` (`GET /model/info`, +requires key). Keys are sent as `Authorization: Bearer `. The script never sends +data anywhere except the proxy you name. + +## Operating loop + +1. **Identify the deployment**: pinned version/image digest, how it runs (bare, + Docker, Compose, Helm), config source (file, `store_model_in_db`, or both), and + data stores (Postgres? Redis?). +2. **Collect evidence**: `litellm-health --json`; `GET /v1/models` and + `/model/info` with a key; response headers (`x-litellm-call-id`, + `x-litellm-model-id`, `x-litellm-model-api-base`, `x-litellm-version`); + `--detailed_debug` logs or `LITELLM_LOG=DEBUG` for the outbound request. +3. **Triage against the symptom**: classify provider vs gateway errors (see + [troubleshooting](references/08-troubleshooting.md)); check cooldown state, + budgets, DB connectivity. +4. **Act with confirmation**: bounded, scoped changes after a human directive, with + the rollback path named first. +5. **Verify**: re-run the probe and a representative chat request at the delivery + boundary. + +## Quickstart: one config, many providers + +```yaml +model_list: + - model_name: gpt-4o # name clients request + litellm_params: + model: openai/gpt-4o # routed string (provider prefix required) + api_key: os.environ/OPENAI_API_KEY # resolved inside the proxy process + - model_name: claude-sonnet + litellm_params: + model: anthropic/claude-sonnet-4-5 + api_key: os.environ/ANTHROPIC_API_KEY + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY # require auth on every call +``` + +Start with `litellm --config config.yaml --port 4000`. Success logs +`Proxy initialized with Config, Set models:`. Clients call the OpenAI surface: +`/v1/chat/completions`, `/chat/completions`, `/v1/embeddings`, `/v1/images/generations`, +`/v1/audio/transcriptions`, plus `/responses`, Anthropic-compatible `/messages`, +`/model/info`, `/health/liveliness`, `/health/readiness`. Any OpenAI SDK works +unchanged: `openai.OpenAI(base_url="http://localhost:4000", api_key=)`. +Details and the SDK surface: [quickstart reference](references/01-quickstart-and-sdk.md). + +## Config and routing + +- Entries sharing a `model_name` form one load-balanced group; each entry is a + deployment with its own hashed `model_id` used for health and cooldown tracking. +- `router_settings.routing_strategy` — `simple-shuffle` (default, recommended; + weighted by `rpm`/`tpm` or `weight` under `litellm_params`), `least-busy`, + `latency-based-routing`, `usage-based-routing` (docs warn against it in prod), + `cost-based-routing`. +- Reliability: `litellm_settings.num_retries` (per-deployment and request-level + overrides exist; `num_retries` is not the provider SDK's `max_retries`), + `fallbacks` / `context_window_fallbacks` / `content_policy_fallbacks`, + cooldowns (`allowed_fails`, `cooldown_time`), deployment `order` for priority, + `enable_pre_call_checks: true` to enforce context windows and region filters + pre-call (opt-in). +- With `store_model_in_db: true`, UI/API writes deep-merge over YAML in Postgres and + win on key conflicts — editing those YAML keys later has no effect while the DB row + exists. Details: [config and routing reference](references/02-config-and-routing.md). + +## Keys, teams, budgets, spend + +- `general_settings.master_key` (must start `sk-`) is the admin credential and UI + password. Virtual keys (`POST /key/generate`) scope models, budgets, and rpm/tpm + per workload; keys are stored hashed and never contain provider credentials. +- **Budgets require Postgres.** Without a connected DB, budgets fail open (a startup + warning is the only signal) and key endpoints return `No connected db.` — never run + a budget-sensitive deployment DB-less. +- Team keys enforce team (+ team-member) budgets only; the owner's personal budget + does not apply. Rate limits do not apply to proxy admins. Spend lands in + `/spend/logs` and `/global/spend`; `store_prompts_in_spend_logs` defaults to false. + Details: [keys and budgets reference](references/03-keys-teams-budgets-spend.md). + +## Caching and guardrails + +- Response cache: `litellm_settings.cache: true` + `cache_params.type: redis` for + multi-instance production (in-memory is per-process; disk/S3/GCS exist). Per-request + controls: `cache: {ttl, no-cache, namespace}` in the body. +- Semantic caches (`qdrant-semantic`, `redis-semantic`, `valkey-semantic`) embed the + whole messages array and can replay stale answers across similar multi-turn turns — + docs recommend excluding agentic traffic from semantic caching. +- Guardrails run `pre_call`, `post_call`, `during_call`, or `logging_only` (there is + no `all` mode); Presidio PII masking is OSS. Violations fail with HTTP 400 and an + embedded verdict; `x-litellm-applied-guardrails` names what ran. + Details: [caching and guardrails reference](references/04-caching-and-guardrails.md). + +## Observability and logging + +- Callbacks: `litellm_settings.success_callback` / `failure_callback` / `callbacks` + (Langfuse, OTel, Prometheus, Datadog, Sentry, ...). Prometheus `/metrics` requires + auth since 1.85.0 — give the scraper a bearer key or set + `require_auth_for_metrics_endpoint: false`. +- Forensic response headers: `x-litellm-call-id`, `x-litellm-model-id`, + `x-litellm-model-api-base`, `x-litellm-version`, `x-litellm-response-cost`. +- Privacy: `turn_off_message_logging: true` keeps metadata but drops content from + callbacks; `redact_user_api_key_info: true` redacts key/user/team identifiers. + Debug with `--detailed_debug`, `LITELLM_LOG=DEBUG`, or per-request + `"litellm_request_debug": true`. + Details: [observability reference](references/05-observability-and-logging.md). + +## Deployment + +- Postgres is mandatory for keys, teams, spend, budgets, and UI state; Redis >=7 is + required for more than one instance (shared rate-limit counters, cooldowns, cache). +- Pin image tags (`ghcr.io/berriai/litellm:vX.Y.Z` — semver tags since 1.84.0; + `-stable` suffixes are gone, `main-latest` is deprecated). Images are cosign-signed. +- Prisma migrations run at startup by default; on Kubernetes use the migration job + pattern with `DISABLE_SCHEMA_UPDATE=true` on serving pods. One Uvicorn worker per + pod; size the DB pool as `MAX_DB_CONNECTIONS / (instances x workers)`. + Details: [deployment reference](references/06-deployment.md). + +## Security and public hosting + +- Version floor for any internet-reachable proxy: **>=1.83.7** (CVE-2026-42208 + pre-auth SQLi, CVE-2026-42203 SSTI, CVE-2026-42271 command injection, plus + Starlette >=1.0.1 for the CVE-2026-48710 host-header chain). Two of these were + CISA KEV-listed and actively exploited in 2026. +- Never expose management routes (`/key/*`, `/user/*`, `/team/*`, `/config/*`, + `/model/*`, `/spend/*`, `/ui`, `/prompts/test`, `/mcp-rest/*`). Route lockdown via + `allowed_routes` is Enterprise — on OSS, enforce at the reverse proxy. +- `LITELLM_SALT_KEY` encrypts DB-stored provider credentials; set it once and never + rotate it after adding models. Rotate the master key only via the documented flow. +- March 2026 supply-chain incident: backdoored `litellm==1.82.7/.8` PyPI wheels + (~40 minutes). Prefer cosign-verified pinned images over unpinned pip installs. + Hardening checklist: [security reference](references/07-security-and-public-hosting.md). + +## Troubleshooting: the master diagnostic rule + +**If the error contains `Exception`, the provider failed — not the +gateway.** `AnthropicException`, `OpenAIException`, `BedrockException`, ... mean the +upstream call happened and its response is the evidence. No provider name means the +gateway itself rejected the call (bad LiteLLM key, unknown model, cooldowns, budget). + +| Symptom | First move | +|---|---| +| `Invalid model name passed in model=X` | Name not in `model_list` or not granted to the key; check `GET /v1/models` with the same key | +| `No deployments available for selected model, Try again in N seconds` | All deployments cooling down (usually upstream 429s) or a missing provider prefix on `litellm_params.model` | +| `AnthropicException - Overloaded` (HTTP 500, Anthropic's 529) | Provider-side overload; retry/fail over — not a gateway bug | +| `Authentication Error ... ExceededTokenBudget` | Key/team budget exhausted; check `GET /key/info` | +| `ImportError: cannot import name 'get_flat_dependant'` at startup | fastapi too new for the pinned litellm; pin `fastapi==0.136.3` for 1.97.0 | + +Full taxonomy and fixes: [troubleshooting reference](references/08-troubleshooting.md). + +## Reference routing + +| Load when | Reference | +|---|---| +| Sources, version observations, refresh procedure | `references/00-source-index.md` | +| Proxy quickstart, config.yaml, Python SDK, OpenAI-SDK drop-in | `references/01-quickstart-and-sdk.md` | +| model_list, routing strategies, retries/fallbacks/cooldowns | `references/02-config-and-routing.md` | +| Virtual keys, teams, budgets, rate limits, spend | `references/03-keys-teams-budgets-spend.md` | +| Response caching and guardrails | `references/04-caching-and-guardrails.md` | +| Callbacks, Prometheus, headers, privacy switches | `references/05-observability-and-logging.md` | +| Docker/Compose/K8s/Helm, scaling, migrations, upgrades | `references/06-deployment.md` | +| Public-facing hardening, CVE floor, supply chain | `references/07-security-and-public-hosting.md` | +| Error taxonomy, failure modes, debugging workflow | `references/08-troubleshooting.md` | + +## Included artifacts + +- `scripts/litellm-health`: read-only proxy probe (stdlib-only, `--json`, `--check` + subsets, `--key` for authenticated routes, `--help` without a server). +- `tests/test_litellm_health.py`: deterministic tests against a local stub HTTP + server, including the read-only contract. +- `templates/proxy-config-record.md` and `templates/proxy-deployment.md`: fillable + records — the config record is the rollback unit; the deployment record freezes the + runtime (image digest, ports, env, data stores, probes, rollback). +- `references/`: nine dated, source-indexed references covering the topics above. +- `evals/evals.json`: six output-quality evaluation cases. + +## Verification boundary + +| Claim | Minimum evidence | +|---|---| +| The proxy is alive | `litellm-health --check health` reports `/health/liveliness` 200 | +| The proxy is ready | `--check readiness` reports `/health/readiness` 200 (503 means DB down) | +| The right models are registered | `/v1/models` (with the calling key) lists the expected aliases | +| A deployment is configured correctly | `/model/info` shows the expected `litellm_params` with keys redacted | +| Inference works | A representative `/v1/chat/completions` request returns tokens and `x-litellm-model-id` names the intended deployment | +| Budgets are enforced | A connected DB is verified (readiness) and `/key/info` shows spend tracking for the key | +| A diagnosis is sound | Evidence (error string, headers, logs) was collected before the claim, and the fix was verified by re-running the probe and a representative request | + +## Hard boundaries + +- Never mutate a production proxy (config, keys, teams, budgets, image, DB) without + an explicit human directive naming the target and a stated rollback path. Read-only + discovery may proceed freely. +- Never expose the master key, management routes, or `/ui` beyond the trust boundary; + authentication is not a substitute for network and TLS controls. +- Never commit provider keys, `DATABASE_URL`, `LITELLM_MASTER_KEY`, or + `LITELLM_SALT_KEY` anywhere; use `os.environ/` references and a secret manager. +- Never run a budget-sensitive public deployment without Postgres — budgets fail + open without one. +- Never treat a 200 from `/health/liveliness` as proof the gateway serves; verify at + the delivery boundary. + +## When not to use + +- **Engine selection, serving methodology, quantization decisions, evaluation + design** — that is [ml-engineering](../ml-engineering/SKILL.md). +- **Operating a single inference engine** — [vllm](../vllm/SKILL.md) for vLLM, + [llama-cpp](../llama-cpp/SKILL.md) for the llama.cpp stack. LiteLLM routes *to* + engines; it does not replace their own operation. +- **Kubernetes/Docker fundamentals and reverse-proxy/TLS configuration** — that is + [kubernetes](../kubernetes/SKILL.md), [docker-compose](../docker-compose/SKILL.md), + and [traefik](../traefik/SKILL.md); this skill covers the LiteLLM-specific layer. +- **Building applications on top of an LLM API** (app architecture, agent frameworks) + — that is backend/frontend engineering; this skill owns the gateway and its SDK. diff --git a/litellm/evals/evals.json b/litellm/evals/evals.json new file mode 100644 index 0000000..ee3d6bd --- /dev/null +++ b/litellm/evals/evals.json @@ -0,0 +1,80 @@ +{ + "schema_version": 1, + "skill_name": "litellm", + "evals": [ + { + "id": "quickstart-config", + "prompt": "Stand up a LiteLLM proxy that exposes one stable model alias 'gpt-4o' backed by two deployments: an OpenAI gpt-4o and an Azure deployment named gpt-4o-eu. Give me the config.yaml, the command to run it, how clients call it with the OpenAI SDK, and exactly what to check to know it actually works.", + "expected_output": "A config.yaml whose two entries share model_name 'gpt-4o' so they form a load-balanced group; each entry's litellm_params.model carries the provider prefix (openai/gpt-4o vs azure/gpt-4o-eu, using the Azure deployment name not the raw model name) with api_key set via os.environ/ references rather than literals, plus api_base for the Azure entry. Start with litellm --config config.yaml --port 4000 and confirm the log line 'Proxy initialized with Config, Set models:' appears — its absence means the config didn't load. Clients use any OpenAI SDK pointed at base_url http://localhost:4000 with a LiteLLM key as api_key, never a provider key; with general_settings.master_key set every route requires Authorization: Bearer. Verification at the delivery boundary: GET /health/liveliness returns 200 'I'm alive!' unauthenticated, GET /v1/models with the calling key lists gpt-4o, /model/info shows both deployments with keys redacted, and a representative chat completion returns tokens with x-litellm-model-id identifying which deployment served it. Note the proxy binds 0.0.0.0 by default, so exposure is a deliberate decision.", + "assertions": [ + "Both entries share model_name so they form one load-balanced group", + "litellm_params.model strings carry provider prefixes and Azure uses the deployment name with api_base", + "API keys are os.environ/ references, never inline values", + "Startup success is verified by the 'Proxy initialized with Config' log line", + "Verification includes /health/liveliness, /v1/models with the calling key, and a representative chat request returning tokens with x-litellm-model-id", + "Clients point an OpenAI SDK at the proxy base_url with a LiteLLM key, not provider credentials" + ] + }, + { + "id": "routing-and-reliability", + "prompt": "Our gateway name 'chat-main' spans three provider deployments with different capacities, and we need it to survive upstream 429 storms and single-deployment outages without returning errors to clients. Design the routing and reliability configuration and explain what happens on failure.", + "expected_output": "A design where all three entries share model_name chat-main forming one LB group, weighted by rpm or weight under litellm_params (simple-shuffle default is the recommended strategy — usage-based-routing is discouraged for production latency). Reliability layers: litellm_settings.num_retries (distinct from the provider SDK's max_retries, which the router pins to 0), fallbacks mapping chat-main to another model group after retries exhaust, cooldown tuning via allowed_fails/cooldown_time with per-error-class allowed_fails_policy. Failure mechanics explained: an upstream 429 immediately cools that deployment (~5s default), retries pick among remaining peers, order tiers can hold cheaper capacity first with costlier tier absorbing failures, and when everything is cooling the client sees HTTP 429 'No deployments available for selected model'. enable_pre_call_checks: true is needed before context windows are enforced pre-call. Every fallback target must be a registered model_name alias, not a raw provider string, and the failover path should be proven once by forcing a deployment failure before trusting it.", + "assertions": [ + "Same model_name across entries forms the load-balanced group with weight/rpm under litellm_params", + "simple-shuffle is recommended and num_retries is distinguished from the provider SDK max_retries pinned to 0", + "Cooldown behavior on 429 is explained including the 'No deployments available' client-facing 429", + "Fallback targets are model_name aliases and context-window enforcement requires enable_pre_call_checks", + "The configuration is verified by forcing a deployment failure and observing failover" + ] + }, + { + "id": "keys-budgets-spend", + "prompt": "We're putting our LiteLLM gateway in front of three internal teams. Each team needs its own spend ceiling, per-service virtual keys, and protection against runaway spend from a buggy job. One staging proxy currently runs without a database — does anything change there? Design the scheme.", + "expected_output": "A scheme built on Postgres-backed virtual keys: master key stays in a secret manager and is never shared; each service gets a key from POST /key/generate scoped to allowed models with max_budget + budget_duration and tpm/rpm limits; teams via /team/new with team budgets, noting that a key belonging to a team enforces only team (+ member) budgets, not the owner's personal budget. Runaway-job protection stacks several controls: hard budgets that reject requests when crossed, soft_budget warnings, rate limits (which do not apply to proxy admins — test with an internal-user role), upperbound_key_generate_params so self-service cannot mint oversized keys, instant block/unblock revocation, and optionally fail_closed_budget_enforcement for hard ceilings across replicas. The DB question is decisive: budgets require Postgres and fail open without one — on the DB-less staging proxy no budget will ever block a request and /key/* endpoints return 'No connected db.', so staging must connect DATABASE_URL (or accept that only upstream/provider-side limits protect it). Verification: prove enforcement once by setting a tiny test budget, exceeding it, observing the ExceededBudget/ExceededTokenBudget error, then restoring; confirm live spend via GET /key/info.", + "assertions": + ["Virtual keys are scoped per service with budgets, durations, and rpm/tpm limits while the master key stays in a secret manager", + "Team keys enforce team/member budgets only, not the owner's personal budget", + "Rate limits do not apply to proxy admins and upperbound_key_generate_params caps self-service keys", + "The answer states budgets require Postgres and FAIL OPEN without a DB, so the DB-less staging proxy enforces nothing", + "Enforcement is verified empirically with a tiny test budget exceeded on purpose and /key/info confirming spend tracking"] + }, + { + "id": "caching-guardrails-choice", + "prompt": "We have two workloads on one LiteLLM gateway: (1) a high-volume RAG FAQ bot with mostly repeated identical prompts, and (2) a multi-step coding agent whose consecutive turns are near-identical. We also must mask emails and card numbers before anything reaches providers. Recommend caching and guardrail configurations, and name the trap people hit with semantic caching here.", + "expected_output": "Caching split by workload: exact-match caching (cache: true with type redis for multi-instance correctness, in-memory only for single process) serves workload 1 well since identical prompts produce identical cache keys, with ttl chosen deliberately and per-request controls (no-store/no-cache) available. Workload 2 should NOT use semantic caching: semantic caches embed the entire messages array and serve nearest neighbors above a similarity threshold, and consecutive agent turns are ~0.99 similar, so stale tool results get replayed as hits — the documented recommendation is excluding agentic/multi-turn traffic from semantic caching entirely (exact-match redis instead, opt-in mode, or per-request no-store). Guardrails: a Presidio guardrail in pre_call mode with pii_entities_config masking EMAIL_ADDRESS and CREDIT_CARD (MASK rewrites content; BLOCK rejects), score thresholds tuned, applied on every request via default_on or requested via the guardrails body param; violations/masking are observable through x-litellm-applied-guardrails and the logging payload. Verification: identical request twice yields a cache hit, and a prompt containing a masked entity reaches providers with plaintext absent (bounded log check).", + "assertions": [ + "Exact-match redis caching is recommended for repeated identical prompts with multi-instance awareness", + "Semantic caching is excluded for the agent workload because near-identical consecutive turns replay stale responses", + "Presidio PII guardrail configured pre_call with MASK semantics for email/credit-card entities", + "default_on or per-request guardrails body param decides when the guardrail runs", + "Verification covers a demonstrated cache hit and confirmation that masked content never reached the provider" + ] + }, + { + "id": "security-hardening-public-proxy", + "prompt": "We're about to expose our LiteLLM proxy to the internet behind an ALB. Current image is ghcr.io/berriai/litellm:1.82.5, we installed via pip without pinning, master key is sk-1234, and the UI is reachable. What must change before this goes live?", + "expected_output": "A hardening review that leads with the version floor: >=1.83.7 is mandatory for internet-reachable proxies because CVE-2026-42208 (pre-auth SQL injection via crafted Authorization header, CISA KEV, actively exploited), CVE-2026-42203 (SSTI in /prompts/test), and CVE-2026-42271 (command injection in MCP test endpoints) were all fixed in v1.83.7, and the Starlette host-header chain (CVE-2026-48710) additionally needs Starlette >=1.0.1 — running 1.82.5 publicly should be treated as compromised until patched, with provider keys rotated. Supply chain: unpinned pip installs are how the March 2026 backdoored wheels (1.82.7/.8) would have landed; switch to a cosign-verified pinned semver image tag or digest. Credentials: replace sk-1234 (scanner-fingerprinted placeholder) with a strong random sk- key from a secret manager; restrict or disable the Admin UI, which is equivalent to holding the master key; issue scoped virtual keys per workload instead of sharing the master key; set LITELLM_SALT_KEY once if DB-stored credentials are used. Exposure: terminate TLS at the ALB, expose only LLM routes plus health probes, deny management paths (/key/*, /user/*, /team/*, /config/*, /model/*, /spend/*, /ui, /prompts/test, /mcp-rest/*) — noting allowed_routes lockdown is Enterprise so OSS enforces at the reverse proxy. Add budgets/rate limits on every public key (budgets need Postgres and fail open without it), alerting, and verify from outside: management routes 403/404, health probes answer, revoked keys fail immediately.", + "assertions": [ + "Version floor >=1.83.7 justified by CVE-2026-42208/42203/42271 plus Starlette >=1.0.1 for the host-header chain", + "The actively-exploited pre-auth SQLi and KEV listing make 1.82.5 public exposure treated as compromised pending patch and key rotation", + "Unpinned pip installs tied to the March 2026 backdoored-wheel incident; cosign-verified pinned images required", + "sk-1234 replaced with a strong random key, Admin UI restricted/disabled, scoped virtual keys issued", + "Management paths denied at the edge with allowed_routes noted as Enterprise, TLS terminated up front", + "Budgets/rate limits on public keys with the Postgres fail-open caveat, verified from outside the trust boundary" + ] + }, + { + "id": "troubleshooting-error-batch", + "prompt": "Diagnose these four client reports against our LiteLLM gateway and give the evidence-led fix for each: (a) 404 'Invalid model name passed in model=gpt-4.1-mini', (b) intermittent HTTP 429 'No deployments available for selected model, Try again in 60 seconds', (c) litellm.InternalServerError: AnthropicException - Overloaded, (d) a request dies after 10 minutes with a timeout despite a healthy proxy. For each: what evidence you'd collect first and what fixes it.", + "expected_output": "(a) Apply the master rule — no ProviderException means the gateway rejected it: the alias isn't registered in model_list or isn't granted to this key; evidence is GET /v1/models called with the same key (grants are per-key); fix by adding the entry or correcting the model string. (b) Router-level 429: every deployment of the group was cooling down, typically after upstream 429 storms — possibly compounded by a missing provider prefix leaving no valid deployment; evidence is /health?model= per deployment, debug logs showing cooldown state, and x-litellm headers; fix the underlying provider limits, tune cooldown_time/allowed_fails_policy, add capacity or fallbacks — not disable_cooldowns. (c) The AnthropicException prefix proves the provider failed: Anthropic's HTTP 529 overload surfaces as InternalServerError (500-class), it's retryable and not a gateway bug; handle via retries/failover and provider status checks. (d) A bounded timeout: recent litellm defaults are long (request_timeout around 6000s; SDK completion timeout 600s), so a 10-minute death matches an explicit or derived limit; evidence is which layer timed out from debug logs (proxy vs provider) and stream_timeout behavior for streams; fix by setting deliberate router_settings.timeout/request_timeout/per-deployment timeouts sized to the workload. Throughout: collect error string, status code, x-litellm-call-id/model-id headers and debug logs BEFORE claiming cause, then verify the fix with a representative request.", + "assertions": [ + "Provider-vs-gateway classification drives each diagnosis using the Exception presence rule", + "(a) resolved via /v1/models queried with the same key and model_list/grant correction", + "(b) tied to deployment cooldowns after upstream 429s with disable_cooldowns explicitly rejected", + "(c) identified as Anthropic HTTP 529 overload surfaced as InternalServerError and handled by retry/failover", + "(d) connected to explicit/derived long timeout defaults (request_timeout ~6000s, completion 600s) with bounded settings recommended", + "Evidence collection precedes each claim and each fix is verified with a representative request" + ] + } + ] +} diff --git a/litellm/references/00-source-index.md b/litellm/references/00-source-index.md new file mode 100644 index 0000000..cb86ffa --- /dev/null +++ b/litellm/references/00-source-index.md @@ -0,0 +1,93 @@ +# LiteLLM Operations — Source Index + +> **Last Updated:** 2026-08-22 +> Sources: https://docs.litellm.ai/docs/ and https://github.com/BerriAI/litellm + +This index tracks the authoritative upstream sources behind the LiteLLM operational +skill and the refresh procedure for keeping it current. LiteLLM releases weekly; +flags, defaults, endpoint behavior, and Enterprise boundaries change between +releases. Treat any claim in this skill as version-sensitive and re-verify against +the installed release. + +## Canonical sources + +| Topic | Source | +|---|---| +| Documentation home | https://docs.litellm.ai/docs/ | +| Releases and release notes | https://github.com/BerriAI/litellm/releases | +| Release cycle (weekly cadence, versioning) | https://docs.litellm.ai/docs/proxy/release_cycle | +| Proxy quickstart | https://docs.litellm.ai/docs/proxy/quick_start | +| Docker quickstart (UI-first flow, DB-less caveats) | https://docs.litellm.ai/docs/proxy/docker_quick_start | +| Config reference (`config.yaml` settings) | https://docs.litellm.ai/docs/proxy/configs and https://docs.litellm.ai/docs/proxy/config_settings | +| Routing / load balancing | https://docs.litellm.ai/docs/routing and https://docs.litellm.ai/docs/proxy/load_balancing | +| Reliability (retries, fallbacks, cooldowns) | https://docs.litellm.ai/docs/proxy/reliability | +| Virtual keys | https://docs.litellm.ai/docs/proxy/virtual_keys | +| Budgets / rate limits / teams | https://docs.litellm.ai/docs/proxy/users | +| Caching | https://docs.litellm.ai/docs/proxy/caching | +| Guardrails | https://docs.litellm.ai/docs/proxy/guardrails/quick_start | +| Logging / observability | https://docs.litellm.ai/docs/proxy/logging | +| Prometheus metrics | https://docs.litellm.ai/docs/proxy/prometheus | +| Health endpoints | https://docs.litellm.ai/docs/proxy/health | +| Response headers | https://docs.litellm.ai/docs/proxy/response_headers | +| Exception mapping | https://docs.litellm.ai/docs/exception_mapping | +| Error diagnosis (provider vs gateway rule) | https://docs.litellm.ai/docs/proxy/error_diagnosis | +| Debugging | https://docs.litellm.ai/docs/proxy/debugging | +| Timeouts | https://docs.litellm.ai/docs/proxy/timeout | +| Production checklist | https://docs.litellm.ai/docs/proxy/prod | +| Deployment (Docker/Helm/K8s/Terraform) | https://docs.litellm.ai/docs/proxy/deploy | +| Security best practices | https://docs.litellm.ai/docs/proxy/security_best_practices | +| Public/private routes (Enterprise) | https://docs.litellm.ai/docs/proxy/public_routes | +| Master key rotations / salt key | https://docs.litellm.ai/docs/proxy/master_key_rotations | +| Image security / cosign | https://docs.litellm.ai/docs/proxy/docker_image_security | +| Enterprise features and support policy | https://docs.litellm.ai/docs/enterprise | +| Python SDK input params | https://docs.litellm.ai/docs/completion/input | +| Provider pages (env vars, model strings) | https://docs.litellm.ai/docs/providers | +| Model cost map (community-maintained) | `model_prices_and_context_window.json` at the BerriAI/litellm repo root | + +## Version observations (as of this refresh) + +- Latest stable release: **litellm 1.97.0** (published 2026-08-16), checked live on + 2026-08-22 via `importlib.metadata.version("litellm")`. Pre-releases v1.98.0-rc.1 + and v1.99.0-dev.* were visible upstream. Stable cadence is weekly since 1.84.0. +- The proxy requires the `[proxy]` extra (`pip install 'litellm[proxy]'`); a bare + install lacks websockets and friends. Python >=3.10 is required since 1.84.0. +- Known packaging gotcha verified on 1.97.0: the declared fastapi range admits a + breaking 0.141.x where the proxy fails at startup with + `ImportError: cannot import name 'get_flat_dependant'`; pinning + `fastapi==0.136.3` fixes it. +- Endpoint behavior verified live against a 1.97.0 proxy with a master key set: + `GET /health/liveliness` → 200 "I'm alive!" unauthenticated; `GET /health/readiness` + → 200 unauthenticated; `GET /v1/models` → 500 without auth, 200 with a bearer key, + returning `{"data": [...]}`; `GET /model/info` → 200 with a key and api_key values + redacted as `"*************"`. The proxy binds 0.0.0.0 by default. +- `litellm.__version__` no longer exists (lazy module attrs); use + `importlib.metadata.version("litellm")` or `litellm --version` for the CLI. +- Image tags are plain semver (`vX.Y.Z`) since 1.84.0: `-stable`/`-nightly` suffixes + are gone, `main-latest` is deprecated and no longer updated. GHCR images are + cosign-signed; docs also publish to docker.litellm.ai. +- Support policy (effective June 2026): only the four most recent stable minor lines + receive updates. +- Route lockdown (`public_routes`, `admin_only_routes`, `allowed_routes`) is an + Enterprise feature as of this refresh; JWT principals carry their own route lists. + +## Refresh procedure + +1. Check the releases page for the new stable; read its release notes for breaking + changes (`!` markers), changed defaults, and security fixes. +2. Re-install into a scratch venv (`pip install 'litellm[proxy]'==` plus the + fastapi pin if needed), start a proxy with a dummy-key config, and re-verify the + health endpoints with the bundled probe: + `scripts/litellm-health --url http://127.0.0.1: --check health --check readiness --check models --key --json`. +3. Update the version observations above and any version-pinned claims in SKILL.md + and references (CVE floor, `/metrics` auth, budget semantics, EE boundaries). +4. Re-run the bundled tests: `.venv/bin/python -m pytest litellm/tests/`. + +## Related skill sources + +- `ml-engineering` owns engine selection, quantization decisions, serving + methodology, and evaluation design — the layer above gateway operations. +- `vllm` and `llama-cpp` own operating those inference engines themselves; LiteLLM + routes to them via `openai/...`-style prefixes or dedicated ones (`hosted_vllm/`, + `vllm/`, `lm_studio/`). +- `kubernetes`, `docker-compose`, and `traefik` own the infrastructure and TLS + termination layers beneath a public proxy deployment. diff --git a/litellm/references/01-quickstart-and-sdk.md b/litellm/references/01-quickstart-and-sdk.md new file mode 100644 index 0000000..98ee2b8 --- /dev/null +++ b/litellm/references/01-quickstart-and-sdk.md @@ -0,0 +1,185 @@ +# LiteLLM Quickstart: Proxy Config and Python SDK + +> **Last Updated:** 2026-08-22 +> Sources: https://docs.litellm.ai/docs/proxy/quick_start , +> https://docs.litellm.ai/docs/proxy/configs , https://docs.litellm.ai/docs/completion/input , +> https://docs.litellm.ai/docs/proxy/user_keys + +This reference covers standing up a proxy with `config.yaml`, the config skeleton and +its top-level sections, endpoint surface, OpenAI-SDK drop-in usage, and the Python SDK +basics an operator needs. Scope: getting a correct deployment running and verified; +routing depth lives in [02-config-and-routing.md](02-config-and-routing.md). + +## Install and first run + +```bash +# The proxy server needs the [proxy] extra; bare litellm lacks websockets etc. +pip install 'litellm[proxy]' +litellm --version # CLI reports its version +``` + +Python >=3.10 is required since 1.84.0 (on 3.9 pip silently installs <=1.83.9). +Packaging gotcha verified on 1.97.0: if startup fails with +`ImportError: cannot import name 'get_flat_dependant' from 'fastapi.dependencies.utils'`, +the installed fastapi is too new for this litellm — pin `fastapi==0.136.3`. + +Three documented ways to start: + +```bash +litellm --config /path/to/config.yaml [--port 4000] [--detailed_debug] +litellm --model huggingface/bigcode/starcoder # single-model CLI mode +docker run -v $(pwd)/config.yaml:/app/config.yaml \ + -e LITELLM_MASTER_KEY=sk- -p 4000:4000 \ + ghcr.io/berriai/litellm:v1.97.0 --config /app/config.yaml +``` + +Success line to look for in the logs: `LiteLLM: Proxy initialized with Config, +Set models:` — its absence means the config did not load. The default bind is +`0.0.0.0:4000`; set `--host` deliberately for anything network-reachable. + +## Config skeleton and top-level sections + +```yaml +model_list: + - model_name: gpt-4o # name clients request (alias) + litellm_params: + model: azure/gpt-4o-eu # string sent to the provider layer + api_base: https://my-endpoint-europe.openai.azure.com/ + api_key: "os.environ/AZURE_API_KEY_EU" # os.environ/ prefix => getenv at load + rpm: 6 # per-deployment limit informs weighted pick + - model_name: "*" # wildcard catch-all (needs default creds in env) + litellm_params: + model: "*" + +litellm_settings: # SDK-wide behavior + drop_params: true # drop unsupported OPENAI params instead of erroring + num_retries: 3 + request_timeout: 600 # seconds; built-in default is 6000 on recent releases + success_callback: ["langfuse"] + +router_settings: # Router/load-balancer behavior + routing_strategy: simple-shuffle # default and recommended + model_group_alias: {"gpt-4": "gpt-4o"} + timeout: 30 # whole-call timeout passed to completion() + redis_host: os.environ/REDIS_HOST # required when >1 proxy instance shares state + +general_settings: # proxy-server settings + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL # or DATABASE_URL env var; both accepted + alerting: ["slack"] + background_health_checks: true + health_check_interval: 300 + +environment_variables: # extra env vars set inside the proxy process + LANGFUSE_PUBLIC_KEY: ... +``` + +Details that matter: + +- `os.environ/VARNAME` interpolation works for any value anywhere in the file. + Resolution happens **inside the proxy process** — a variable present in your shell + but not in the container produces opaque failures visible only via + `--detailed_debug`. +- There is no standalone schema validator command; validation is at load time. YAML + indentation/aliasing typos are the most common cause of "weird" behavior. +- Full spec is browsable as Swagger at `/#/config.yaml`. `NO_DOCS="True"` + disables that UI. +- With `store_model_in_db: true`, DB rows deep-merge over these YAML sections + (`general_settings`, `router_settings`, `litellm_settings`, `environment_variables`) + and win key conflicts; see [02-config-and-routing.md](02-config-and-routing.md). +- Enterprise license: `LITELLM_LICENSE` env var. + +## Endpoint surface + +| Route | Purpose | +|---|---| +| `/v1/chat/completions`, `/chat/completions` | Chat (OpenAI-compatible) | +| `/v1/completions` | Text completion | +| `/v1/embeddings`, `/embeddings` | Embeddings | +| `/v1/images/generations` | Image generation | +| `/v1/audio/transcriptions`, `/v1/audio/speech` | Transcription / TTS | +| `/responses` | OpenAI Responses API surface | +| `/messages`, `/anthropic/v1/messages` | Anthropic-compatible messages | +| `/v1/models` | Model aliases visible to the calling key (auth required when master_key set) | +| `/model/info` | Per-deployment detail incl. cost/max-token info (auth required) | +| `/health/liveliness` | Unauthenticated liveness → `"I'm alive!"` (spelling: liveliness) | +| `/health/readiness` | Unauthenticated readiness; 503 when the configured DB is unreachable | + +Verified against a live 1.97.0 proxy: with `master_key` set, `/v1/models` returns 500 +without auth and 200 with `Authorization: Bearer `; `/health/liveliness` and +`/health/readiness` are unauthenticated by design. A representative call: + +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer $LITELLM_VIRTUAL_KEY" \ + -H 'Content-Type: application/json' \ + -d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "Say hello"}]}' +``` + +The response carries `_response_ms` plus `x-litellm-*` headers (call id, model id, +resolved api_base, version) useful for forensics. + +## OpenAI-SDK drop-in (any OpenAI-compatible client) + +```python +import openai +client = openai.OpenAI( + api_key="sk-virtual-key", # virtual or master key, NOT a provider key + base_url="http://localhost:4000", # or https://gateway.example.com +) +resp = client.chat.completions.create( + model="gpt-4o", + messages=[{"role": "user", "content": "hello"}], + extra_body={"metadata": {"tags": ["production"]}}, # optional pass-through metadata +) +``` + +The same base-url swap works for LangChain (`ChatOpenAI`), LlamaIndex, Instructor, +Aider/LibreChat-style tools, and the Anthropic SDK pointed at the proxy's +`/messages` surface. Pass-through `metadata.tags` feed cost tracking and tag-based +features downstream. + +## Python SDK essentials + +```python +from litellm import completion, acompletion, embedding + +resp = completion( + model="openai/gpt-4o", # provider/prefixed model string + messages=[{"role": "user", "content": "hello"}], + # timeout defaults to 600s; unsupported OpenAI params raise unless dropped: + # drop_params=True here, or litellm.drop_params=True module-wide +) +resp.choices[0].message.content # dict-style access also works +resp.usage.total_tokens +resp._hidden_params["response_cost"] # USD cost from the model cost map + +async for chunk in await acompletion(model="gpt-4o", messages=msgs, stream=True): + print(chunk.choices[0].delta.content or "", end="") +``` + +Operator-relevant SDK facts: + +- Model strings carry a provider prefix (`openai/`, `anthropic/`, `azure/`, + `bedrock/`, `vertex_ai/`, `gemini/`); bare names are inferred only for well-known + families. Azure uses the **deployment name**, not the model name. +- Streaming chunks expose reasoning fields for reasoning models + (`delta.reasoning_content`, `thinking_blocks`); Anthropic thinking maps differ by + model generation — verify against the installed release. +- Cost/token helpers: `token_counter(model=..., messages=...)`, + `completion_cost(response)`, `get_max_tokens(model)`, and the `litellm.model_cost` + dict loaded from the community-maintained + `model_prices_and_context_window.json` (there is no file named `model_cost.json`). + Set `LITELLM_LOCAL_MODEL_COST_MAP="True"` to use the bundled copy offline. +- Check the installed version with `importlib.metadata.version("litellm")`; + `litellm.__version__` raises AttributeError on current releases. +- Prefer `get_model_info(model=...)` over the partial `litellm.supports_*()` exports + for capability flags like prompt caching. + +## Verification at the delivery boundary + +- Startup log shows `Proxy initialized with Config, Set models:`. +- `scripts/litellm-health --check health --check readiness --json` passes; readiness + failing with 503 means a configured DB is unreachable. +- `/v1/models` with the calling key lists the expected alias. +- One bounded chat request returns tokens and the expected `x-litellm-model-id`. diff --git a/litellm/references/02-config-and-routing.md b/litellm/references/02-config-and-routing.md new file mode 100644 index 0000000..5cb24c7 --- /dev/null +++ b/litellm/references/02-config-and-routing.md @@ -0,0 +1,146 @@ +# LiteLLM Config and Routing: model_list, Strategies, Reliability + +> **Last Updated:** 2026-08-22 +> Sources: https://docs.litellm.ai/docs/routing , +> https://docs.litellm.ai/docs/proxy/load_balancing , +> https://docs.litellm.ai/docs/proxy/reliability , +> https://docs.litellm.ai/docs/proxy/configs + +This reference covers how `model_list` entries become load-balanced groups, the +routing strategies, and the reliability machinery — retries, fallbacks, cooldowns, +ordering, pre-call checks. Scope: making one gateway name resilient across many +provider deployments; keys/budgets live in [03-keys-teams-budgets-spend.md](03-keys-teams-budgets-spend.md). + +## Model groups: same model_name = one LB group + +Multiple entries sharing a `model_name` form a routing group; requests for that name +are distributed across deployments. Each entry is a distinct deployment with an +auto-generated deterministic `model_id` (hash of its `litellm_params`) used for +health, cooldown, and header forensics. + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + weight: 1 + rpm: 100 + - model_name: gpt-4o + litellm_params: + model: azure/gpt-4o-eu # Azure: DEPLOYMENT name, not model name + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + weight: 2 # picked ~2x as often (simple-shuffle) + model_info: + base_model: openai/gpt-4o # correct context/cost math for Azure aliases +``` + +Details that matter: + +- `weight`, `rpm`, and `tpm` live under `litellm_params` and drive weighted picks. +- `model_info.base_model` fixes cost/context mapping when a provider alias echoes a + generic model name (Azure especially). +- `router_settings.model_group_alias` maps extra request names onto a group; + per-entry form supports `hidden: true` to keep aliases out of `/v1/models`. +- Wildcard entries (`model_name: "azure/*"`) expose whole provider families; pair + with key-level model grants. + +## Routing strategies + +| Strategy | Behavior | Notes | +|---|---|---| +| `simple-shuffle` (default) | Weighted random by rpm/tpm/weight | Docs recommend it for production performance | +| `least-busy` | Fewest in-flight requests | Good at high concurrency | +| `latency-based-routing` | Lowest avg latency | Tune via `routing_strategy_args: {ttl, lowest_latency_buffer}` | +| `usage-based-routing` | Lowest TPM usage this minute | Redis-tracked; docs warn against prod use (latency) | +| `cost-based-routing` | Cheapest per cost map | Missing models assumed $1 unless priced | + +There is no `routing_strategy: weighted` value — weighting rides on `simple-shuffle` +via `weight`/`rpm`. Newer releases add **routing groups** (`router_settings.routing_groups`) +to give specific groups their own strategy; group names are callable as model names, +appear in `/v1/models`, and must not collide with existing names. + +## Retries + +Precedence, highest first: request header `x-litellm-num-retries`, body +`num_retries`, per-deployment `num_retries` in `litellm_params`, +`litellm_settings.num_retries`. Rate-limit errors retry with exponential backoff; +a provider `retry-after` sets the minimum wait. + +Critical distinction: LiteLLM's `num_retries` is its own loop; the provider SDK's +`max_retries` is pinned to 0 through the router so retries don't multiply +`(1+N)^2`. Setting `max_retries` in a request body has no effect through the router. + +## Fallbacks + +Three families plus a default, executed in list order: + +```yaml +litellm_settings: + fallbacks: [{"zephyr-beta": ["gpt-4o"]}] + content_policy_fallbacks: [{"claude-2": ["my-fallback-model"]}] + context_window_fallbacks: [{"gpt-4o-mini": ["gpt-4o"]}] + default_fallbacks: ["claude-opus"] +``` + +- Fallback targets must be `model_name` aliases (or a specific deployment's + `model_info.id`), not provider strings — pointing them at raw provider strings is a + classic silent misconfig discovered mid-incident. +- Disable per request with `"disable_fallbacks": true` in the body. +- Context-window enforcement needs `router_settings.enable_pre_call_checks: true`; + without it oversized prompts go to the provider regardless. With it, prompts over a + deployment's limit raise ContextWindowExceededError locally before dispatch. +- Test fallback behavior by pointing one deployment at a deliberately bad key, + observing failover, then restoring. + +## Cooldowns + +Per-deployment, not per-group. Triggers: immediate cooldown on upstream 429; +failure-rate threshold within the current minute (`allowed_fails`, default 3); +non-retryable 401/404/408. Duration via `cooldown_time`; deployments recover +automatically and counters reset. Per-error-class tuning: + +```yaml +router_settings: + allowed_fails_policy: + RateLimitErrorAllowedFails: 100 + InternalServerErrorAllowedFails: 3 + cooldown_time: 30 +``` + +When every deployment of a group is cooling down clients see +`No deployments available for selected model, Try again in N seconds...` (HTTP 429). +Docs do not recommend `disable_cooldowns: true` — it routes over exhausted limits. +Note `allowed_fails` belongs under `model_info`/policy blocks rather than loose +`litellm_params` (loose params leak into the provider request body). + +## Deployment ordering and weighted failover + +- `order: 1 / order: 2` in `litellm_params` gives priority tiers: tier 1 absorbs + traffic until it fails/cools, then tier 2 serves; each tier gets its own retries + before escalation, and configured `fallbacks` apply after all tiers. +- `router_settings.enable_weighted_failover: true` re-picks among same-group peers + by weight on retryable failures, excluding already-failed ids (async calls only; + not triggered for context-window or content-policy errors). + +## Config-in-DB overlay semantics + +With `store_model_in_db: true` (env `STORE_MODEL_IN_DB="True"`), writes from UI/API +land in Postgres and deep-merge over YAML for `general_settings`, +`router_settings`, `litellm_settings`, and `environment_variables` — DB wins key +conflicts. Editing those sections in YAML later has no effect while a DB row exists +(delete the row or the setting to restore YAML control). Models added via UI land in +a dedicated table and load-balance alongside same-named YAML models rather than +replacing them. Cross-pod config sync is polling +(`proxy_config_reload_interval_seconds`, default 30). Without `store_model_in_db`, +YAML is fully authoritative. + +## Verification at the delivery boundary + +- `/v1/models` lists each alias once per group; `/model/info` shows one entry per + deployment with distinct `model_id`s. +- A representative request returns tokens and `x-litellm-model-id` identifies which + deployment served it; repeat a few times to observe weighted distribution. +- Force one failure path (bad key on a low-weight deployment) and confirm the + configured fallback/ordering actually fires before trusting it in production. diff --git a/litellm/references/03-keys-teams-budgets-spend.md b/litellm/references/03-keys-teams-budgets-spend.md new file mode 100644 index 0000000..9f6b917 --- /dev/null +++ b/litellm/references/03-keys-teams-budgets-spend.md @@ -0,0 +1,123 @@ +# LiteLLM Keys, Teams, Budgets, Rate Limits, and Spend + +> **Last Updated:** 2026-08-22 +> Sources: https://docs.litellm.ai/docs/proxy/virtual_keys , +> https://docs.litellm.ai/docs/proxy/users , +> https://docs.litellm.ai/docs/enterprise + +This reference covers the credential model (master key vs virtual keys), teams and +users, budget and rate-limit knobs with their enforcement semantics — including the +fail-open-without-DB trap — and spend tracking. Scope: governing who spends what; +routing mechanics live in [02-config-and-routing.md](02-config-and-routing.md). + +## Master key vs virtual keys + +- `general_settings.master_key` (or env `LITELLM_MASTER_KEY`) must start with `sk-`. + It is the admin API credential **and** the Admin UI password. If both config and + env are set, the config value wins. +- Virtual keys are minted with `POST /key/generate` under a master-key bearer and + returned once. They authorize and meter requests; they never contain provider + credentials, which stay in `model_list[].litellm_params` (`os.environ/...`) or, + with `STORE_MODEL_IN_DB=True`, encrypted in Postgres via `LITELLM_SALT_KEY`. +- Key lifecycle: `POST /key/generate`, `GET /key/info?key=...` (spend, expiry, + models), `POST /key/update`, `POST /key/block` / `/key/unblock` for instant + revocation, `/key/delete`. Regeneration with grace periods and scheduled + auto-rotation are Enterprise features. +- What a key inherits: model/MCP access is evaluated against the key row itself; + management-route power comes from the owner's role — an admin-owned key can hit + admin endpoints. Admin-created keys without an explicit `user_id` have no owner + and inherit nothing. +- Self-service guardrails: `litellm_settings.upperbound_key_generate_params` caps + what any caller can grant itself; `default_key_generate_params` fills omissions; + `key_generation_settings` restricts who may mint keys. Policy hooks: + `custom_key_generate` runs on generation only — pair it with `custom_key_update` + or edits bypass policy. + +```bash +curl -X POST 'http://localhost:4000/key/generate' \ + -H 'Authorization: Bearer sk-master' -H 'Content-Type: application/json' \ + -d '{"models": ["gpt-4o"], "max_budget": 50, "budget_duration": "30d", + "tpm_limit": 80000, "rpm_limit": 60, "duration": "90d"}' +``` + +## The database requirement — budgets fail open + +Keys, teams, budgets, spend logs, and UI state live in Postgres +(`DATABASE_URL`). Without a connected DB: + +- `max_budget` is **not enforced** — global spend cannot be loaded, one startup + warning is logged, requests keep serving past budget. +- `/key/*` endpoints fail with `No connected db.`. + +Never run a budget-sensitive deployment DB-less; bound spend upstream instead if you +must run DB-less. + +## Budgets + +Where things live: + +| Scope | Setting | Notes | +|---|---|---| +| Global proxy | `litellm_settings.max_budget` + `budget_duration` | Under litellm_settings, NOT general_settings | +| Team | `/team/new` fields `max_budget`, `budget_duration` | | +| Team member | `/team/member_add` with `max_budget_in_team` | | +| Internal user default | `litellm_settings.max_internal_user_budget` + duration | | +| Virtual key | `/key/generate` fields `max_budget`, `budget_duration` | Multi-window via `budget_limits: [{budget_duration, max_budget}, ...]` | +| End users/customers | `/budget/new` then `litellm_settings.max_end_user_budget_id` | Float `max_end_user_budget` is no longer enforced | + +Semantics that matter: + +- Crossing a hard budget fails requests (`ExceededBudget` / `ExceededTokenBudget` + errors); `soft_budget` warns without blocking. Resets are checked by a scheduler + roughly every 10 minutes (`proxy_budget_rescheduler_min_time/max_time`). +- **Team-key rule:** a key belonging to a team enforces only team (+ member) + budgets; the owner's personal budget does not apply. +- Cost reservation is ON by default: estimated max cost is reserved before the + provider call to prevent concurrency overspend. For hard ceilings across replicas + set `general_settings.fail_closed_budget_enforcement: true` (rejects with 503 when + Redis+DB cannot verify spend). +- Per-model budgets on keys/users are Enterprise. + +## Rate limits + +- Knobs on keys/teams/users: `tpm_limit`, `rpm_limit`, `max_parallel_requests`; + per-model dicts (`model_rpm_limit`, `model_tpm_limit`) supported. Proxy-wide + concurrency cap: `general_settings.global_max_parallel_requests`. +- Deployment-level `rpm`/`tpm` in `litellm_params` inform weighted routing by + default; to enforce them as hard limits add + `router_settings.optional_pre_call_checks: [enforce_model_rate_limits]` + (RPM exact; TPM best-effort). Needs Redis when multi-instance. +- TPM counting type: `general_settings.token_rate_limit_type: input|output|total`. +- Rate limits do **not** apply to proxy admins — test with an internal-user role. +- Remaining-quota headers: `x-litellm-key-remaining-requests[-]`, + `x-litellm-key-remaining-tokens[-]`. + +## Teams and users + +`POST /team/new` (with `members_with_roles`, limits), `/team/info`, +`/team/member_add`, `/team/update`; `POST /user/new`, `GET /user/info`. Roles: +PROXY_ADMIN, PROXY_ADMIN_VIEW_ONLY, ORG_ADMIN (EE), INTERNAL_USER, +INTERNAL_USER_VIEW_ONLY, TEAM, CUSTOMER. Model access groups +(`model_info.access_groups`) let keys/teams be granted a group name instead of +enumerated models. + +## Spend tracking + +- Every request writes a spend log row (tokens, cost, model, key hash, end user); + rollups land on key/user/team tables via LiteLLM's cost map. Query surfaces: + `GET /spend/logs`, `GET /global/spend`, plus the UI. +- `general_settings.disable_spend_logs` turns off per-transaction rows; + `store_prompts_in_spend_logs` (default **false**) opts into storing full + prompt/response content per row — a privacy decision, see + [07-security-and-public-hosting.md](07-security-and-public-hosting.md). +- Retention: `maximum_spend_logs_retention_period` (e.g. `30d`) plus a cleanup + interval. Batched writes via `proxy_batch_write_at`; high-RPS deployments should + enable the Redis transaction buffer. + +## Verification at the delivery boundary + +- Readiness 200 confirms DB connectivity; `/key/info` returns live spend for the key. +- A key restricted to one model gets a clean rejection requesting another model. +- Set a tiny test budget, exceed it, observe the documented error, then restore — + proving enforcement rather than assuming it (and confirming budgets are not + silently failing open). diff --git a/litellm/references/04-caching-and-guardrails.md b/litellm/references/04-caching-and-guardrails.md new file mode 100644 index 0000000..ab04830 --- /dev/null +++ b/litellm/references/04-caching-and-guardrails.md @@ -0,0 +1,105 @@ +# LiteLLM Caching and Guardrails + +> **Last Updated:** 2026-08-22 +> Sources: https://docs.litellm.ai/docs/proxy/caching , +> https://docs.litellm.ai/docs/proxy/guardrails/quick_start , +> https://docs.litellm.ai/docs/enterprise + +This reference covers response caching backends and controls, the semantic-caching +stale-multi-turn caveat, and guardrail configuration with Presidio PII masking. +Scope: choosing and configuring these correctly for a workload; privacy defaults +live in [07-security-and-public-hosting.md](07-security-and-public-hosting.md). + +## Response caching + +```yaml +litellm_settings: + cache: true + cache_params: + type: redis # production default for multi-instance + host: os.environ/REDIS_HOST + port: 6379 + password: os.environ/REDIS_PASSWORD + namespace: "litellm.caching.caching" + ttl: 600 + max_connections: 100 +``` + +Backends: in-memory (per-process — wrong for >1 replica), disk, redis, +redis-cluster (`redis_startup_nodes`), sentinel, S3/GCS, and semantic variants +(`qdrant-semantic`, `redis-semantic`, `valkey-semantic`). Env alternatives: +`REDIS_URL` or `REDIS_HOST/PORT/PASSWORD/SSL` (+ arbitrary `REDIS_`); +docs recommend `REDIS_*` over `REDIS_URL` in production. + +Controls that matter: + +- Cacheable call types default to completion/embedding-style routes; scope via + `cache_params.supported_call_types`. +- Per-request body controls: `"cache": {"ttl": 60, "s-maxage": 600, "no-cache": + true, "no-store": true, "namespace": "..."}`. Opt-in mode + (`cache_params.mode: default_off`) makes caching request-scoped only. +- Debug endpoints `/cache/ping` and `/cache/delete`; header `x-litellm-cache-key` + exposes the key used. +- Provider-specific optional params are excluded from cache keys by default; opt in + with `enable_caching_on_provider_specific_optional_params: true`. + +### Semantic caching caveat + +Semantic caches embed the **entire messages array** and serve nearest neighbors above +a similarity threshold. Consecutive agentic turns are often ~0.99 similar, so agents +get stale tool results replayed as cache hits — current docs warn against semantic +caching for multi-turn/agentic traffic outright. For agent workloads use exact-match +redis caching, exclude those keys from caching, or force `no-store` per request. + +## Guardrails + +```yaml +guardrails: + - guardrail_name: "presidio-pii" + litellm_params: + guardrail: presidio + mode: pre_call # pre_call | post_call | during_call | logging_only + presidio_language: en + pii_entities_config: + CREDIT_CARD: MASK + EMAIL_ADDRESS: MASK + US_SSN: BLOCK + presidio_score_thresholds: + CREDIT_CARD: 0.8 + EMAIL_ADDRESS: 0.6 + +litellm_settings: + guardrails: ["presidio-pii"] # or per-request "guardrails": [...] +``` + +Modes are event hooks: `pre_call` (before the LLM call), `post_call` (after, on +input+output), `during_call` (parallel with the LLM call, blocking until the check +completes), `logging_only`. List form (`mode: [pre_call, post_call]`) is valid. +Older material describing a single `"all"` mode is outdated. + +Behavior and invocation: + +- Blocking providers fail the request with HTTP 400 embedding the provider verdict; + masking providers (Presidio MASK) rewrite content instead of blocking. +- `default_on: true` runs a guardrail on every request regardless of client choice; + otherwise clients pass `"guardrails": ["name"]` in the body. +- Applied guardrails surface in `x-litellm-applied-guardrails` and in the logging + payload (`applied_guardrails`, `guardrail_information`, masked-entity counts) — + feed these to your SIEM. +- OSS vs Enterprise: the framework, custom guardrails, Presidio PII masking, and + always-on/request-scoped usage are free; several moderation integrations + (llmguard, llamaguard, hide_secrets, openai/google moderations, lakera prompt + injection, aporia prompt injection), per-key/per-team scoping, dynamic params, + tag-based modes, model-level attach, and team lock-downs require an Enterprise + license. +- `skip_system_message_in_guardrail` excludes system prompts on the unified path + (Presidio, Bedrock, content filter, OpenAI Moderations, generic API, custom + apply_guardrail); raw-hook providers are unaffected. + +## Verification at the delivery boundary + +- Send one identical request twice with caching enabled and confirm a cache hit + (`x-litellm-cache-key` present; latency drops; spend not double-counted). +- Send one request containing a masked entity through the presidio guardrail and + confirm the provider never sees the plaintext (check callback logs, bounded). +- Confirm `x-litellm-applied-guardrails` names what ran on each request. diff --git a/litellm/references/05-observability-and-logging.md b/litellm/references/05-observability-and-logging.md new file mode 100644 index 0000000..eabcde0 --- /dev/null +++ b/litellm/references/05-observability-and-logging.md @@ -0,0 +1,108 @@ +# LiteLLM Observability: Callbacks, Metrics, Headers, Privacy + +> **Last Updated:** 2026-08-22 +> Sources: https://docs.litellm.ai/docs/proxy/logging , +> https://docs.litellm.ai/docs/proxy/prometheus , +> https://docs.litellm.ai/docs/proxy/config_settings , +> https://docs.litellm.ai/docs/proxy/debugging + +This reference covers logging integrations (callbacks), Prometheus metrics, the +forensic response headers, debugging workflow switches, and the privacy flags that +decide what content leaves the proxy. Scope: seeing and controlling what happened; +error-specific fixes live in [08-troubleshooting.md](08-troubleshooting.md). + +## Callbacks + +```yaml +litellm_settings: + success_callback: ["langfuse"] # success-only + failure_callback: ["sentry"] # failure-only + callbacks: ["otel"] # both + service_callbacks: ["datadog", "prometheus"] # system health (redis/postgres/auth) + turn_off_message_logging: true # metadata yes, message content no + redact_user_api_key_info: true # redact key/user/team identifiers in traces +``` + +- Langfuse needs `LANGFUSE_PUBLIC_KEY/SECRET_KEY/HOST`; request `metadata` passes + through (`trace_id`, `tags`, ...). OTel needs `OTEL_EXPORTER=otlp_http|otlp_grpc|console`, + `OTEL_ENDPOINT`, `OTEL_HEADERS`; per-callback redaction via + `callback_settings.otel.message_logging: false`. +- Every event carries a standardized payload (`standard_logging_object`) documented + at the logging spec page — build dashboards/SIEM rules on it rather than scraping + free-text logs. + +## Prometheus metrics + +```yaml +litellm_settings: + callbacks: + - prometheus +``` + +- The `/metrics` endpoint **requires auth since v1.85.0**: configure the scraper + with `authorization: Bearer ` or open it explicitly with + `require_auth_for_metrics_endpoint: false`. Multiple workers need a writable + `PROMETHEUS_MULTIPROC_DIR`. +- Key series: `litellm_proxy_total_requests_metric`, + `litellm_proxy_failed_requests_metric`, `litellm_spend_metric`, + `litellm_deployment_success_responses/_failure_responses`, + `litellm_deployment_state` (0 healthy / 1 partial / 2 outage), + `litellm_deployment_cooled_down`, latency family including TTFT for streaming, + cache hit metrics, and budget gauges. +- Official Grafana dashboard JSON ships in the upstream repo's cookbook directory; + cardinality controls (`custom_prometheus_metadata_labels`, metric filtering) exist + for large fleets — end-user labels are opt-in for good reason. + +## Forensic response headers + +``` +x-litellm-call-id correlate one request across logs/callbacks +x-litellm-model-id which deployment served this request +x-litellm-model-api-base resolved provider base URL +x-litellm-version proxy version +x-litellm-response-cost computed USD cost +x-litellm-key-tpm-limit / x-litellm-key-rpm-limit applied limits +x-litellm-applied-guardrails (when guardrails ran) +``` + +Some cost-detail headers are documented as non-streaming only; verify which headers +survive on streamed responses for your release before building alerts on them. + +## Debugging workflow + +1. Reproduce through the proxy with `--detailed_debug` (CLI) or + `LITELLM_LOG=DEBUG`; logs show the resolved outbound curl (masked key) and raw + provider response. Single-request variant: `"litellm_request_debug": true` in the + body emits raw request/response for that request only. +2. Classify provider vs gateway from the error string (see + [08-troubleshooting.md](08-troubleshooting.md)). +3. Check what the router sees: `/v1/models`, `/model/info`, `/health?model=`, + `/health/readiness/details` (authenticated diagnostics). +4. Correlate with `x-litellm-call-id` in callback logs; enable JSON logs + (`json_logs: true`) and `request_correlation_in_logs` to stamp trace ids. +5. CLI helpers: `litellm --config config.yaml --health` health-checks configured + models; `--test` fires a test chat request. Keep debug off in production + (`LITELLM_LOG=ERROR`); `set_verbose` is deprecated. + +## Privacy switches + +| Flag | Effect | +|---|---| +| `store_prompts_in_spend_logs` (default false) | Opt-in full prompt/response storage in Postgres; raises memory floor | +| `turn_off_message_logging: true` | Metadata reaches callbacks, content does not | +| `redact_user_api_key_info: true` | Redacts hashed token/user/team info in supported callbacks | +| `"no-log": true` (per request) | Skips logging for that request (globally disableable) | +| UI Spend Log settings toggle | Overrides config-file values at runtime — audit it on managed deployments | + +The Admin UI can flip prompt storage on without a restart and without touching your +config file — treat the UI state as part of the effective configuration when +auditing privacy posture (details: +[07-security-and-public-hosting.md](07-security-and-public-hosting.md)). + +## Verification at the delivery boundary + +- One test request appears in each configured destination (Langfuse trace, OTel + span, `/metrics` counters move). +- With `turn_off_message_logging: true`, confirm prompts are absent from the + callback destination while metadata still arrives. +- `/metrics` scrape succeeds with the exact auth configuration production will use. diff --git a/litellm/references/06-deployment.md b/litellm/references/06-deployment.md new file mode 100644 index 0000000..d410dc5 --- /dev/null +++ b/litellm/references/06-deployment.md @@ -0,0 +1,137 @@ +# LiteLLM Deployment: Docker, Compose, Kubernetes, Scaling, Upgrades + +> **Last Updated:** 2026-08-22 +> Sources: https://docs.litellm.ai/docs/proxy/deploy , +> https://docs.litellm.ai/docs/proxy/prod , +> https://docs.litellm.ai/docs/proxy/docker_quick_start , +> https://docs.litellm.ai/docs/proxy/docker_image_security + +This reference covers running the proxy in production: images and pinning, the two +data stores and what breaks without them, Compose/Kubernetes/Helm patterns, +multi-instance mechanics, migrations, and upgrade/rollback practice. Scope: the +LiteLLM-specific layer; cluster fundamentals belong to `kubernetes` / +`docker-compose`. + +## Images and pinning + +```bash +docker run -v $(pwd)/config.yaml:/app/config.yaml \ + -e DATABASE_URL=... -e LITELLM_MASTER_KEY=sk-... -e LITELLM_SALT_KEY=sk-... \ + -p 4000:4000 ghcr.io/berriai/litellm:v1.97.0 --config /app/config.yaml +``` + +- Registries: `ghcr.io/berriai/litellm` (Helm default) mirrored at + `docker.litellm.ai/berriai/litellm`. Variants include `-database` (bundled Prisma + toolchain) and `-non_root`. +- Tag policy since 1.84.0: plain semver (`vX.Y.Z`), immutable and cosign-signed. + The `-stable`/`-nightly` suffix scheme is gone; `main-latest` is deprecated — + never ship it. Pin tag or digest; verify signatures: + `cosign verify --key https://raw.githubusercontent.com/BerriAI/litellm//cosign.pub ghcr.io/berriai/litellm:`. +- Support policy: only the four most recent stable minor lines receive updates. + +## Core environment + +```bash +DATABASE_URL="postgresql://.../litellm" # keys, teams, spend, budgets, UI state +LITELLM_MASTER_KEY="sk-..." # admin credential + UI password +LITELLM_SALT_KEY="sk-..." # encrypts DB-stored provider credentials +STORE_MODEL_IN_DB="True" # manage models via UI/API (DB overlay) +DISABLE_SCHEMA_UPDATE="true" # pods never migrate; a migration job does +``` + +`LITELLM_SALT_KEY` must be set once and **never rotated** after models are added — +stored credentials become unreadable, with no migration path. + +## Data stores and what breaks without them + +| Store | Used for | Without it | +|---|---|---| +| PostgreSQL | Keys, teams, users, spend logs, budgets, config-in-DB, UI state | No virtual keys/spend/budgets; master-key-only auth; budgets fail open | +| Redis >=7 | Cross-instance rate-limit counters, router cooldowns/usage, response cache, auth cache | Per-instance state only; "works on pod 1, fails on pod 2" bugs | + +## Docker Compose quickstart + +The one-line bootstrap (`curl -sSL https://docs.litellm.ai/docker-compose.yml | +docker compose -f - up -d`) starts gateway + Postgres; log into `/ui` as `admin` +with the master key. For anything beyond evaluation, write your own compose file +with: pinned image tag, Postgres healthcheck plus +`depends_on: {condition: service_healthy}` to avoid the Prisma cold-start race, env +files outside git, and a named volume for Postgres data. + +## Kubernetes / Helm + +Two official charts: + +1. **Monolithic** `litellm-helm`: + `helm install litellm oci://ghcr.io/berriai/litellm-helm -f values.yaml`. + Supports HPA or KEDA (mutually exclusive), PDBs, ServiceMonitor, graceful drain, + and a migrations Job hook. Chart versions track LiteLLM releases. +2. **Microservices** chart (from v1.89.0): gateway (:4000) + backend (:4001) + + ui (:3000) scaled independently; requires external Postgres/Redis; pin chart + versions that resolve to existing component image tags. + +Both charts run migrations via Job with `DISABLE_SCHEMA_UPDATE=true` on pods. +Probes: use `/health/liveliness` for liveness and `/health/readiness` for readiness; +readiness reports 503 while the DB is unreachable, which is exactly what you want +traffic to avoid. Raw-manifest equivalents are documented upstream; Terraform +modules exist for AWS (ECS Fargate/Aurora/ElastiCache/ALB) and GCP +(Cloud Run/Cloud SQL/Memorystore). + +## Multi-instance mechanics + +- Stateless gateway replicas share Postgres + Redis and run the same master key; + cooldowns and rate-limit counters live in Redis + (`router_settings.redis_host/port/password`). Config-in-DB sync across pods is + polling (`proxy_config_reload_interval_seconds`, default 30). +- Background jobs register per worker process; without coordination they run on + every pod. Split traffic from jobs with `LITELLM_JOB_ROLE=serving` on serving + pods plus one dedicated `LITELLM_JOB_ROLE=worker` replica so budget resets and + cleanups execute once. Stagger jobs after rollouts with + `scheduled_job_stagger.window_seconds`. +- Connection math: Prisma pool is per worker — size it + `MAX_DB_CONNECTIONS / (instances x workers)` (default pool 10). A default Helm + `maxReplicas=100` can demand ~1000 connections; derive maxReplicas from DB + capacity instead. +- Spend writes batch (`proxy_batch_write_at`); at high RPS enable the Redis + transaction buffer and watch its queue gauges. + +## Workers, sizing, runtime hygiene + +- One Uvicorn worker per pod on Kubernetes (`--num_workers 1`) so CPU-based HPA + reads cleanly; on VMs size workers to vCPUs. Memory floor ~4Gi per worker (the + Prisma engine high-water mark ratchets); recycle long-running workers with + `--max_requests_before_restart`. Autoscale on CPU (~60% target); leave memory + targets unset because of the ratchet. +- `LITELLM_MODE=PRODUCTION` disables `.env` loading; JSON logs via + `json_logs: true`; keep `LITELLM_LOG=ERROR` in prod. +- Non-root / read-only rootfs is fully supported: non-root image variant or + `runAsNonRoot` + `readOnlyRootFilesystem` with writable emptyDirs for UI assets, + migration dir, and cache paths (documented in the production checklist). +- Graceful degradation options: `allow_requests_on_db_unavailable` (requests + proceed during DB outages; use deliberately) and the drain endpoint for K8s + preStop hooks (keep the port cluster-internal). + +## Migrations and upgrades + +- `prisma migrate deploy` runs at startup by default (no shadow DB, no drift + detection). In orchestrated deployments prefer a dedicated migration job (Helm + PreSync/ArgoCD hook) with `DISABLE_SCHEMA_UPDATE=true` on all serving pods. + Migration files ship in the `litellm-proxy-extras` package, so older cores keep + their own migrations during rolling upgrades. +- Upgrade path: read release notes for the full version span (breaking commits are + marked with `!`), take a DB backup before migrating, rehearse on a scratch + instance with real config, then roll serving pods forward keeping the jobs + deployment in lockstep. Rollback = previous pinned image + previous config + record; do not assume cross-version config compatibility without re-validation. +- Behavioral changes recent enough to bite upgrades: `/metrics` auth default flipped + in 1.85.0; team-key budget hierarchy churned across 1.94.0–1.95.0; deprecated + flags (`USE_PRISMA_MIGRATE`, `set_verbose`) were removed. + +## Verification at the delivery boundary + +- Pods pass `/health/liveliness` and `/health/readiness`; readiness failing means + fix the DB first, not the probes. +- `scripts/litellm-health --check models --key ` lists expected aliases through + the service route (not just inside the cluster). +- One representative request returns tokens; `x-litellm-version` matches the pinned + tag you intended to deploy. diff --git a/litellm/references/07-security-and-public-hosting.md b/litellm/references/07-security-and-public-hosting.md new file mode 100644 index 0000000..3b38d65 --- /dev/null +++ b/litellm/references/07-security-and-public-hosting.md @@ -0,0 +1,138 @@ +# LiteLLM Security and Public-Facing Hosting + +> **Last Updated:** 2026-08-22 +> Sources: https://docs.litellm.ai/docs/proxy/security_best_practices , +> https://docs.litellm.ai/docs/proxy/public_routes , +> https://docs.litellm.ai/docs/proxy/master_key_rotations , +> https://docs.litellm.ai/blog/cve-2026-42208-litellm-proxy-sql-injection , +> https://docs.litellm.ai/blog/security-hardening-april-2026 , +> https://docs.litellm.ai/blog/security-update-march-2026 + +This reference covers hardening an internet-reachable proxy: the CVE floor, auth +model, route exposure, secrets and salt-key discipline, supply-chain posture, +privacy defaults, and abuse controls. Scope: LiteLLM-specific security; TLS +termination and network plumbing belong to `traefik`/`kubernetes`. + +## Version floor: >=1.83.7 + +Any internet-reachable proxy must run **litellm >=1.83.7** (and Starlette >=1.0.1): + +| Vuln | Type | Auth needed | Fixed | +|---|---|---|---| +| CVE-2026-42208 | Pre-auth SQL injection via crafted Authorization header; read/modify DB incl. keys | None (Critical, CISA KEV, actively exploited 2026) | v1.83.7 | +| CVE-2026-42203 | SSTI in `/prompts/test` → code exec in proxy process | Valid key | v1.83.7 | +| CVE-2026-42271 | Command injection in MCP stdio test endpoints | Valid key (CISA KEV) | v1.83.7 | +| CVE-2026-48710 | Starlette host-header bypass; chained with 42271 → unauthenticated RCE | None | Starlette >=1.0.1 + litellm >=1.83.7 | +| CVE-2026-35030 | OIDC userinfo cache collision → session inheritance (only with `enable_jwt_auth`) | None | v1.83.0 | +| CVE-2026-35029 | `/config/update` missing role check → any key could change runtime config | Any key | v1.83.0 | + +Two of these sat in CISA KEV during 2026 with exploitation observed within days of +disclosure — a public proxy below the floor should be treated as compromised until +patched and its provider keys rotated. + +## Supply-chain posture + +- March 2026 incident: backdoored PyPI wheels `litellm==1.82.7` and `1.82.8` + (~40 minutes; credential stealer harvesting env vars, SSH keys, cloud/k8s creds). + Official Docker-image users were unaffected. Clean builds resumed at v1.83.0 via a + rebuilt CI pipeline. +- Consequences for operators: pin exact versions or digests; prefer the cosign-signed + official images over unpinned pip installs; verify signatures in CI/admission; + never `pip install litellm` unversioned on a shared host. +- Images are cosign-signed since v1.83.0 with the pinned-commit public key shown in + every release body. + +## Authentication model + +- With Postgres connected, clients authenticate with virtual keys; without one, the + master key is the only credential. Health probes (`/health/liveliness`, + `/health/readiness`) are deliberately unauthenticated and low-detail. +- Always set a strong random master key (`sk-` + 32+ random bytes). The quickstart's + `sk-1234` placeholder is fingerprinted by vulnerability scanners, and the login + page advertises default credentials unless hidden — never ship it beyond throwaway + local testing. +- The Admin UI is effectively equivalent to holding the master key: restrict it to + admin networks, prefer SSO (EE beyond 5 users), or set `DISABLE_ADMIN_UI=True` on + API-only edges. +- Key-management sharp edges: management-route power follows the key **owner's + role** (an admin-owned virtual key can manage the proxy); `custom_key_generate` + policy hooks do not run on updates unless paired with `custom_key_update`. +- Enterprise-only auth extras: SSO/SAML/SCIM beyond 5 users, JWT/OIDC auth, + audit logs, IP allowlists. + +## Route exposure + +Routes that must never be publicly exposed: `/key/*`, `/user/*`, `/team/*`, +`/config/*`, `/model/*`, `/spend/*`, `/ui`, `/prompts/test`, `/mcp-rest/*`. Each of +these maps to a real incident class above (config write = takeover; prompt-test SSTI; +MCP test command injection). + +Route lockdown settings (`public_routes`, `admin_only_routes`, `allowed_routes`) +are **Enterprise** as of this refresh — do not present them as generally available. +The OSS path is enforcing at the reverse proxy: expose only the LLM route groups you +serve plus health probes, and deny management paths at the edge before they reach the +proxy. Terminate TLS at the LB/reverse proxy; never publish port 4000 raw. + +## Secrets, salt key, rotations + +- Provider credentials live only as `os.environ/VAR` references in config or in a + secret manager; nothing secret belongs in `config.yaml` or git. +- `LITELLM_SALT_KEY` encrypts DB-stored provider credentials. Set once, store in a + secret manager, never rotate after adding models (stored data becomes unreadable). +- Master-key rotation: if a salt key is set, rotate by changing the secret and + restarting — not via the regenerate flow, which would re-encrypt stored + credentials under a key the proxy then cannot use. Back up the DB before any + rotation flow. +- Virtual keys are hashed in the DB (hashing survives master-key rotation); instant + revocation is block/unblock; grace-period regeneration and scheduled rotation are + Enterprise. +- Keep Postgres and Redis on private subnets with TLS and least-privilege roles — + `DATABASE_URL` grants direct read/write to keys, budgets, and spend logs. + +## Data privacy defaults + +- Self-hosting sends nothing to BerriAI; requests do flow to whichever providers are + configured — residency comes from provider/region choice and guardrails. +- `store_prompts_in_spend_logs` defaults to false; spend logs carry metadata only. + Enabling it stores full messages/responses per row. The Admin UI Spend Log toggle + overrides config values at runtime — audit UI state on managed deployments. +- `turn_off_message_logging: true` keeps content out of callbacks; + `redact_user_api_key_info: true` redacts identity hashes in traces; + `overwrite_user_with_key_hash: true` stops caller-controlled `user` fields from + reaching providers. +- Presidio PII masking (OSS) can mask emails/cards/SSNs pre-dispatch — see + [04-caching-and-guardrails.md](04-caching-and-guardrails.md). + +## Abuse and cost control + +Budgets and rate limits are abuse controls as much as finance tools — a stolen +gateway key is stolen provider quota: + +- Global budget as circuit breaker; per-key caps sized ~2x expected load with alerts + at 80%; `upperbound_key_generate_params` so self-service cannot out-cap you; + end-user budgets via `max_end_user_budget_id`; rate limits on every public-facing + key (admins exempt — test accordingly). +- Budgets require Postgres and fail open without one (see + [03-keys-teams-budgets-spend.md](03-keys-teams-budgets-spend.md)). +- Slack/email alerting covers budget crossings, DB failures, hanging requests, and + outages; Prometheus deployment-state metrics catch cooldown cascades. + +## Hardening checklist (condensed) + +1. Pinned image/digest >=1.83.7, cosign verified, within the supported four-line window. +2. Strong master key from a secret manager; `sk-1234` nowhere; Admin UI restricted or disabled. +3. Scoped virtual keys per workload with expiry, budgets, rpm/tpm; block/unblock ready. +4. Edge exposes only LLM routes + health probes; management paths denied at the proxy; TLS terminated up front. +5. Postgres/Redis private, TLS, least privilege; DB pool bounded by instance math. +6. Salt key set once; documented master-key rotation flow rehearsed; DB backups before migrations. +7. Prompt-retention posture decided explicitly; message logging off where not needed. +8. Budgets + rate limits + alerting live; deployment-state and spend dashboards wired. + +## Verification at the delivery boundary + +- From outside the trust boundary: management routes return 403/404, health probes + answer, and no endpoint echoes configuration details. +- `scripts/litellm-health --check readiness` confirms DB connectivity without leaking + diagnostics; richer diagnostics stay behind auth. +- A revoked (blocked) key fails immediately; a key over its tiny test budget is + rejected with the documented error. diff --git a/litellm/references/08-troubleshooting.md b/litellm/references/08-troubleshooting.md new file mode 100644 index 0000000..9c4c7f8 --- /dev/null +++ b/litellm/references/08-troubleshooting.md @@ -0,0 +1,154 @@ +# LiteLLM Troubleshooting: Error Taxonomy, Failure Modes, Debugging + +> **Last Updated:** 2026-08-22 +> Sources: https://docs.litellm.ai/docs/proxy/error_diagnosis , +> https://docs.litellm.ai/docs/exception_mapping , +> https://docs.litellm.ai/docs/proxy/debugging , +> https://docs.litellm.ai/docs/proxy/timeout + +This reference is the evidence-led playbook for diagnosing proxy and SDK failures: +the provider-vs-gateway rule, the exception taxonomy, the common failure modes with +their exact error strings, and the debugging loop. Scope: diagnosis and fixes; +observability plumbing lives in [05-observability-and-logging.md](05-observability-and-logging.md). + +## The master diagnostic rule + +**If the error contains `Exception`, it came from the provider — not the +gateway.** `AnthropicException`, `OpenAIException`, `AzureException`, +`BedrockException`, `VertexAIException` mean the upstream call happened; the +provider's response is your evidence. No provider name in the error means LiteLLM +itself rejected or failed the call (bad LiteLLM key, unknown model name, cooldowns, +budget). + +- Provider example: `litellm.BadRequestError: BedrockException - {...validation...}` +- Gateway example: `Invalid API Key. Please check your LiteLLM API key.` with + `"type": "auth_error"` — that is your **LiteLLM** key being wrong. +- An opaque gateway-side 500 often hides a provider auth failure (a key present in + your shell but not in the container); read the proxy's debug logs rather than the + client exception. + +## Exception taxonomy (SDK + proxy surface) + +All importable from `litellm`; all carry `.status_code`, `.message`, `.llm_provider`; +most inherit OpenAI exceptions so existing client handlers keep working. + +| Status | Exception | Notes | +|---|---|---| +| 400 | `BadRequestError` | Base 400 | +| 400 | `UnsupportedParamsError` | Unsupported OpenAI param passed (drop via `drop_params`) | +| 400 | `ContextWindowExceededError` | Exists to enable context-window fallbacks | +| 400 | `ContentPolicyViolationError` | Enables content-policy fallbacks | +| 401 | `AuthenticationError` | Provider or gateway auth failure — apply the rule above | +| 403 | `PermissionDeniedError` | Includes EE route restrictions | +| 404 | `NotFoundError` | Invalid model name for the calling key | +| 408 | `Timeout` | Call exceeded timeout/stream_timeout | +| 422 | `UnprocessableEntityError` | Malformed request values | +| 429 | `RateLimitError` | Provider, key/team, or router cooldown exhaustion | +| 500 | `APIConnectionError` / `InternalServerError` | Unmapped errors incl. Anthropic's HTTP 529 overload | +| 503 | `ServiceUnavailableError` | Upstream unavailable | +| n/a | `BudgetExceededError` | Proxy-side budget exhausted | + +Retryability helper: `litellm._should_retry(status_code)`. + +## Failure modes: string → cause → fix + +### A) "No deployments available for selected model, Try again in N seconds" + +HTTP 429 from the router. Causes: every deployment of the group is in cooldown +(usually after upstream 429 storms), or a deployment is misconfigured so no valid +one exists. Fixes: check `/health?model=` per deployment; correct missing +provider prefixes (`model: gemini/gemini-2.5-flash`, not bare names); raise +`cooldown_time`/tune `allowed_fails_policy`; do not reach for +`disable_cooldowns: true` — docs warn it routes over exhausted limits. + +### B) "Invalid model name passed in model=X. Call /v1/models to view available models" + +The requested alias is not registered or not granted to this key. Fixes: compare +with `GET /v1/models` **using the same key** (model grants are per key); add the +missing `model_name` entry; fix the client's model string. Related SDK variant: +`LLM Provider NOT provided...` means a missing `provider/` prefix on the model +string. + +Gemini-specific gotcha: without the `gemini/` prefix a Gemini model string can route +to Vertex AI and demand GCP credentials — a classic first-config 401. + +### C) Authentication errors — disambiguate first + +Gateway 401 (`auth_error`, no provider name): bad or absent LiteLLM key. Verify +which value actually resolved — `general_settings.master_key` in config overrides +the `LITELLM_MASTER_KEY` env var. Provider 401 (`Exception ... Incorrect +API key provided`): the provider credential in the proxy process is wrong or absent; +confirm inside the container (`printenv`), not in your shell. + +Master-key rotation hazard: with `LITELLM_SALT_KEY` set, rotate by changing the +secret and restarting; the regenerate flow re-encrypts stored credentials under an +unusable key and bricks the deployment. Symptom of salt-key trouble at startup: +`Error decrypting value`. + +### D) 429 rate limits — three different sources + +Read the wording: key/team rpm/tpm rejections come before any provider call and +name the limit; budget exhaustion raises Budget/TokenBudget errors naming spend vs +max (check `GET /key/info`); provider 429 carries `Exception`, gets retried +with backoff, then cools the deployment down (mode A). Remember budgets fail open +without a DB and rate limits don't apply to admins. + +### E) ContextWindowExceededError + +Mapping to this exception is best-effort across providers — some overflows surface +as generic BadRequestError. Prefer preventing dispatch entirely: +`router_settings.enable_pre_call_checks: true` enforces context windows pre-call; +per-deployment `model_info.max_input_tokens` overrides detection; Azure needs +`model_info.base_model` set for correct window/cost mapping. Remedies ladder: +context-window fallbacks → client-side truncation/summarization → larger-context +deployment. + +### F) Timeouts and hanging streams + +Knobs: `router_settings.timeout` (whole call), +`litellm_settings.request_timeout` (recent releases default to 6000s — bound it), +per-deployment `timeout` and `stream_timeout` (time-to-first-chunk guard). +Idle load-balancers killing silent streams are mitigated by SSE keepalive pings +(`keepalive_seconds`). Known sharp edges around stream_timeout enforcement have been +reported on specific versions — pin and verify on yours. Long non-streaming calls +behind LBs can hit 504s; prefer streaming for long generations. + +### G) Connection errors and startup failures + +`APIConnectionError` is the catch-all unmapped mapping. Diagnosis order: +`--detailed_debug` shows the resolved outbound curl (masked); verify egress/DNS/ +proxy env vars from inside the pod; if no provider request-id appears anywhere, the +call never left the proxy. Startup `ImportError: cannot import name +'get_flat_dependant'` = fastapi too new for the pinned litellm (pin +`fastapi==0.136.3` on 1.97.0). Startup `Error decrypting value` = salt-key problem +(see C). + +### H) Streaming failures + +Mid-stream stalls can be misclassified as read timeouts; partial-chunk decode errors +have appeared on specific provider paths in specific versions. Fallbacks fire on +stream *start* failures reliably but mid-stream fallback behavior has varied across +releases — test your pinned version. Client disconnects mid-stream can leave +incomplete spend records; reconcile against callbacks if billing-grade accuracy +matters. + +## Debugging workflow + +1. Reproduce through the proxy with `--detailed_debug` or `LITELLM_LOG=DEBUG`; the + log shows the outbound request (masked) and raw response. Per-request: + `"litellm_request_debug": true`. +2. Classify with the master rule; capture the full error string, status, and + `x-litellm-call-id`. +3. Inspect router state: `/v1/models` (same key!), `/model/info`, + `/health?model=`, `/health/readiness/details`. +4. Read response headers: which deployment served (`x-litellm-model-id`), what + api_base was used, retries/fallbacks attempted. +5. Fix the smallest thing consistent with the evidence; verify with the health probe + plus one representative request; record the incident in the config/deployment + template so the next operator inherits the knowledge. + +## Verification at the delivery boundary + +A diagnosis counts as sound only when the fix was observed working through the same +boundary the client uses: probe green, one bounded chat request returning tokens, +and the original failing request shape succeeding again. diff --git a/litellm/scripts/litellm-health b/litellm/scripts/litellm-health new file mode 100755 index 0000000..cc9ce8c --- /dev/null +++ b/litellm/scripts/litellm-health @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""litellm-health - read-only probe for a LiteLLM AI gateway (proxy server). + +Answers, over HTTP(S), four operational questions about a running proxy and +nothing else: Is it alive? Is it ready (database reachable)? Which model +aliases does this key see? How many deployments are registered? The probe +issues GET requests only, never writes files, never mutates proxy state, and +never sends data anywhere except the gateway you point it at. + +Standard library only; --help works with no gateway running. Output is +bounded JSON with --json or short human-readable lines otherwise. + +Exit codes: 0 all checks passed, 1 issues found or a fatal error, +2 usage error, 124 timeout. +""" +import argparse +import json +import socket +import sys +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, Callable, Dict, List, Optional + +PROBE_NAME = "litellm-health" +DEFAULT_URL = "http://127.0.0.1:4000" +DEFAULT_TIMEOUT = 10 +MODEL_INFO_BOUND_BYTES = 64 * 1024 + +CHECKS = ( + "health", + "readiness", + "models", + "model_info", +) + +# Checks whose routes sit behind master-key/virtual-key authentication. +KEYED_CHECKS = frozenset({"models", "model_info"}) + + +class ProbeTimeout(Exception): + """A single request exceeded its time budget.""" + + +def parse_args(argv: List[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog=PROBE_NAME, + description=( + "Read-only probe for a running LiteLLM AI gateway: liveliness, " + "readiness, visible model aliases, and bounded deployment info." + ), + ) + parser.add_argument( + "--url", + default=DEFAULT_URL, + help=f"Base URL of the LiteLLM gateway (default: {DEFAULT_URL})", + ) + parser.add_argument( + "--check", + action="append", + choices=list(CHECKS), + help="Run only these checks; repeat the flag for several (default: all)", + ) + parser.add_argument("--json", action="store_true", help="Emit bounded JSON output") + parser.add_argument( + "--timeout", + type=float, + default=DEFAULT_TIMEOUT, + help=f"Seconds allowed per request (default: {DEFAULT_TIMEOUT})", + ) + parser.add_argument( + "--key", + default=None, + help="Bearer key (master or virtual) needed by models/model_info", + ) + return parser.parse_args(argv) + + +class Gateway: + """Thin GET-only view of one gateway's HTTP surface.""" + + def __init__(self, base_url: str, timeout: float, key: Optional[str]): + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self.key = key + + def get(self, path: str) -> tuple[int, bytes]: + url = urllib.parse.urljoin(self.base_url + "/", path.lstrip("/")) + headers = {"Authorization": f"Bearer {self.key}"} if self.key else {} + request = urllib.request.Request(url, method="GET", headers=headers) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + return response.status, response.read() + except (socket.timeout, TimeoutError) as error: + raise ProbeTimeout( + f"request timed out after {self.timeout}s" + ) from error + except urllib.error.HTTPError as error: + return error.code, error.read() + except urllib.error.URLError as error: + raise RuntimeError(f"connection failed: {error.reason}") from error + + +def _auth_hint(result: Dict[str, Any], status: int) -> Dict[str, Any]: + if status in (401, 403): + result["hint"] = "route requires --key " + return result + + +def check_health(gateway: Gateway) -> Dict[str, Any]: + status, body = gateway.get("/health/liveliness") + text = body.decode("utf-8", errors="replace") + # A live 1.97.0 gateway answers this unauthenticated route with 200 "I'm alive!". + ok = status == 200 and "alive" in text.lower() + return {"name": "health", "status_code": status, "ok": ok, "body": text[:200]} + + +def check_readiness(gateway: Gateway) -> Dict[str, Any]: + status, body = gateway.get("/health/readiness") + result: Dict[str, Any] = { + "name": "readiness", + "status_code": status, + "ok": status == 200, + } + if status == 200: + try: + payload = json.loads(body.decode("utf-8")) + if isinstance(payload, dict): + result["status"] = payload.get("status", "") + result["db"] = payload.get("db", "") + except (ValueError, TypeError): + pass + elif status == 503: + result["hint"] = "503 usually means the configured database is unreachable" + return result + + +def check_models(gateway: Gateway) -> Dict[str, Any]: + status, body = gateway.get("/v1/models") + aliases: List[str] = [] + if status == 200: + try: + payload = json.loads(body.decode("utf-8")) + entries = payload.get("data", []) if isinstance(payload, dict) else [] + aliases = [ + str(entry["id"]) + for entry in entries + if isinstance(entry, dict) and entry.get("id") + ] + except (ValueError, TypeError, KeyError): + aliases = [] + result = { + "name": "models", + "status_code": status, + "ok": bool(aliases), + "model_ids": aliases[:20], + } + return _auth_hint(result, status) + + +def check_model_info(gateway: Gateway) -> Dict[str, Any]: + status, raw = gateway.get("/model/info") + raw = raw[: MODEL_INFO_BOUND_BYTES + 1] + truncated = len(raw) > MODEL_INFO_BOUND_BYTES + deployments = 0 + if status == 200: + try: + payload = json.loads(raw.decode("utf-8", errors="replace")) + if isinstance(payload, list): + deployments = len(payload) + elif isinstance(payload, dict) and isinstance(payload.get("data"), list): + deployments = len(payload["data"]) + except (ValueError, TypeError): + deployments = 0 + result = { + "name": "model_info", + "status_code": status, + "ok": status == 200, + "deployment_count": deployments, + "truncated": truncated, + } + return _auth_hint(result, status) + + +CHECK_RUNNERS: Dict[str, Callable[[Gateway], Dict[str, Any]]] = { + "health": check_health, + "readiness": check_readiness, + "models": check_models, + "model_info": check_model_info, +} + + +def run_checks( + gateway: Gateway, selected: List[str], key: Optional[str] +) -> List[Dict[str, Any]]: + results: List[Dict[str, Any]] = [] + for name in selected: + if name in KEYED_CHECKS and not key: + results.append( + { + "name": name, + "ok": False, + "skipped_missing_key": True, + "hint": "requires --key ", + } + ) + continue + try: + results.append(CHECK_RUNNERS[name](gateway)) + except ProbeTimeout as error: + results.append( + {"name": name, "ok": False, "timed_out": True, "error": str(error)} + ) + except RuntimeError as error: + results.append({"name": name, "ok": False, "error": str(error)}) + return results + + +def render_text(results: List[Dict[str, Any]]) -> None: + for result in results: + verdict = "OK" if result.get("ok") else "FAIL" + detail = result.get("error") or result.get("status_code") or "" + print(f"[{verdict}] {result.get('name')} {detail}") + for alias in result.get("model_ids", []): + print(f" model: {alias}") + hint = result.get("hint") + if not result.get("ok") and hint: + print(f" hint: {hint}") + + +def exit_code(results: List[Dict[str, Any]]) -> int: + if any(result.get("timed_out") for result in results): + return 124 + if any(not result.get("ok") for result in results): + return 1 + return 0 + + +def main(argv: List[str]) -> int: + try: + args = parse_args(argv) + except SystemExit as error: + return int(error.code) if error.code is not None else 2 + + if args.timeout <= 0: + print("error: --timeout must be positive", file=sys.stderr) + return 2 + + gateway = Gateway(args.url, args.timeout, args.key) + results = run_checks(gateway, args.check or list(CHECKS), args.key) + + if args.json: + print(json.dumps({"url": gateway.base_url, "checks": results}, indent=2)) + else: + render_text(results) + return exit_code(results) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/litellm/templates/proxy-config-record.md b/litellm/templates/proxy-config-record.md new file mode 100644 index 0000000..9662651 --- /dev/null +++ b/litellm/templates/proxy-config-record.md @@ -0,0 +1,68 @@ +# LiteLLM Proxy Configuration Record + +Fill this record before changing a proxy configuration. It is the rollback unit: +the previous record plus the previous pinned image is the rollback path. + +## Deployment identity + +- Requested outcome: _[fill: what this gateway must do and for whom]_ +- Deployment type: _[fill: bare litellm / Docker / Compose / Helm / raw manifests]_ +- Target and scope confirmed with: _[fill: who confirmed, when]_ +- Rollback path: _[fill: previous image tag + previous config record]_ + +## Pinned artifacts + +- LiteLLM version or image tag/digest: _[fill: ghcr.io/berriai/litellm:vX.Y.Z or pip litellm==...]_ +- Cosign verification status: _[fill: verified against pinned key commit / not verified]_ +- fastapi pin (pip installs): _[fill: e.g. fastapi==0.136.3 for 1.97.0, or n/a]_ +- Config file source and path: _[fill: repo path or S3/GCS bucket reference]_ +- Config-in-DB state: _[fill: store_model_in_db true/false; if true, note DB overlay wins]_ +- Enterprise license in use: _[fill: yes/no]_ + +## Model list summary + +| model_name | litellm_params.model | weight/rpm/tpm | order | notes | +|---|---|---|---|---| +| _[fill]_ | _[fill: provider/prefixed string]_ | _[fill]_ | _[fill]_ | _[fill: base_model, access_groups, ...]_ | + +## Routing and reliability + +- routing_strategy: _[fill: simple-shuffle default unless deliberately changed]_ +- num_retries (settings/deployment/request): _[fill]_ +- fallbacks / context_window_fallbacks / content_policy_fallbacks: _[fill: alias targets only]_ +- cooldown settings: _[fill: allowed_fails, cooldown_time, policy overrides]_ +- enable_pre_call_checks / optional_pre_call_checks: _[fill: on/off and why]_ + +## Keys, budgets, limits + +- master_key source: _[fill: secret manager reference — never the value]_ +- salt_key set (never rotated after models added): _[fill: yes/no]_ +- global budget / duration: _[fill]_ +- team/key budget scheme summary: _[fill: scopes and caps]_ +- rate limits (tpm/rpm per scope; admin exemption acknowledged): _[fill]_ + +## Caching, guardrails, observability + +- cache backend and ttl: _[fill: redis/none; semantic caching excluded for agents?]_ +- guardrails configured (names, modes): _[fill]_ +- callbacks wired: _[fill: langfuse/otel/prometheus/...]_ +- privacy posture: _[fill: turn_off_message_logging, redact flags, store_prompts_in_spend_logs off?]_ + +## Data stores + +- Postgres endpoint and pool math: _[fill: MAX_DB_CONNECTIONS / (instances x workers)]_ +- Redis version and endpoints: _[fill: >=7.0 required when >1 instance]_ +- Backup schedule for Postgres: _[fill]_ + +## Verification checklist + +- [ ] `/health/liveliness` returns 200 (`litellm-health --check health`) +- [ ] `/health/readiness` returns 200 (DB reachable) +- [ ] `/v1/models` lists expected aliases for a representative key +- [ ] A representative chat request returns tokens; `x-litellm-model-id` matches intent +- [ ] Budget enforcement proven once with a tiny test budget +- [ ] Fallback path proven once by forcing a deployment failure + +## Changes from the previous record + +- _[fill: what changed, why, which verification backs it]_ diff --git a/litellm/templates/proxy-deployment.md b/litellm/templates/proxy-deployment.md new file mode 100644 index 0000000..a27296d --- /dev/null +++ b/litellm/templates/proxy-deployment.md @@ -0,0 +1,70 @@ +# LiteLLM Proxy Deployment Record + +Fill this record for the runtime deployment: how the proxy runs, where its state +lives, and how it is rolled back. Pair it with the config record +(`proxy-config-record.md`), which captures what the proxy serves. + +## Deployment identity + +- Requested outcome: _[fill: availability, scale, and exposure requirements]_ +- Runtime: _[fill: Docker / docker compose / Kubernetes + Helm / raw manifests]_ +- Target and scope confirmed with: _[fill: who confirmed, when]_ +- Rollback path: _[fill: previous image tag/digest + previous deployment record]_ + +## Pinned image + +- Image and tag: _[fill: ghcr.io/berriai/litellm:vX.Y.Z — never latest/main-latest]_ +- Digest: _[fill: sha256:...]_ +- Cosign verification: _[fill: command/CI gate used]_ +- Version floor check: _[fill: >=1.83.7 confirmed for public deployments]_ +- Component images (microservices chart): _[fill: gateway/backend/ui tags]_ + +## Runtime shape + +- Replicas and worker count: _[fill: --num_workers 1 per pod on K8s]_ +- CPU/memory per replica: _[fill: ~1 vCPU / 4Gi floor per worker; memory ratchets]_ +- Autoscaling: _[fill: HPA CPU target ~60% or KEDA; maxReplicas bounded by DB pool math]_ +- Job role split: _[fill: LITELLM_JOB_ROLE=serving pods + dedicated worker replica]_ +- Security context: _[fill: runAsNonRoot, readOnlyRootFilesystem, writable emptyDirs]_ + +## Network and exposure + +- Ports: _[fill: 4000 gateway; 4001 backend; 3000 ui if microservices]_ +- Bind address: _[fill: explicit --host; default 0.0.0.0 acknowledged]_ +- TLS termination: _[fill: LB/reverse proxy; port 4000 never raw]_ +- Edge route policy: _[fill: LLM routes + health probes exposed; management paths denied]_ +- Admin UI policy: _[fill: restricted network / SSO / DISABLE_ADMIN_UI]_ +- Probes: _[fill: liveness /health/liveliness; readiness /health/readiness; thresholds]_ + +## Environment (references only — values live in the secret manager) + +- `DATABASE_URL`: _[fill: secret reference]_ +- `LITELLM_MASTER_KEY`: _[fill: secret reference]_ +- `LITELLM_SALT_KEY`: _[fill: secret reference; set once, never rotate]_ +- `STORE_MODEL_IN_DB`: _[fill: True/False]_ +- `DISABLE_SCHEMA_UPDATE`: _[fill: true on pods when a migration job runs]_ +- `LITELLM_JOB_ROLE`: _[fill: serving | worker]_ +- Provider keys: _[fill: os.environ/ references only — never values]_ + +## Data stores + +- Postgres: _[fill: endpoint, version, private subnet, TLS, least-privilege role]_ +- Redis: _[fill: endpoint, >=7.0, private subnet, TLS; required when >1 replica]_ +- Backup/restore: _[fill: schedule, last restore test date]_ + +## Migrations + +- Migration strategy: _[fill: startup default vs dedicated job (Helm PreSync/hook)]_ +- DB backup taken before last migration: _[fill: date]_ + +## Verification checklist + +- [ ] `litellm-health --check health --check readiness --json` passes via the service route +- [ ] `x-litellm-version` on a live response matches the pinned tag +- [ ] A representative chat request returns tokens through the public edge +- [ ] Management routes return 403/404 from outside the trust boundary +- [ ] Blocked test key fails immediately (revocation path works) + +## Changes from the previous record + +- _[fill: what changed, why, and the verification that backs it]_ diff --git a/litellm/tests/test_litellm_health.py b/litellm/tests/test_litellm_health.py new file mode 100644 index 0000000..aacd37e --- /dev/null +++ b/litellm/tests/test_litellm_health.py @@ -0,0 +1,412 @@ +#!/usr/bin/env python3 +"""Deterministic tests for litellm/scripts/litellm-health. + +The probe runs as a subprocess so every assertion lands on the real CLI: +--help, --json, --check subsets, --key handling, exit codes, and JSON +payloads. A local stdlib HTTP server impersonates the four LiteLLM gateway +routes the probe reads, so no network or real proxy is involved. The final +test class pins the probe's read-only contract twice over: against observed +stub traffic (GET only) and against the script source (no write-mode opens). +""" +import json +import socket +import subprocess +import sys +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "litellm-health" + +MASTER_KEY = "sk-test-master-key" + + +def run_probe(*args): + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + timeout=30, + ) + + +def parse_stdout_json(proc): + return json.loads(proc.stdout) + + +class FakeGatewayRoutes: + """Route table + request journal shared by the fake gateway servers.""" + + def __init__(self): + self.seen = [] + + def handler_class(self): + journal = self.seen + + class Routes(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def _bearer(self): + return self.headers.get("Authorization", "") + + def _reply(self, code, payload): + body = payload.encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + journal.append(("GET", self.path)) + if self.path == "/health/liveliness": + self._reply(200, "I'm alive!") + elif self.path == "/health/readiness": + self._reply(200, json.dumps({"status": "healthy", "db": "connected"})) + elif self.path == "/v1/models": + if self._bearer() != f"Bearer {MASTER_KEY}": + self._reply(401, json.dumps({"error": "Unauthorized"})) + else: + self._reply( + 200, + json.dumps( + { + "object": "list", + "data": [ + {"id": "gpt-4o"}, + {"id": "claude-sonnet"}, + ], + } + ), + ) + elif self.path == "/model/info": + if self._bearer() != f"Bearer {MASTER_KEY}": + self._reply(401, json.dumps({"error": "Unauthorized"})) + else: + self._reply( + 200, + json.dumps( + [ + { + "model_name": "gpt-4o", + "litellm_params": {"api_key": "*************"}, + "model_info": {"max_tokens": 16384}, + } + ] + ), + ) + else: + self._reply(404, "not found") + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0) or 0) + if length: + self.rfile.read(length) + journal.append(("POST", self.path)) + self._reply(405, "probe must never POST") + + return Routes + + +class FakeGatewayServer: + """One-shot ThreadingHTTPServer bound to an ephemeral loopback port.""" + + def __init__(self, routes): + self.routes = routes + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), routes.handler_class()) + self.port = self.httpd.server_address[1] + self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True) + + def start(self): + self.thread.start() + + def stop(self): + self.httpd.shutdown() + self.httpd.server_close() + + @property + def url(self): + return f"http://127.0.0.1:{self.port}" + + +def idle_port(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + return port + + +def server_with_overridden_get(get_behavior): + """Build a running fake gateway whose GET replies come from get_behavior. + + get_behavior(handler) is called with the live handler instance so it can + use _reply(). The journal still records each GET. + """ + routes = FakeGatewayRoutes() + base_handler_class = routes.handler_class() + + class OverriddenRoutes(base_handler_class): + def do_GET(self): + routes.seen.append(("GET", self.path)) + get_behavior(self) + + server = FakeGatewayServer.__new__(FakeGatewayServer) + server.routes = routes + server.httpd = ThreadingHTTPServer(("127.0.0.1", 0), OverriddenRoutes) + server.port = server.httpd.server_address[1] + server.thread = threading.Thread(target=server.httpd.serve_forever, daemon=True) + server.start() + return server + + +class CliSurfaceTests(unittest.TestCase): + def test_help_without_any_server(self): + proc = run_probe("--help") + self.assertEqual(proc.returncode, 0) + for flag in ("--json", "--check", "--key", "--timeout"): + self.assertIn(flag, proc.stdout) + + def test_unknown_flag_is_a_usage_error(self): + proc = run_probe("--bogus") + self.assertEqual(proc.returncode, 2) + + def test_zero_timeout_rejected_as_usage_error(self): + proc = run_probe("--timeout", "0", "--check", "health") + self.assertEqual(proc.returncode, 2) + + def test_negative_timeout_rejected_as_usage_error(self): + proc = run_probe("--timeout", "-3", "--check", "health") + self.assertEqual(proc.returncode, 2) + + +class LivenessReadinessTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.routes = FakeGatewayRoutes() + cls.gateway = FakeGatewayServer(cls.routes) + cls.gateway.start() + + @classmethod + def tearDownClass(cls): + cls.gateway.stop() + + def test_liveliness_ok_body_and_exit_code(self): + proc = run_probe( + "--url", self.gateway.url, "--check", "health", "--json" + ) + self.assertEqual(proc.returncode, 0) + check = parse_stdout_json(proc)["checks"][0] + self.assertTrue(check["ok"]) + self.assertEqual(check["status_code"], 200) + self.assertIn("alive", check["body"].lower()) + + def test_readiness_reports_db_state(self): + proc = run_probe( + "--url", self.gateway.url, "--check", "readiness", "--json" + ) + self.assertEqual(proc.returncode, 0) + check = parse_stdout_json(proc)["checks"][0] + self.assertTrue(check["ok"]) + self.assertEqual((check["status"], check["db"]), ("healthy", "connected")) + + def test_readiness_503_fails_with_database_hint(self): + def always_503(inner_self): + inner_self._reply(503, json.dumps({"error": "db unavailable"})) + + gateway = server_with_overridden_get(always_503) + try: + proc = run_probe( + "--url", gateway.url, "--check", "readiness", "--json" + ) + self.assertEqual(proc.returncode, 1) + check = parse_stdout_json(proc)["checks"][0] + self.assertFalse(check["ok"]) + self.assertEqual(check["status_code"], 503) + self.assertIn("database", check["hint"].lower()) + finally: + gateway.stop() + + def test_liveliness_500_fails(self): + def broken_liveliness(inner_self): + inner_self._reply(500, "boom") + + gateway = server_with_overridden_get(broken_liveliness) + try: + proc = run_probe("--url", gateway.url, "--check", "health", "--json") + self.assertEqual(proc.returncode, 1) + self.assertFalse(parse_stdout_json(proc)["checks"][0]["ok"]) + finally: + gateway.stop() + + +class KeyedRouteTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.routes = FakeGatewayRoutes() + cls.gateway = FakeGatewayServer(cls.routes) + cls.gateway.start() + + @classmethod + def tearDownClass(cls): + cls.gateway.stop() + + def test_models_without_key_is_reported_not_run(self): + proc = run_probe("--url", self.gateway.url, "--check", "models", "--json") + self.assertEqual(proc.returncode, 1) + check = parse_stdout_json(proc)["checks"][0] + self.assertTrue(check["skipped_missing_key"]) + self.assertFalse(check["ok"]) + + def test_wrong_key_surfaces_the_401(self): + proc = run_probe( + "--url", + self.gateway.url, + "--check", + "models", + "--key", + "sk-not-the-key", + "--json", + ) + self.assertEqual(proc.returncode, 1) + check = parse_stdout_json(proc)["checks"][0] + self.assertEqual(check["status_code"], 401) + self.assertIn("key", check["hint"].lower()) + + def test_models_lists_aliases_for_master_key(self): + proc = run_probe( + "--url", self.gateway.url, "--check", "models", "--key", MASTER_KEY, "--json" + ) + self.assertEqual(proc.returncode, 0) + check = parse_stdout_json(proc)["checks"][0] + self.assertTrue(check["ok"]) + self.assertEqual(check["model_ids"], ["gpt-4o", "claude-sonnet"]) + + def test_model_info_counts_registered_deployments(self): + proc = run_probe( + "--url", + self.gateway.url, + "--check", + "model_info", + "--key", + MASTER_KEY, + "--json", + ) + self.assertEqual(proc.returncode, 0) + check = parse_stdout_json(proc)["checks"][0] + self.assertTrue(check["ok"]) + self.assertEqual(check["deployment_count"], 1) + + +class ExitCodeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.routes = FakeGatewayRoutes() + cls.gateway = FakeGatewayServer(cls.routes) + cls.gateway.start() + + @classmethod + def tearDownClass(cls): + cls.gateway.stop() + + def test_everything_green_is_exit_zero(self): + proc = run_probe( + "--url", + self.gateway.url, + "--check", + "health", + "--check", + "readiness", + "--check", + "models", + "--check", + "model_info", + "--key", + MASTER_KEY, + "--json", + ) + self.assertEqual(proc.returncode, 0) + checks = parse_stdout_json(proc)["checks"] + self.assertEqual(len(checks), 4) + for check in checks: + self.assertTrue(check["ok"], f"{check['name']} should pass: {check}") + + def test_dead_port_maps_to_exit_one(self): + proc = run_probe( + "--url", f"http://127.0.0.1:{idle_port()}", "--check", "health" + ) + self.assertEqual(proc.returncode, 1) + self.assertIn("FAIL", proc.stdout) + + def test_hanging_route_maps_to_exit_124(self): + def sleepy_get(inner_self): + time.sleep(2.0) + inner_self._reply(200, "I'm alive!") + + gateway = server_with_overridden_get(sleepy_get) + try: + proc = run_probe( + "--url", gateway.url, "--check", "health", "--timeout", "0.5" + ) + self.assertEqual(proc.returncode, 124) + self.assertIn("timed out", proc.stdout) + finally: + gateway.stop() + + +class ReadOnlyContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.routes = FakeGatewayRoutes() + cls.gateway = FakeGatewayServer(cls.routes) + cls.gateway.start() + + @classmethod + def tearDownClass(cls): + cls.gateway.stop() + + def test_observed_traffic_is_exclusively_get(self): + marker = len(self.routes.seen) + run_probe( + "--url", + self.gateway.url, + "--check", + "health", + "--check", + "readiness", + "--check", + "models", + "--check", + "model_info", + "--key", + MASTER_KEY, + ) + issued = list(self.routes.seen[marker:]) + self.assertGreater(len(issued), 0, "probe made no requests") + for method, path in issued: + self.assertEqual( + method, "GET", f"probe issued {method} against {path}" + ) + + def test_source_contains_no_write_mode_file_opens(self): + source = SCRIPT.read_text(encoding="utf-8") + for line in source.splitlines(): + stripped = line.strip() + if stripped.startswith("#"): + continue + for forbidden in ("'w'", '"w"', "'a'", '"a"'): + self.assertNotIn(forbidden, stripped) + + def test_source_declares_get_and_no_other_method(self): + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn('method="GET"', source) + for other in ('method="POST"', 'method="PUT"', 'method="DELETE"', "data="): + self.assertNotIn(other, source) + + +if __name__ == "__main__": + unittest.main() diff --git a/llms.txt b/llms.txt index ccfe719..02628cb 100644 --- a/llms.txt +++ b/llms.txt @@ -74,6 +74,7 @@ - [legal-strategy](legal-strategy/SKILL.md): CLO/General Counsel methodology — regulatory landscape analysis (GDPR, CCPA, AI Act, sector-specific), IP strategy (patent, trademark, trade secret, open source licensing), contract risk assessment (indemnification, liability caps, force majeure), data privacy frameworks (privacy-by-design, DPIAs, data mapping), corporate governance (board responsibilities, fiduciary duties, shareholder rights), employment law (classification, IP assignment, non-competes). - [life-coach](life-coach/SKILL.md): Guide a bounded, user-led coaching process for personal goals, decisions, transitions, habits, recurring nonclinical patterns, accountability, and progress review. Use when a person explicitly asks to be coached, wants reflective challenge, or wants help examining ambivalence while retaining ownership. Do not use for therapy, crisis support, diagnosis, direct factual or action requests, product or stakeholder discovery, or medical, legal, financial, addiction, domestic-violence, or other specialist advice. - [linear](linear/SKILL.md): Manage Linear teams, projects, cycles, issues, comments, workflow state, and documents from a terminal through Linear's public GraphQL API. Use when a user asks to list, search, inspect, create, update, move, or comment on Linear work, or to find Linear documents. Do not use to embed a live agent inside Linear or to build an MCP integration. +- [litellm](litellm/SKILL.md): Operate, configure, secure, and troubleshoot the LiteLLM AI gateway (proxy) and Python SDK: run the proxy (litellm --config), route to 100+ providers through one OpenAI-compatible API, configure model lists and routing/reliability, virtual keys, teams, budgets, rate limits, caching, guardrails, observability, and spend, and diagnose request failures. Use when deploying or running a LiteLLM proxy or gateway (config.yaml, ghcr.io/berriai/litellm), wiring the Python SDK or OpenAI SDK through it, or hardening a public-facing deployment. Do not use for operating a single inference engine (vllm, llama-cpp), for engine-selection methodology (ml-engineering), or for building applications on top of an LLM API (backend/frontend engineering). - [llama-cpp](llama-cpp/SKILL.md): Operate, configure, benchmark, and troubleshoot llama.cpp across CPU, Metal, CUDA, HIP/ROCm, Vulkan, SYCL, and hybrid or multi-GPU systems. Use when installing or building llama.cpp, selecting or inspecting GGUF models, running llama-cli, serving an OpenAI-compatible API with llama-server, tuning memory and performance, or diagnosing backend, context, template, and API failures. Do not use for model training or fine-tuning, general inference-framework selection, llama-cpp-python or other bindings, LlamaIndex, Ollama, or LM Studio operation. - [llamaindex](llamaindex/SKILL.md): Expert skill for building LLM applications with the LlamaIndex framework — RAG pipelines, multi-agent orchestration, event-driven workflows, knowledge graph construction, production deployment, and evaluation. Use when working with LlamaIndex or comparing RAG and agent orchestration frameworks. - [mermaid-diagrams](mermaid-diagrams/SKILL.md): Author, render, troubleshoot, and review Mermaid diagrams for documentation, architecture, processes, and technical communication. Use when a versionable diagram must communicate a defined audience job across a real renderer, including narrative, hierarchy, labels, legends, accessibility fallback, and uncertainty. Do not use for C4 level/model ownership, architecture decisions, or full accessibility conformance. diff --git a/references/skill-triggers.md b/references/skill-triggers.md index 1354b50..b938625 100644 --- a/references/skill-triggers.md +++ b/references/skill-triggers.md @@ -40,6 +40,7 @@ Each skill's `description` field is the canonical routing contract. This conveni | "kubernetes", "k8s", "kubectl", "k3s", "RKE2", "MicroK8s", "k0s", "Talos", "OpenShift", "EKS", "AKS", "GKE", "Pod", "Deployment", "StatefulSet", "CRD", "RBAC", "NetworkPolicy", "Helm on Kubernetes", "cluster upgrade", "Kubernetes troubleshooting" | [kubernetes](../kubernetes/SKILL.md) | | "langgraph", "multi-agent", "state machine", "graph-based workflow", "LangGraph", "supervisor pattern", "swarm pattern", "agent orchestration", "graph state", "subgraph", "agent routing", "tool-calling loop", "agent loop", "stateful agent", "durable execution", "human in the loop langgraph", "checkpointer", "langgraph persistence" | [langgraph](../langgraph/SKILL.md) | | "Linear", "Linear API", "Linear issue", "Linear project", "Linear cycle", "Linear comment", "Linear document" | [linear](../linear/SKILL.md) | +| "LiteLLM", "litellm proxy", "AI gateway", "LLM gateway", "config.yaml model_list", "virtual keys", "master key", "budgets and spend tracking", "LLM load balancing", "fallbacks and cooldowns", "Presidio PII guardrail", "litellm-health", "ghcr.io/berriai/litellm" | [litellm](../litellm/SKILL.md) | | "llama.cpp", "llama-cpp", "llama-cli", "llama-server", "llama-bench", "GGUF model", "GGUF quantization", "GPU layers", "KV cache", "Metal llama.cpp", "CUDA llama.cpp", "ROCm llama.cpp", "Vulkan llama.cpp", "llama.cpp OpenAI API", "llama.cpp chat template", "llama.cpp multi-GPU" | [llama-cpp](../llama-cpp/SKILL.md) | | "debate", "council", "multi-perspective", "structured debate", "get multiple perspectives", "expert panel", "decision landscape", "what would experts say", "what are we missing", "convergence", "false consensus", "agent-council", "pre-mortem" | [agent-council](../agent-council/SKILL.md) | | "skill format", "how do I make a skill", "agentskills.io" | [agent-skills](../agent-skills/SKILL.md) |