From dd97e846ecd1f68a449946088eb64d005219476b Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 02:10:49 -0400 Subject: [PATCH 01/40] refactor(skills): drop -cli suffix from six consumer-API skills Rename ghost-cli, jira-cli, jellyfin-cli, openlibrary-cli, tmdb-cli, and tempest-cli to ghost, jira, jellyfin, openlibrary, tmdb, and tempest via git mv. Rewrite frontmatter name fields to match new directories, rename bundled scripts preserving executable bits, update internal invocation strings and README quick-start examples, and relocate the jellyfin pytest suite to jellyfin/scripts/ with its SCRIPT constant now resolving to the renamed sibling script. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- {ghost-cli => ghost}/README.md | 2 +- {ghost-cli => ghost}/SKILL.md | 58 +++++++-------- .../scripts/ghost-cli => ghost/scripts/ghost | 12 ++-- {jellyfin-cli => jellyfin}/README.md | 8 +-- {jellyfin-cli => jellyfin}/SKILL.md | 66 ++++++++--------- .../jellyfin-cli => jellyfin/scripts/jellyfin | 32 ++++----- .../scripts}/test_jellyfin_cli.py | 2 +- {jira-cli => jira}/README.md | 2 +- {jira-cli => jira}/SKILL.md | 54 +++++++------- .../scripts/jira-cli => jira/scripts/jira | 16 ++--- {openlibrary-cli => openlibrary}/README.md | 8 +-- {openlibrary-cli => openlibrary}/SKILL.md | 56 +++++++-------- .../scripts/openlibrary | 18 ++--- {tempest-cli => tempest}/README.md | 6 +- {tempest-cli => tempest}/SKILL.md | 40 +++++------ .../references/tempest-api-field-layouts.md | 0 .../tempest-cli => tempest/scripts/tempest | 14 ++-- {tmdb-cli => tmdb}/README.md | 6 +- {tmdb-cli => tmdb}/SKILL.md | 70 +++++++++---------- .../scripts/tmdb-cli => tmdb/scripts/tmdb | 16 ++--- 20 files changed, 243 insertions(+), 243 deletions(-) rename {ghost-cli => ghost}/README.md (94%) rename {ghost-cli => ghost}/SKILL.md (65%) rename ghost-cli/scripts/ghost-cli => ghost/scripts/ghost (97%) rename {jellyfin-cli => jellyfin}/README.md (88%) rename {jellyfin-cli => jellyfin}/SKILL.md (64%) rename jellyfin-cli/scripts/jellyfin-cli => jellyfin/scripts/jellyfin (94%) rename {jellyfin-cli/tests => jellyfin/scripts}/test_jellyfin_cli.py (99%) rename {jira-cli => jira}/README.md (95%) rename {jira-cli => jira}/SKILL.md (61%) rename jira-cli/scripts/jira-cli => jira/scripts/jira (97%) rename {openlibrary-cli => openlibrary}/README.md (83%) rename {openlibrary-cli => openlibrary}/SKILL.md (66%) rename openlibrary-cli/scripts/openlibrary-cli => openlibrary/scripts/openlibrary (94%) rename {tempest-cli => tempest}/README.md (92%) rename {tempest-cli => tempest}/SKILL.md (77%) rename {tempest-cli => tempest}/references/tempest-api-field-layouts.md (100%) rename tempest-cli/scripts/tempest-cli => tempest/scripts/tempest (98%) rename {tmdb-cli => tmdb}/README.md (88%) rename {tmdb-cli => tmdb}/SKILL.md (54%) rename tmdb-cli/scripts/tmdb-cli => tmdb/scripts/tmdb (96%) diff --git a/ghost-cli/README.md b/ghost/README.md similarity index 94% rename from ghost-cli/README.md rename to ghost/README.md index 256d7b4..e600f7e 100644 --- a/ghost-cli/README.md +++ b/ghost/README.md @@ -16,7 +16,7 @@ When your agent loads this skill, it can **manage your Ghost CMS content** witho | Directory | Purpose | |-----------|---------| | `SKILL.md` | Complete command reference with setup and examples | -| `scripts/ghost-cli` | CLI tool for Ghost Admin API operations | +| `scripts/ghost` | CLI tool for Ghost Admin API operations | ## Quick Start diff --git a/ghost-cli/SKILL.md b/ghost/SKILL.md similarity index 65% rename from ghost-cli/SKILL.md rename to ghost/SKILL.md index c36a7e9..546aaf0 100644 --- a/ghost-cli/SKILL.md +++ b/ghost/SKILL.md @@ -1,5 +1,5 @@ --- -name: ghost-cli +name: ghost description: Manage Ghost CMS content from the terminal — create and list posts, pages, and tags, and fetch site info via the Ghost Admin API (v5/v6). Use when the user asks about ghost, cms, blog, blogging, posts, pages, tags, publishing, or site configuration. @@ -12,7 +12,7 @@ metadata: sources: https://ghost.org/docs/admin-api/, https://ghost.org/docs/ --- -# ghost-cli — Ghost CMS from the Terminal +# ghost — Ghost CMS from the Terminal Manage content on a Ghost CMS site: view site info, list and create posts and pages, manage tags — all via the Ghost Admin API (v5/v6). @@ -33,9 +33,9 @@ export GHOST_ADMIN_KEY="your-id:your-secret" # from Ghost Admin → Integ ### site — Get site information ```bash -ghost-cli site # show site title, URL, description -ghost-cli --json site # machine-readable JSON -ghost-cli --dry-run site # preview without API call +ghost site # show site title, URL, description +ghost --json site # machine-readable JSON +ghost --dry-run site # preview without API call ``` Shows: site title, URL, description. @@ -43,12 +43,12 @@ Shows: site title, URL, description. ### posts — List blog posts ```bash -ghost-cli posts # 20 most recent posts -ghost-cli posts --limit 50 # more results -ghost-cli posts --status published # only published posts -ghost-cli posts --status draft # only draft posts -ghost-cli posts --status scheduled # only scheduled posts -ghost-cli posts --limit 10 --json # 10 most recent as JSON +ghost posts # 20 most recent posts +ghost posts --limit 50 # more results +ghost posts --status published # only published posts +ghost posts --status draft # only draft posts +ghost posts --status scheduled # only scheduled posts +ghost posts --limit 10 --json # 10 most recent as JSON ``` Shows: title, status, slug, and last-updated date for each post. @@ -56,12 +56,12 @@ Shows: title, status, slug, and last-updated date for each post. ### create-post — Create a new blog post ```bash -ghost-cli create-post --title "My First Post" # draft, no HTML -ghost-cli create-post --title "Hello World" --html "

Hello!

" # with HTML content -ghost-cli create-post --title "Ready" --html "

Published

" --status published # publish immediately -ghost-cli create-post --title "Scheduled" --html "

Later

" --status scheduled # schedule -ghost-cli create-post --title "Custom Slug" --slug "my-custom-url" # custom URL slug -ghost-cli create-post --title "Draft" --dry-run # preview without creating +ghost create-post --title "My First Post" # draft, no HTML +ghost create-post --title "Hello World" --html "

Hello!

" # with HTML content +ghost create-post --title "Ready" --html "

Published

" --status published # publish immediately +ghost create-post --title "Scheduled" --html "

Later

" --status scheduled # schedule +ghost create-post --title "Custom Slug" --slug "my-custom-url" # custom URL slug +ghost create-post --title "Draft" --dry-run # preview without creating ``` Creates the post and returns its title, slug, and status. @@ -69,9 +69,9 @@ Creates the post and returns its title, slug, and status. ### pages — List pages ```bash -ghost-cli pages # 20 most recent pages -ghost-cli pages --limit 50 # more results -ghost-cli pages --json # machine-readable JSON +ghost pages # 20 most recent pages +ghost pages --limit 50 # more results +ghost pages --json # machine-readable JSON ``` Shows: title, status, slug, and last-updated date for each page. @@ -79,9 +79,9 @@ Shows: title, status, slug, and last-updated date for each page. ### tags — List tags ```bash -ghost-cli tags # 50 tags with post counts -ghost-cli tags --limit 100 # more results -ghost-cli tags --json # machine-readable JSON +ghost tags # 50 tags with post counts +ghost tags --limit 100 # more results +ghost tags --json # machine-readable JSON ``` Shows: tag name, slug, and number of posts using each tag. @@ -91,11 +91,11 @@ Shows: tag name, slug, and number of posts using each tag. These flags work anywhere in the command — before or after the subcommand: ```bash -ghost-cli --json posts # JSON output -ghost-cli posts --json # same result, after subcommand -ghost-cli --dry-run create-post --title "Test" # preview without API call -ghost-cli --quiet posts # suppress diagnostic output -ghost-cli --verbose site # verbose logging +ghost --json posts # JSON output +ghost posts --json # same result, after subcommand +ghost --dry-run create-post --title "Test" # preview without API call +ghost --quiet posts # suppress diagnostic output +ghost --verbose site # verbose logging ``` | Flag | Effect | @@ -119,6 +119,6 @@ ghost-cli --verbose site # verbose logging ## References -- [scripts/ghost-cli](scripts/ghost-cli) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. +- [scripts/ghost](scripts/ghost) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. - [Ghost Admin API Docs](https://ghost.org/docs/admin-api/) — Official Ghost Admin API documentation. - [Ghost Integrations](https://ghost.org/docs/integrations/) — How to create Custom Integrations and get your Admin API key. diff --git a/ghost-cli/scripts/ghost-cli b/ghost/scripts/ghost similarity index 97% rename from ghost-cli/scripts/ghost-cli rename to ghost/scripts/ghost index 04abb51..56a9068 100755 --- a/ghost-cli/scripts/ghost-cli +++ b/ghost/scripts/ghost @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""ghost-cli — Ghost CMS from the terminal. +"""ghost — Ghost CMS from the terminal. Manage content on a Ghost CMS site: create and edit posts and pages, manage tags, and configure metadata. Requires GHOST_URL and GHOST_ADMIN_KEY. @@ -204,7 +204,7 @@ def fmt_post(p): def cmd_posts(client, args): - p = argparse.ArgumentParser(prog="ghost-cli posts") + p = argparse.ArgumentParser(prog="ghost posts") p.add_argument("--limit", type=int, default=20) p.add_argument("--status", choices=["published", "draft", "scheduled"]) parsed, _ = p.parse_known_args(args) @@ -224,7 +224,7 @@ def cmd_posts(client, args): def cmd_create_post(client, args): - p = argparse.ArgumentParser(prog="ghost-cli posts create") + p = argparse.ArgumentParser(prog="ghost posts create") p.add_argument("--title", required=True) p.add_argument("--html", default="") p.add_argument("--status", default="draft", choices=["draft", "published", "scheduled"]) @@ -245,7 +245,7 @@ def cmd_create_post(client, args): def cmd_pages(client, args): - p = argparse.ArgumentParser(prog="ghost-cli pages") + p = argparse.ArgumentParser(prog="ghost pages") p.add_argument("--limit", type=int, default=20) parsed, _ = p.parse_known_args(args) @@ -262,7 +262,7 @@ def cmd_pages(client, args): def cmd_tags(client, args): - p = argparse.ArgumentParser(prog="ghost-cli tags") + p = argparse.ArgumentParser(prog="ghost tags") p.add_argument("--limit", type=int, default=50) parsed, _ = p.parse_known_args(args) @@ -302,7 +302,7 @@ def main(): if GLOBAL_FLAGS.get("json", False): warnings.simplefilter("ignore") - parser = argparse.ArgumentParser(prog="ghost-cli", description="Ghost CMS CLI.", + parser = argparse.ArgumentParser(prog="ghost", description="Ghost CMS CLI.", epilog="Set GHOST_URL and GHOST_ADMIN_KEY. Key format: id:secret from Integrations.") sub = parser.add_subparsers(dest="command") sub.add_parser("site", help="Site info") diff --git a/jellyfin-cli/README.md b/jellyfin/README.md similarity index 88% rename from jellyfin-cli/README.md rename to jellyfin/README.md index 59df69b..36ff5ee 100644 --- a/jellyfin-cli/README.md +++ b/jellyfin/README.md @@ -17,12 +17,12 @@ When your agent loads this skill, it can **navigate your home media server** wit | Directory | Purpose | |-----------|---------| | `SKILL.md` | Complete command reference with setup and examples | -| `scripts/jellyfin-cli` | CLI tool for Jellyfin API operations | +| `scripts/jellyfin` | CLI tool for Jellyfin API operations | ## Quick Start ```bash -scripts/jellyfin-cli --help +scripts/jellyfin --help export JELLYFIN_URL="http://your-server:8096" export JELLYFIN_API_KEY="your-api-key" export JELLYFIN_USER_ID="your-jellyfin-user-id" # required by recent, next-up, and item @@ -31,8 +31,8 @@ export JELLYFIN_USER_ID="your-jellyfin-user-id" # required by recent, next-up, a API key from Dashboard → API Keys in the Jellyfin admin panel. ```bash -scripts/jellyfin-cli search --query "dune" --type Movie -scripts/jellyfin-cli next-up --limit 5 +scripts/jellyfin search --query "dune" --type Movie +scripts/jellyfin next-up --limit 5 ``` ## Triggers diff --git a/jellyfin-cli/SKILL.md b/jellyfin/SKILL.md similarity index 64% rename from jellyfin-cli/SKILL.md rename to jellyfin/SKILL.md index 0c2da70..d9bbc87 100644 --- a/jellyfin-cli/SKILL.md +++ b/jellyfin/SKILL.md @@ -1,5 +1,5 @@ --- -name: jellyfin-cli +name: jellyfin description: Query your Jellyfin media server from the terminal — recently added media, search, item details, next-up episodes, library browsing, server info, and stats. Use when the user asks about Jellyfin, media server, movies, TV shows, next episodes, or @@ -14,7 +14,7 @@ metadata: sources: https://jellyfin.org/docs/general/clients/api, https://jellyfin.org/downloads --- -# jellyfin-cli — Jellyfin Media Server from the Terminal +# jellyfin — Jellyfin Media Server from the Terminal Query recently added movies and TV episodes, search and inspect media, browse libraries, see next-up episodes, check server info, and view library statistics — all from your Jellyfin server's REST API. @@ -30,16 +30,16 @@ export JELLYFIN_API_KEY="your-api-key-here" export JELLYFIN_USER_ID="your-jellyfin-user-id" # required by recent, next-up, and item ``` -Run the bundled CLI as `scripts/jellyfin-cli`. `--help` and `--dry-run` work without credentials. +Run the bundled CLI as `scripts/jellyfin`. `--help` and `--dry-run` work without credentials. ## Essential Commands ### info — Server information ```bash -scripts/jellyfin-cli info # server name, version, OS, user count -scripts/jellyfin-cli info --json # machine-readable -scripts/jellyfin-cli --dry-run info # preview API requests +scripts/jellyfin info # server name, version, OS, user count +scripts/jellyfin info --json # machine-readable +scripts/jellyfin --dry-run info # preview API requests ``` Shows: server name, version, operating system, number of users. @@ -47,13 +47,13 @@ Shows: server name, version, operating system, number of users. ### recent — Recently added media ```bash -scripts/jellyfin-cli recent # last 10 items added -scripts/jellyfin-cli recent --limit 20 # more results -scripts/jellyfin-cli recent --movies # only recently added movies -scripts/jellyfin-cli recent --episodes # only recently added episodes -scripts/jellyfin-cli recent --user-id USER_ID # override JELLYFIN_USER_ID -scripts/jellyfin-cli recent --movies --limit 5 # top 5 recently added movies -scripts/jellyfin-cli recent --json # machine-readable +scripts/jellyfin recent # last 10 items added +scripts/jellyfin recent --limit 20 # more results +scripts/jellyfin recent --movies # only recently added movies +scripts/jellyfin recent --episodes # only recently added episodes +scripts/jellyfin recent --user-id USER_ID # override JELLYFIN_USER_ID +scripts/jellyfin recent --movies --limit 5 # top 5 recently added movies +scripts/jellyfin recent --json # machine-readable ``` Uses Jellyfin's current `/Items/Latest` endpoint. `--movies` and `--episodes` send `includeItemTypes` to the server, so the requested limit applies to the selected media type. Shows: name, type (Movie/Episode), production year, series name (for episodes), date added. @@ -61,11 +61,11 @@ Uses Jellyfin's current `/Items/Latest` endpoint. `--movies` and `--episodes` se ### search — Search your media library ```bash -scripts/jellyfin-cli search --query "dune" # search everything -scripts/jellyfin-cli search --query "dune" --type Movie # movies only -scripts/jellyfin-cli search --query "star trek" --type Series,Episode -scripts/jellyfin-cli search --query "inception" --limit 5 # top 5 results -scripts/jellyfin-cli search --query "dune" --json # machine-readable +scripts/jellyfin search --query "dune" # search everything +scripts/jellyfin search --query "dune" --type Movie # movies only +scripts/jellyfin search --query "star trek" --type Series,Episode +scripts/jellyfin search --query "inception" --limit 5 # top 5 results +scripts/jellyfin search --query "dune" --json # machine-readable ``` The `--type` flag accepts a comma-separated list of item types (e.g. `Movie,Series,Episode`). @@ -73,13 +73,13 @@ The `--type` flag accepts a comma-separated list of item types (e.g. `Movie,Seri ### Navigation — Inspect media and browse libraries ```bash -scripts/jellyfin-cli search --query "dune" --type Movie # find an item ID -scripts/jellyfin-cli item --id ITEM_ID # inspect that item -scripts/jellyfin-cli libraries # find a library ID -scripts/jellyfin-cli browse --library-id LIBRARY_ID --type Movie --limit 20 -scripts/jellyfin-cli browse --library-id LIBRARY_ID --start-index 20 -scripts/jellyfin-cli next-up --limit 10 # next episodes for JELLYFIN_USER_ID -scripts/jellyfin-cli next-up --user-id USER_ID --json +scripts/jellyfin search --query "dune" --type Movie # find an item ID +scripts/jellyfin item --id ITEM_ID # inspect that item +scripts/jellyfin libraries # find a library ID +scripts/jellyfin browse --library-id LIBRARY_ID --type Movie --limit 20 +scripts/jellyfin browse --library-id LIBRARY_ID --start-index 20 +scripts/jellyfin next-up --limit 10 # next episodes for JELLYFIN_USER_ID +scripts/jellyfin next-up --user-id USER_ID --json ``` Use `search -> item` to look up a result's metadata, and `libraries -> browse` to page through a collection. `next-up` returns the next unwatched episodes for the selected user. `item` and `next-up` require `JELLYFIN_USER_ID` or `--user-id`; all three commands are read-only. @@ -87,8 +87,8 @@ Use `search -> item` to look up a result's metadata, and `libraries -> browse` t ### libraries — List media libraries ```bash -scripts/jellyfin-cli libraries # all configured libraries -scripts/jellyfin-cli libraries --json # machine-readable +scripts/jellyfin libraries # all configured libraries +scripts/jellyfin libraries --json # machine-readable ``` Shows: library name, collection type (movies, tvshows, music, etc.), library ID. @@ -96,8 +96,8 @@ Shows: library name, collection type (movies, tvshows, music, etc.), library ID. ### stats — Library statistics ```bash -scripts/jellyfin-cli stats # movie, series, episode, song counts -scripts/jellyfin-cli stats --json # machine-readable +scripts/jellyfin stats # movie, series, episode, song counts +scripts/jellyfin stats --json # machine-readable ``` Shows: total count of movies, series, episodes, and songs in the library. @@ -107,9 +107,9 @@ Shows: total count of movies, series, episodes, and songs in the library. These flags work anywhere in the command — before or after the subcommand: ```bash -scripts/jellyfin-cli --json recent --limit 5 # JSON output -scripts/jellyfin-cli recent --limit 5 --json # same result, after subcommand -scripts/jellyfin-cli --dry-run search --query "dune" # preview request without API call +scripts/jellyfin --json recent --limit 5 # JSON output +scripts/jellyfin recent --limit 5 --json # same result, after subcommand +scripts/jellyfin --dry-run search --query "dune" # preview request without API call ``` | Flag | Effect | @@ -129,6 +129,6 @@ scripts/jellyfin-cli --dry-run search --query "dune" # preview request wit ## References -- [scripts/jellyfin-cli](scripts/jellyfin-cli) — The bundled read-only CLI binary with `--json`, `--dry-run`, and lazy authentication. +- [scripts/jellyfin](scripts/jellyfin) — The bundled read-only CLI binary with `--json`, `--dry-run`, and lazy authentication. - [Jellyfin API Docs](https://jellyfin.org/docs/general/clients/api) — Official API documentation. - [Jellyfin Downloads](https://jellyfin.org/downloads) — Server download and setup guide. diff --git a/jellyfin-cli/scripts/jellyfin-cli b/jellyfin/scripts/jellyfin similarity index 94% rename from jellyfin-cli/scripts/jellyfin-cli rename to jellyfin/scripts/jellyfin index 4778db8..662da94 100755 --- a/jellyfin-cli/scripts/jellyfin-cli +++ b/jellyfin/scripts/jellyfin @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""jellyfin-cli — Jellyfin media server from the terminal. +"""jellyfin — Jellyfin media server from the terminal. Query recently added media, search your library, browse by collection, and check server status. Requires JELLYFIN_URL and JELLYFIN_API_KEY. @@ -179,7 +179,7 @@ def cmd_info(client, args): def cmd_recent(client, args): - p = argparse.ArgumentParser(prog="jellyfin-cli recent") + p = argparse.ArgumentParser(prog="jellyfin recent") p.add_argument("--limit", type=int, default=10) media_type = p.add_mutually_exclusive_group() media_type.add_argument("--movies", action="store_true") @@ -221,7 +221,7 @@ def cmd_recent(client, args): def cmd_search(client, args): - p = argparse.ArgumentParser(prog="jellyfin-cli search") + p = argparse.ArgumentParser(prog="jellyfin search") p.add_argument("--query", "-q", required=True) p.add_argument("--type", help="Comma-separated types (Movie,Series,Episode)") p.add_argument("--limit", type=int, default=20) @@ -254,7 +254,7 @@ def cmd_search(client, args): def cmd_next_up(client, args): - p = argparse.ArgumentParser(prog="jellyfin-cli next-up") + p = argparse.ArgumentParser(prog="jellyfin next-up") p.add_argument("--user-id", default=ENV_USER_ID) p.add_argument("--limit", type=int, default=10) parsed, _ = p.parse_known_args(args) @@ -276,7 +276,7 @@ def cmd_next_up(client, args): def cmd_item(client, args): - p = argparse.ArgumentParser(prog="jellyfin-cli item") + p = argparse.ArgumentParser(prog="jellyfin item") p.add_argument("--id", required=True) p.add_argument("--user-id", default=ENV_USER_ID) parsed, _ = p.parse_known_args(args) @@ -295,7 +295,7 @@ def cmd_item(client, args): def cmd_browse(client, args): - p = argparse.ArgumentParser(prog="jellyfin-cli browse") + p = argparse.ArgumentParser(prog="jellyfin browse") p.add_argument("--library-id", required=True) p.add_argument("--type") p.add_argument("--limit", type=int, default=50) @@ -365,35 +365,35 @@ def main(): if GLOBAL_FLAGS.get("json", False): warnings.simplefilter("ignore") - parser = argparse.ArgumentParser(prog="jellyfin-cli", description="Jellyfin media server CLI.", - epilog="Example: jellyfin-cli search --query dune") + parser = argparse.ArgumentParser(prog="jellyfin", description="Jellyfin media server CLI.", + epilog="Example: jellyfin search --query dune") parser.add_argument("--json", action="store_true", help="Output machine-readable JSON") parser.add_argument("--dry-run", action="store_true", help="Preview API requests without network access") sub = parser.add_subparsers(dest="command") - sub.add_parser("info", help="Server info", description="Show Jellyfin server details.", epilog="Example: jellyfin-cli info") - re = sub.add_parser("recent", help="Recently added", description="Show recently added movies or episodes.", epilog="Example: jellyfin-cli recent --movies --limit 5") + sub.add_parser("info", help="Server info", description="Show Jellyfin server details.", epilog="Example: jellyfin info") + re = sub.add_parser("recent", help="Recently added", description="Show recently added movies or episodes.", epilog="Example: jellyfin recent --movies --limit 5") re.add_argument("--limit", type=int, default=10, help="Maximum items to return (default: 10)") media_type = re.add_mutually_exclusive_group() media_type.add_argument("--movies", action="store_true", help="Show only movies") media_type.add_argument("--episodes", action="store_true", help="Show only episodes") re.add_argument("--user-id", help="Jellyfin user ID (default: JELLYFIN_USER_ID)") - se = sub.add_parser("search", help="Search media", description="Search the Jellyfin media library.", epilog="Example: jellyfin-cli search --query dune --type Movie") + se = sub.add_parser("search", help="Search media", description="Search the Jellyfin media library.", epilog="Example: jellyfin search --query dune --type Movie") se.add_argument("--query", "-q", required=True, help="Text to search for") se.add_argument("--type", help="Comma-separated item types, such as Movie,Series") se.add_argument("--limit", type=int, default=20, help="Maximum results to return (default: 20)") - nu = sub.add_parser("next-up", help="Next unwatched episodes", description="Show the next unwatched episodes for a Jellyfin user.", epilog="Example: jellyfin-cli next-up --user-id USER_ID --limit 5") + nu = sub.add_parser("next-up", help="Next unwatched episodes", description="Show the next unwatched episodes for a Jellyfin user.", epilog="Example: jellyfin next-up --user-id USER_ID --limit 5") nu.add_argument("--user-id", help="Jellyfin user ID (default: JELLYFIN_USER_ID)") nu.add_argument("--limit", type=int, default=10, help="Maximum episodes to return (default: 10)") - it = sub.add_parser("item", help="Show item details", description="Show metadata for one Jellyfin library item.", epilog="Example: jellyfin-cli item --id ITEM_ID --user-id USER_ID") + it = sub.add_parser("item", help="Show item details", description="Show metadata for one Jellyfin library item.", epilog="Example: jellyfin item --id ITEM_ID --user-id USER_ID") it.add_argument("--id", required=True, help="Jellyfin item ID") it.add_argument("--user-id", help="Jellyfin user ID (default: JELLYFIN_USER_ID)") - br = sub.add_parser("browse", help="Browse a library", description="List items in a Jellyfin media library.", epilog="Example: jellyfin-cli browse --library-id LIBRARY_ID --type Movie --limit 20") + br = sub.add_parser("browse", help="Browse a library", description="List items in a Jellyfin media library.", epilog="Example: jellyfin browse --library-id LIBRARY_ID --type Movie --limit 20") br.add_argument("--library-id", required=True, help="Jellyfin library ID") br.add_argument("--type", help="Comma-separated item types, such as Movie,Series") br.add_argument("--limit", type=int, default=50, help="Maximum items to return (default: 50)") br.add_argument("--start-index", type=int, default=0, help="Zero-based result offset (default: 0)") - sub.add_parser("libraries", help="List libraries", description="List configured media libraries.", epilog="Example: jellyfin-cli libraries") - sub.add_parser("stats", help="Library statistics", description="Show media library item counts.", epilog="Example: jellyfin-cli stats") + sub.add_parser("libraries", help="List libraries", description="List configured media libraries.", epilog="Example: jellyfin libraries") + sub.add_parser("stats", help="Library statistics", description="Show media library item counts.", epilog="Example: jellyfin stats") args = parser.parse_args(filtered_argv[1:]) if not args.command: diff --git a/jellyfin-cli/tests/test_jellyfin_cli.py b/jellyfin/scripts/test_jellyfin_cli.py similarity index 99% rename from jellyfin-cli/tests/test_jellyfin_cli.py rename to jellyfin/scripts/test_jellyfin_cli.py index 428f56a..fd6fcca 100644 --- a/jellyfin-cli/tests/test_jellyfin_cli.py +++ b/jellyfin/scripts/test_jellyfin_cli.py @@ -8,7 +8,7 @@ import subprocess import unittest -SCRIPT = pathlib.Path(__file__).parents[1] / "scripts" / "jellyfin-cli" +SCRIPT = pathlib.Path(__file__).resolve().parent / "jellyfin" LOADER = importlib.machinery.SourceFileLoader("jellyfin_cli", str(SCRIPT)) SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER) jellyfin_cli = importlib.util.module_from_spec(SPEC) diff --git a/jira-cli/README.md b/jira/README.md similarity index 95% rename from jira-cli/README.md rename to jira/README.md index e13c84a..0a5d87e 100644 --- a/jira-cli/README.md +++ b/jira/README.md @@ -18,7 +18,7 @@ When your agent loads this skill, it can **manage your entire Jira workflow** wi | Directory | Purpose | |-----------|---------| | `SKILL.md` | Complete command reference with setup and examples | -| `scripts/jira-cli` | CLI tool for Jira REST API v3 | +| `scripts/jira` | CLI tool for Jira REST API v3 | ## Quick Start diff --git a/jira-cli/SKILL.md b/jira/SKILL.md similarity index 61% rename from jira-cli/SKILL.md rename to jira/SKILL.md index 12c7bfd..5dcf659 100644 --- a/jira-cli/SKILL.md +++ b/jira/SKILL.md @@ -1,5 +1,5 @@ --- -name: jira-cli +name: jira description: 'Interact with Atlassian Jira from the terminal: search issues, view details, create issues, add comments, list projects, and transition status. Use when the user mentions Jira, a ticket key (e.g. PROJ-123), or asks about issues, @@ -13,7 +13,7 @@ metadata: sources: https://developer.atlassian.com/cloud/jira/platform/rest/v3/, https://id.atlassian.com/manage/api-tokens --- -# jira-cli — Jira Issue Tracker from the Terminal +# jira — Jira Issue Tracker from the Terminal Interact with Atlassian Jira Cloud via the REST API v3. Search issues, view details, create issues, add comments, list projects, and transition status. @@ -35,17 +35,17 @@ export JIRA_SERVER="https://your-domain.atlassian.net" # defaults to this forma ### me — Current user profile ```bash -jira-cli me # your account info -jira-cli me --json # machine-readable +jira me # your account info +jira me --json # machine-readable ``` ### list — Search issues ```bash -jira-cli list # recent issues -jira-cli list --project PROJ # by project -jira-cli list --jql 'assignee=currentuser() AND status=Open' # custom JQL -jira-cli list --project PROJ --max 5 --json # top 5 as JSON +jira list # recent issues +jira list --project PROJ # by project +jira list --jql 'assignee=currentuser() AND status=Open' # custom JQL +jira list --project PROJ --max 5 --json # top 5 as JSON ``` The `--jql` flag accepts any valid JQL. The `--project` flag is a shortcut for `project=KEY`. @@ -53,8 +53,8 @@ The `--jql` flag accepts any valid JQL. The `--project` flag is a shortcut for ` ### view — Issue details ```bash -jira-cli view PROJ-123 # full details -jira-cli view PROJ-123 --json # machine-readable +jira view PROJ-123 # full details +jira view PROJ-123 --json # machine-readable ``` Shows: summary, type, status, priority, assignee, reporter, timestamps, and description (plain text extracted from Atlassian Document Format). @@ -62,33 +62,33 @@ Shows: summary, type, status, priority, assignee, reporter, timestamps, and desc ### projects — List projects ```bash -jira-cli projects # all accessible projects -jira-cli projects --json # machine-readable +jira projects # all accessible projects +jira projects --json # machine-readable ``` ### create — Create an issue ```bash -jira-cli create --project PROJ --summary "Fix login bug" # Task (default) -jira-cli create --project PROJ --summary "Crash on startup" --type Bug -jira-cli create --project PROJ --summary "Add dark mode" --type Story --priority High -jira-cli create --project PROJ --summary "Test" --dry-run # preview +jira create --project PROJ --summary "Fix login bug" # Task (default) +jira create --project PROJ --summary "Crash on startup" --type Bug +jira create --project PROJ --summary "Add dark mode" --type Story --priority High +jira create --project PROJ --summary "Test" --dry-run # preview ``` ### comment — Add a comment ```bash -jira-cli comment PROJ-123 -m "Fixed in latest build" # add comment -jira-cli comment PROJ-123 -m "Looking into it" --dry-run +jira comment PROJ-123 -m "Fixed in latest build" # add comment +jira comment PROJ-123 -m "Looking into it" --dry-run ``` ### transition — Change issue status ```bash -jira-cli transition PROJ-123 --to "In Progress" # by name -jira-cli transition PROJ-123 --to "Done" # by name -jira-cli transition PROJ-123 --to "31" # by ID -jira-cli transition PROJ-123 --to "In Review" --dry-run +jira transition PROJ-123 --to "In Progress" # by name +jira transition PROJ-123 --to "Done" # by name +jira transition PROJ-123 --to "31" # by ID +jira transition PROJ-123 --to "In Review" --dry-run ``` The CLI looks up available transitions for the issue and matches by name or ID. If the transition doesn't exist, it shows available options. @@ -98,10 +98,10 @@ The CLI looks up available transitions for the issue and matches by name or ID. All flags work in any position: ```bash -jira-cli --json list --project PROJ # flag before subcommand -jira-cli list --project PROJ --json # flag after subcommand -jira-cli --dry-run create --project PROJ --summary "Test" # preview -jira-cli --quiet list # suppress non-essential output +jira --json list --project PROJ # flag before subcommand +jira list --project PROJ --json # flag after subcommand +jira --dry-run create --project PROJ --summary "Test" # preview +jira --quiet list # suppress non-essential output ``` ## Known Gotchas @@ -114,6 +114,6 @@ jira-cli --quiet list # suppress non-essential output ## References -- [scripts/jira-cli](scripts/jira-cli) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. +- [scripts/jira](scripts/jira) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. - [Jira REST API v3 docs](https://developer.atlassian.com/cloud/jira/platform/rest/v3/) — Official API reference. - [API Token Management](https://id.atlassian.com/manage/api-tokens) — Generate and revoke tokens. diff --git a/jira-cli/scripts/jira-cli b/jira/scripts/jira similarity index 97% rename from jira-cli/scripts/jira-cli rename to jira/scripts/jira index c3677d6..09aea36 100755 --- a/jira-cli/scripts/jira-cli +++ b/jira/scripts/jira @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""jira-cli — Jira issue tracker from the terminal. +"""jira — Jira issue tracker from the terminal. Interact with Atlassian Jira Cloud via the REST API v3. Requires JIRA_EMAIL and JIRA_API_TOKEN env vars (token from @@ -234,7 +234,7 @@ def cmd_me(client: JiraClient, args: List[str]) -> None: def cmd_list(client: JiraClient, args: List[str]) -> None: - parser = argparse.ArgumentParser(prog="jira-cli list") + parser = argparse.ArgumentParser(prog="jira list") parser.add_argument("--jql", default="", help="JQL query (default: recent issues)") parser.add_argument("--max", type=int, default=20, help="Max results (default: 20)") parser.add_argument("--project", help="Shortcut: filter by project key") @@ -282,7 +282,7 @@ def cmd_list(client: JiraClient, args: List[str]) -> None: def cmd_view(client: JiraClient, args: List[str]) -> None: - parser = argparse.ArgumentParser(prog="jira-cli view") + parser = argparse.ArgumentParser(prog="jira view") parser.add_argument("issue_key", help="Issue key (e.g. PROJ-123)") parsed, _ = parser.parse_known_args(args) @@ -377,7 +377,7 @@ def cmd_projects(client: JiraClient, args: List[str]) -> None: def cmd_create(client: JiraClient, args: List[str]) -> None: - parser = argparse.ArgumentParser(prog="jira-cli create") + parser = argparse.ArgumentParser(prog="jira create") parser.add_argument("--project", required=True, help="Project key (e.g. PROJ)") parser.add_argument("--summary", required=True, help="Issue summary/title") parser.add_argument("--type", default="Task", help="Issue type (Task, Bug, Story, etc.)") @@ -407,7 +407,7 @@ def cmd_create(client: JiraClient, args: List[str]) -> None: def cmd_comment(client: JiraClient, args: List[str]) -> None: - parser = argparse.ArgumentParser(prog="jira-cli comment") + parser = argparse.ArgumentParser(prog="jira comment") parser.add_argument("issue_key", help="Issue key (e.g. PROJ-123)") parser.add_argument("--body", "-m", required=True, help="Comment body text") parsed, _ = parser.parse_known_args(args) @@ -426,7 +426,7 @@ def cmd_comment(client: JiraClient, args: List[str]) -> None: def cmd_transition(client: JiraClient, args: List[str]) -> None: - parser = argparse.ArgumentParser(prog="jira-cli transition") + parser = argparse.ArgumentParser(prog="jira transition") parser.add_argument("issue_key", help="Issue key (e.g. PROJ-123)") parser.add_argument("--to", required=True, help="Transition by name or ID") parsed, _ = parser.parse_known_args(args) @@ -474,9 +474,9 @@ def main() -> None: warnings.simplefilter("ignore") parser = argparse.ArgumentParser( - prog="jira-cli", + prog="jira", description="Jira issue tracker from the terminal.", - epilog="Global flags work anywhere: jira-cli --json list --jql 'project=PROJ'" + epilog="Global flags work anywhere: jira --json list --jql 'project=PROJ'" ) sub = parser.add_subparsers(dest="command", help="Available commands") diff --git a/openlibrary-cli/README.md b/openlibrary/README.md similarity index 83% rename from openlibrary-cli/README.md rename to openlibrary/README.md index 0143260..ba310a4 100644 --- a/openlibrary-cli/README.md +++ b/openlibrary/README.md @@ -17,14 +17,14 @@ When your agent loads this skill, it can **access 50M+ book records** without an | Directory | Purpose | |-----------|---------| | `SKILL.md` | Complete command reference with examples | -| `scripts/openlibrary-cli` | CLI tool for the Open Library API | +| `scripts/openlibrary` | CLI tool for the Open Library API | ## Quick Start ```bash -openlibrary-cli search --query "dune" -openlibrary-cli search --isbn "9780439358064" -openlibrary-cli search-authors --query "asimov" +openlibrary search --query "dune" +openlibrary search --isbn "9780439358064" +openlibrary search-authors --query "asimov" ``` ## Triggers diff --git a/openlibrary-cli/SKILL.md b/openlibrary/SKILL.md similarity index 66% rename from openlibrary-cli/SKILL.md rename to openlibrary/SKILL.md index 3e164ba..35a4103 100644 --- a/openlibrary-cli/SKILL.md +++ b/openlibrary/SKILL.md @@ -1,5 +1,5 @@ --- -name: openlibrary-cli +name: openlibrary description: Search books, authors, and works on Open Library from the terminal. Look up books by ISBN, search titles and authors, and fetch detailed work/author records via the public Open Library API. No API key required. @@ -12,7 +12,7 @@ metadata: sources: https://openlibrary.org/developers/api, https://openlibrary.org --- -# openlibrary-cli — Book Metadata from Open Library +# openlibrary — Book Metadata from Open Library Search books and authors, look up works and ISBNs, and fetch detailed metadata from the public [Open Library](https://openlibrary.org) API. No API key, no registration — just works. @@ -32,13 +32,13 @@ Only Python 3.8+ and the `requests` package are required. `--help` and `--dry-ru ### search — Search books by keyword ```bash -openlibrary-cli search --query "dune" # basic search (20 results) -openlibrary-cli search --query "dune" --limit 5 # just the top 5 -openlibrary-cli search --query "dune" --sort new # newest first -openlibrary-cli search --query "dune" --sort rating # highest rated first -openlibrary-cli search --query "dune" --sort title # alphabetical by title -openlibrary-cli search --query "foundation" --lang fr # French editions -openlibrary-cli search --query "dune" --json # machine-readable JSON +openlibrary search --query "dune" # basic search (20 results) +openlibrary search --query "dune" --limit 5 # just the top 5 +openlibrary search --query "dune" --sort new # newest first +openlibrary search --query "dune" --sort rating # highest rated first +openlibrary search --query "dune" --sort title # alphabetical by title +openlibrary search --query "foundation" --lang fr # French editions +openlibrary search --query "dune" --json # machine-readable JSON ``` Shows: title, first publish year, authors, edition count. Supports `--sort` (editions, new, old, rating, title), `--lang`, `--offset`, and `--availability`. @@ -46,9 +46,9 @@ Shows: title, first publish year, authors, edition count. Supports `--sort` (edi ### search-authors — Search authors by name ```bash -openlibrary-cli search-authors --query "asimov" # search authors -openlibrary-cli search-authors --query "asimov" --limit 5 # top 5 -openlibrary-cli search-authors --query "asimov" --json # machine-readable +openlibrary search-authors --query "asimov" # search authors +openlibrary search-authors --query "asimov" --limit 5 # top 5 +openlibrary search-authors --query "asimov" --json # machine-readable ``` Shows: name, birth/death years, author key, top work. @@ -56,9 +56,9 @@ Shows: name, birth/death years, author key, top work. ### author — Get author details by key ```bash -openlibrary-cli author OL23919A # Isaac Asimov -openlibrary-cli author OL34184A # Ursula K. Le Guin -openlibrary-cli author OL23919A --json # full biography +openlibrary author OL23919A # Isaac Asimov +openlibrary author OL34184A # Ursula K. Le Guin +openlibrary author OL23919A --json # full biography ``` Shows: name, birth/death dates, biography (up to 500 chars), Wikipedia link, personal name. Author keys are the `OL#####A` identifiers from search results. @@ -66,9 +66,9 @@ Shows: name, birth/death dates, biography (up to 500 chars), Wikipedia link, per ### work — Get work details by key ```bash -openlibrary-cli work OL123W # work by key -openlibrary-cli work OL81699W # "Foundation" -openlibrary-cli work OL81699W --json # full description +openlibrary work OL123W # work by key +openlibrary work OL81699W # "Foundation" +openlibrary work OL81699W --json # full description ``` Shows: title, author keys, subjects, description (up to 500 chars). Work keys are the `OL#####W` identifiers found in search results. @@ -76,9 +76,9 @@ Shows: title, author keys, subjects, description (up to 500 chars). Work keys ar ### isbn — Lookup a book by ISBN ```bash -openlibrary-cli isbn 9780451524935 # ISBN lookup -openlibrary-cli isbn 9780553382563 # another book -openlibrary-cli isbn 9780451524935 --json # full edition metadata +openlibrary isbn 9780451524935 # ISBN lookup +openlibrary isbn 9780553382563 # another book +openlibrary isbn 9780451524935 --json # full edition metadata ``` Shows: title, author(s), page count, publish date, publisher, subjects (top 5). Accepts both ISBN-10 and ISBN-13. @@ -88,12 +88,12 @@ Shows: title, author(s), page count, publish date, publisher, subjects (top 5). These flags work anywhere in the command — before or after the subcommand: ```bash -openlibrary-cli --json search --query "dune" # JSON output -openlibrary-cli search --query "dune" --json # same result, after subcommand -openlibrary-cli --dry-run search --query "dune" # preview without API call -openlibrary-cli --quiet search --query "dune" # suppress diagnostic output -openlibrary-cli --verbose author OL23919A # verbose logging -openlibrary-cli --dry-run isbn 9780451524935 # see what URL would be used +openlibrary --json search --query "dune" # JSON output +openlibrary search --query "dune" --json # same result, after subcommand +openlibrary --dry-run search --query "dune" # preview without API call +openlibrary --quiet search --query "dune" # suppress diagnostic output +openlibrary --verbose author OL23919A # verbose logging +openlibrary --dry-run isbn 9780451524935 # see what URL would be used ``` | Flag | Effect | @@ -116,6 +116,6 @@ openlibrary-cli --dry-run isbn 9780451524935 # see what URL would ## References -- [scripts/openlibrary-cli](scripts/openlibrary-cli) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, structured logging. +- [scripts/openlibrary](scripts/openlibrary) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, structured logging. - [Open Library API Docs](https://openlibrary.org/developers/api) — Official API documentation. - [Open Library](https://openlibrary.org) — The open, editable library catalog. diff --git a/openlibrary-cli/scripts/openlibrary-cli b/openlibrary/scripts/openlibrary similarity index 94% rename from openlibrary-cli/scripts/openlibrary-cli rename to openlibrary/scripts/openlibrary index dcf6248..8251300 100755 --- a/openlibrary-cli/scripts/openlibrary-cli +++ b/openlibrary/scripts/openlibrary @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""openlibrary-cli — Open Library book metadata from the terminal. +"""openlibrary — Open Library book metadata from the terminal. Search books, authors, works, and lookup by ISBN using the public Open Library API. No API key required. @@ -20,7 +20,7 @@ import requests DEFAULT_SERVER = "https://openlibrary.org" ENV_SERVER = os.getenv("OL_SERVER", DEFAULT_SERVER) ENV_EMAIL = os.getenv("OL_EMAIL", "") -ENV_USER_AGENT = os.getenv("OL_USER_AGENT", "openlibrary-cli/1.0 (+https://github.com)") +ENV_USER_AGENT = os.getenv("OL_USER_AGENT", "openlibrary/1.0 (+https://github.com)") QUIET = False GLOBAL_FLAGS: Dict[str, Any] = { @@ -115,7 +115,7 @@ def fmt_author(a: Dict) -> str: def cmd_search(client, args): - parser = argparse.ArgumentParser(prog="openlibrary-cli search") + parser = argparse.ArgumentParser(prog="openlibrary search") parser.add_argument("--query", "-q", required=True) parser.add_argument("--limit", type=int, default=20) parser.add_argument("--offset", type=int, default=0) @@ -160,7 +160,7 @@ def cmd_search(client, args): def cmd_isbn(client, args): - parser = argparse.ArgumentParser(prog="openlibrary-cli isbn") + parser = argparse.ArgumentParser(prog="openlibrary isbn") parser.add_argument("isbn", help="ISBN number") parsed, _ = parser.parse_known_args(args) @@ -194,7 +194,7 @@ def cmd_isbn(client, args): def cmd_author(client, args): - parser = argparse.ArgumentParser(prog="openlibrary-cli author") + parser = argparse.ArgumentParser(prog="openlibrary author") parser.add_argument("key", help="Author key (e.g. OL23919A)") parsed, _ = parser.parse_known_args(args) @@ -225,7 +225,7 @@ def cmd_author(client, args): def cmd_work(client, args): - parser = argparse.ArgumentParser(prog="openlibrary-cli work") + parser = argparse.ArgumentParser(prog="openlibrary work") parser.add_argument("key", help="Work key (e.g. OL123W)") parsed, _ = parser.parse_known_args(args) @@ -257,7 +257,7 @@ def cmd_work(client, args): def cmd_search_authors(client, args): - parser = argparse.ArgumentParser(prog="openlibrary-cli search-authors") + parser = argparse.ArgumentParser(prog="openlibrary search-authors") parser.add_argument("--query", "-q", required=True) parser.add_argument("--limit", type=int, default=20) parser.add_argument("--offset", type=int, default=0) @@ -298,9 +298,9 @@ def main(): warnings.simplefilter("ignore") parser = argparse.ArgumentParser( - prog="openlibrary-cli", + prog="openlibrary", description="Open Library book metadata from the terminal. No API key required.", - epilog="Global flags work anywhere: openlibrary-cli --json search --query 'dune'" + epilog="Global flags work anywhere: openlibrary --json search --query 'dune'" ) sub = parser.add_subparsers(dest="command") diff --git a/tempest-cli/README.md b/tempest/README.md similarity index 92% rename from tempest-cli/README.md rename to tempest/README.md index 85a4d8b..bb72a4e 100644 --- a/tempest-cli/README.md +++ b/tempest/README.md @@ -17,15 +17,15 @@ When your agent loads this skill, it can **check hyper-local weather from your o | Directory | Purpose | |-----------|---------| | `SKILL.md` | Complete command reference with examples | -| `scripts/tempest-cli` | CLI tool for WeatherFlow Tempest API | +| `scripts/tempest` | CLI tool for WeatherFlow Tempest API | | `references/` | API field layout reference | ## Quick Start ```bash export TEMPEST_TOKEN="your-token-here" -tempest-cli current -tempest-cli forecast +tempest current +tempest forecast ``` ## Triggers diff --git a/tempest-cli/SKILL.md b/tempest/SKILL.md similarity index 77% rename from tempest-cli/SKILL.md rename to tempest/SKILL.md index a140ae1..1400f5e 100644 --- a/tempest-cli/SKILL.md +++ b/tempest/SKILL.md @@ -1,5 +1,5 @@ --- -name: tempest-cli +name: tempest description: 'Query hyper-local weather from a WeatherFlow Tempest station: current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. Use when the user asks about the weather, temperature, rain, wind, humidity, forecast, @@ -12,7 +12,7 @@ metadata: sources: https://weatherflow.com, https://swd.weatherflow.com/swd/rest --- -# tempest-cli — Hyper-Local Weather from Your Tempest Station +# tempest — Hyper-Local Weather from Your Tempest Station Query live weather data from a WeatherFlow Tempest station. Supports REST API access to current conditions, forecasts, and history via the cloud, plus local UDP broadcast reception from your hub on the same LAN. @@ -32,9 +32,9 @@ The CLI reads `TEMPEST_TOKEN` from the environment. It also falls back to readin ### current — Current conditions ```bash -tempest-cli current # human-readable -tempest-cli current --station-id 12345 --device-id 67890 # specific hardware -tempest-cli current --json # machine-readable +tempest current # human-readable +tempest current --station-id 12345 --device-id 67890 # specific hardware +tempest current --json # machine-readable ``` If you have one station, it auto-selects it and picks the best sensor (ST > SKY > AIR, skips the HB hub). Pass `--station-id` or `--device-id` to override. @@ -42,17 +42,17 @@ If you have one station, it auto-selects it and picks the best sensor (ST > SKY ### forecast — Multi-day forecast (+ current conditions + hourly) ```bash -tempest-cli forecast # current + 5-day daily + 12-hour hourly -tempest-cli forecast --days 3 # fewer days -tempest-cli forecast --station-id 12345 # specific station -tempest-cli forecast --json # machine-readable +tempest forecast # current + 5-day daily + 12-hour hourly +tempest forecast --days 3 # fewer days +tempest forecast --station-id 12345 # specific station +tempest forecast --json # machine-readable ``` ### stations — List your stations and devices ```bash -tempest-cli stations # shows station names, IDs, device types, serials -tempest-cli stations --json # full device inventory +tempest stations # shows station names, IDs, device types, serials +tempest stations --json # full device inventory ``` Use this first if you don't know your station ID or want to see what sensors are online. @@ -60,17 +60,17 @@ Use this first if you don't know your station ID or want to see what sensors are ### obs — Historical observations ```bash -tempest-cli obs --device-id 67890 --days 1 # last 24 hours -tempest-cli obs --device-id 67890 --days 7 # last week -tempest-cli obs --device-id 67890 --json # machine-readable +tempest obs --device-id 67890 --days 1 # last 24 hours +tempest obs --device-id 67890 --days 7 # last week +tempest obs --device-id 67890 --json # machine-readable ``` ### udp listen — Real-time broadcasts from the hub ```bash -tempest-cli udp listen # listen indefinitely (Ctrl-C to stop) -tempest-cli udp listen --timeout 30 # auto-stop after 30s -tempest-cli udp listen --show-all # include hub_status messages +tempest udp listen # listen indefinitely (Ctrl-C to stop) +tempest udp listen --timeout 30 # auto-stop after 30s +tempest udp listen --show-all # include hub_status messages ``` Requires being on the same LAN as the hub (port 50222 UDP broadcast). Receives observations, rapid wind updates, lightning strike events, and precipitation start events in real time. @@ -147,11 +147,11 @@ A station returns all devices including the hub (device_type `HB`). The hub cann `--json`, `--dry-run`, `--quiet`, and `--verbose` work anywhere in the command: ```bash -tempest-cli --json current --device-id 67890 # flag before subcommand -tempest-cli current --device-id 67890 --json # flag after subcommand +tempest --json current --device-id 67890 # flag before subcommand +tempest current --device-id 67890 --json # flag after subcommand ``` ## References - [references/tempest-api-field-layouts.md](references/tempest-api-field-layouts.md) — Full field index maps for obs_st, obs_air, and obs_sky observation arrays. Read when decoding raw JSON output or building on top of the Tempest API. -- [scripts/tempest-cli](scripts/tempest-cli) — The CLI binary itself. Designed following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, idempotent, dual-output via `emit()`, and structured logging. +- [scripts/tempest](scripts/tempest) — The CLI binary itself. Designed following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, idempotent, dual-output via `emit()`, and structured logging. diff --git a/tempest-cli/references/tempest-api-field-layouts.md b/tempest/references/tempest-api-field-layouts.md similarity index 100% rename from tempest-cli/references/tempest-api-field-layouts.md rename to tempest/references/tempest-api-field-layouts.md diff --git a/tempest-cli/scripts/tempest-cli b/tempest/scripts/tempest similarity index 98% rename from tempest-cli/scripts/tempest-cli rename to tempest/scripts/tempest index b4429f9..bd5b050 100755 --- a/tempest-cli/scripts/tempest-cli +++ b/tempest/scripts/tempest @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""tempest-cli — Hyper-local weather from your Tempest station. +"""tempest — Hyper-local weather from your Tempest station. Two data sources: REST API — stations, observations, forecast via WeatherFlow cloud @@ -319,7 +319,7 @@ def cmd_stations(client: TempestClient, args: List[str]) -> None: def cmd_current(client: TempestClient, args: List[str]) -> None: """Get latest observations from your station.""" - parser = argparse.ArgumentParser(prog="tempest-cli current") + parser = argparse.ArgumentParser(prog="tempest current") parser.add_argument("--station-id", type=int, help="Station ID (optional if only one station)") parser.add_argument("--device-id", type=int, help="Device ID (default: first Tempest device)") parsed, _ = parser.parse_known_args(args) @@ -385,7 +385,7 @@ def cmd_current(client: TempestClient, args: List[str]) -> None: def cmd_obs(client: TempestClient, args: List[str]) -> None: """Get historical observations.""" - parser = argparse.ArgumentParser(prog="tempest-cli obs") + parser = argparse.ArgumentParser(prog="tempest obs") parser.add_argument("--device-id", type=int, required=True, help="Device ID (required)") parser.add_argument("--days", type=int, default=1, help="Days back to fetch (default: 1)") parsed, _ = parser.parse_known_args(args) @@ -416,7 +416,7 @@ def cmd_obs(client: TempestClient, args: List[str]) -> None: def cmd_forecast(client: TempestClient, args: List[str]) -> None: """Get forecast — current conditions + daily + hourly.""" - parser = argparse.ArgumentParser(prog="tempest-cli forecast") + parser = argparse.ArgumentParser(prog="tempest forecast") parser.add_argument("--station-id", type=int, help="Station ID (optional if only one station)") parser.add_argument("--days", type=int, default=5, help="Days of daily forecast (default: 5)") parsed, _ = parser.parse_known_args(args) @@ -523,7 +523,7 @@ def cmd_forecast(client: TempestClient, args: List[str]) -> None: def udp_listen(args: List[str]) -> None: """Listen for local UDP broadcasts from the Tempest hub.""" - parser = argparse.ArgumentParser(prog="tempest-cli udp listen") + parser = argparse.ArgumentParser(prog="tempest udp listen") parser.add_argument("--port", type=int, default=DEFAULT_UDP_PORT, help=f"UDP port (default: {DEFAULT_UDP_PORT})") parser.add_argument("--timeout", type=int, default=0, help="Listen for N seconds (0 = indefinite)") parser.add_argument("--show-all", action="store_true", help="Show raw message type even if unknown") @@ -623,9 +623,9 @@ def main() -> None: warnings.simplefilter("ignore") parser = argparse.ArgumentParser( - prog="tempest-cli", + prog="tempest", description="Hyper-local weather from your Tempest station.", - epilog="Global flags can appear anywhere: tempest-cli --json current --device-id X" + epilog="Global flags can appear anywhere: tempest --json current --device-id X" ) sub = parser.add_subparsers(dest="command", help="Available commands") diff --git a/tmdb-cli/README.md b/tmdb/README.md similarity index 88% rename from tmdb-cli/README.md rename to tmdb/README.md index 68d3b3b..9035cfa 100644 --- a/tmdb-cli/README.md +++ b/tmdb/README.md @@ -17,14 +17,14 @@ When your agent loads this skill, it can **access the entire TMDb catalog** with | Directory | Purpose | |-----------|---------| | `SKILL.md` | Complete command reference with compound filter examples | -| `scripts/tmdb-cli` | CLI tool for TMDb v3 API | +| `scripts/tmdb` | CLI tool for TMDb v3 API | ## Quick Start ```bash export TMDB_ACCESS_TOKEN="your-tmdb-access-token" -tmdb-cli movie search --term "dune" -tmdb-cli movie discover --genre horror --certification R +tmdb movie search --term "dune" +tmdb movie discover --genre horror --certification R ``` ## Triggers diff --git a/tmdb-cli/SKILL.md b/tmdb/SKILL.md similarity index 54% rename from tmdb-cli/SKILL.md rename to tmdb/SKILL.md index d853fbb..767da70 100644 --- a/tmdb-cli/SKILL.md +++ b/tmdb/SKILL.md @@ -1,5 +1,5 @@ --- -name: tmdb-cli +name: tmdb description: Search and discover movies, TV shows, and trending content via The Movie Database (TMDb) API v3. Use when the user asks about movies, TV, film, cinema, genres, certifications, ratings, cast, upcoming releases, or trending media. @@ -11,7 +11,7 @@ metadata: sources: https://developer.themoviedb.org/reference, https://www.themoviedb.org/settings/api --- -# tmdb-cli — Movie & TV Discovery from the Terminal +# tmdb — Movie & TV Discovery from the Terminal Search movies and TV shows by keyword, discover by genre/certification/rating/date, check trending and upcoming releases, browse genre lists, and view US certification ratings — all from TMDb's v3 API. @@ -33,9 +33,9 @@ export TMDB_API_KEY="your-tmdb-api-key" ### movie search — Search movies by keyword ```bash -tmdb-cli movie search --term "dune" # basic search -tmdb-cli movie search --term "inception" --limit 5 # top 5 results -tmdb-cli movie search --term "arrival" --json # machine-readable +tmdb movie search --term "dune" # basic search +tmdb movie search --term "inception" --limit 5 # top 5 results +tmdb movie search --term "arrival" --json # machine-readable ``` Shows: title, release year, vote average. @@ -43,29 +43,29 @@ Shows: title, release year, vote average. ### movie discover — Discover movies by genre, certification, rating, and date ```bash -tmdb-cli movie discover --genre horror # horror movies -tmdb-cli movie discover --genre horror --certification R # horror, R-rated -tmdb-cli movie discover --genre comedy --rating 7 --limit 15 # highly-rated comedy -tmdb-cli movie discover --from 2024-01-01 --to 2024-12-31 # released in 2024 -tmdb-cli movie discover --genre scifi --from 2026-05-01 # recent sci-fi -tmdb-cli movie discover --genre thriller --certification R \ +tmdb movie discover --genre horror # horror movies +tmdb movie discover --genre horror --certification R # horror, R-rated +tmdb movie discover --genre comedy --rating 7 --limit 15 # highly-rated comedy +tmdb movie discover --from 2024-01-01 --to 2024-12-31 # released in 2024 +tmdb movie discover --genre scifi --from 2026-05-01 # recent sci-fi +tmdb movie discover --genre thriller --certification R \ --rating 6 --from 2025-01-01 --limit 20 # compound filter ``` ### movie upcoming — Upcoming movie releases ```bash -tmdb-cli movie upcoming # next 10 upcoming -tmdb-cli movie upcoming --limit 20 # more results -tmdb-cli movie upcoming --json # machine-readable +tmdb movie upcoming # next 10 upcoming +tmdb movie upcoming --limit 20 # more results +tmdb movie upcoming --json # machine-readable ``` ### tv search — Search TV shows by keyword ```bash -tmdb-cli tv search --term "severance" # basic TV search -tmdb-cli tv search --term "the expanse" --limit 5 -tmdb-cli tv search --term "silo" --json +tmdb tv search --term "severance" # basic TV search +tmdb tv search --term "the expanse" --limit 5 +tmdb tv search --term "silo" --json ``` Shows: name, first air year, vote average. @@ -73,33 +73,33 @@ Shows: name, first air year, vote average. ### tv discover — Discover TV shows by genre, rating, and air date ```bash -tmdb-cli tv discover --genre sci-fi # sci-fi shows -tmdb-cli tv discover --genre drama --rating 7 # critically-acclaimed drama -tmdb-cli tv discover --genre comedy --from 2025-01-01 # recent comedy +tmdb tv discover --genre sci-fi # sci-fi shows +tmdb tv discover --genre drama --rating 7 # critically-acclaimed drama +tmdb tv discover --genre comedy --from 2025-01-01 # recent comedy ``` ### trending — Trending content across day or week ```bash -tmdb-cli trending # trending movies this week -tmdb-cli trending --type tv # trending TV this week -tmdb-cli trending --type all --window day # all media trending today -tmdb-cli trending --limit 20 --json # top 20 as JSON +tmdb trending # trending movies this week +tmdb trending --type tv # trending TV this week +tmdb trending --type all --window day # all media trending today +tmdb trending --limit 20 --json # top 20 as JSON ``` ### genre list — Browse available genres ```bash -tmdb-cli genre list --type movie # all movie genres -tmdb-cli genre list --type tv # all TV genres -tmdb-cli genre list --type movie --json +tmdb genre list --type movie # all movie genres +tmdb genre list --type tv # all TV genres +tmdb genre list --type movie --json ``` ### certification — View US movie certification ratings ```bash -tmdb-cli certification # US certification list -tmdb-cli certification --json # machine-readable +tmdb certification # US certification list +tmdb certification --json # machine-readable ``` ## Global Flags @@ -107,11 +107,11 @@ tmdb-cli certification --json # machine-readable These flags work in any position before, between, or after subcommands: ```bash -tmdb-cli --json movie search --term "dune" # JSON output -tmdb-cli movie search --term "dune" --json # json after subcommand -tmdb-cli --dry-run movie discover --genre horror # preview without API call -tmdb-cli --quiet trending # suppress diagnostic output -tmdb-cli --verbose movie search --term "alien" # verbose logging +tmdb --json movie search --term "dune" # JSON output +tmdb movie search --term "dune" --json # json after subcommand +tmdb --dry-run movie discover --genre horror # preview without API call +tmdb --quiet trending # suppress diagnostic output +tmdb --verbose movie search --term "alien" # verbose logging ``` ## Known Gotchas @@ -124,6 +124,6 @@ tmdb-cli --verbose movie search --term "alien" # verbose logging ## References -- [scripts/tmdb-cli](scripts/tmdb-cli) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. +- [scripts/tmdb](scripts/tmdb) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. - [TMDb API v3 Reference](https://developer.themoviedb.org/reference) — Official API documentation. - [TMDb API Settings (get a key)](https://www.themoviedb.org/settings/api) — Free API key registration. diff --git a/tmdb-cli/scripts/tmdb-cli b/tmdb/scripts/tmdb similarity index 96% rename from tmdb-cli/scripts/tmdb-cli rename to tmdb/scripts/tmdb index 3b4b69c..0257003 100755 --- a/tmdb-cli/scripts/tmdb-cli +++ b/tmdb/scripts/tmdb @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""tmdb-cli — The Movie Database API for agent-based media discovery. +"""tmdb — The Movie Database API for agent-based media discovery. Discover movies and TV shows by genre, rating, certification, and date. Search by term, get details, check trending, upcoming, and now playing. @@ -170,7 +170,7 @@ def fmt_tv(t, idx=None): def cmd_movie_search(client, args): - p = argparse.ArgumentParser(prog="tmdb-cli movie search") + p = argparse.ArgumentParser(prog="tmdb movie search") p.add_argument("--term", "-t", required=True) p.add_argument("--limit", type=int, default=10) parsed, _ = p.parse_known_args(args) @@ -186,7 +186,7 @@ def cmd_movie_search(client, args): def cmd_movie_discover(client, args): - p = argparse.ArgumentParser(prog="tmdb-cli movie discover") + p = argparse.ArgumentParser(prog="tmdb movie discover") p.add_argument("--genre", help="Genre name (e.g. horror, comedy)") p.add_argument("--certification", help="US certification (G, PG, PG-13, R, NC-17)") p.add_argument("--rating", type=float, help="Min vote average (0-10)") @@ -231,7 +231,7 @@ def _resolve_movie_genre(client, name): def cmd_tv_discover(client, args): - p = argparse.ArgumentParser(prog="tmdb-cli tv discover") + p = argparse.ArgumentParser(prog="tmdb tv discover") p.add_argument("--genre", help="Genre name") p.add_argument("--rating", type=float, help="Min vote average") p.add_argument("--from", dest="air_date_gte", help="Air date from") @@ -268,7 +268,7 @@ def _resolve_tv_genre(client, name): def cmd_trending(client, args): - p = argparse.ArgumentParser(prog="tmdb-cli trending") + p = argparse.ArgumentParser(prog="tmdb trending") p.add_argument("--type", default="movie", choices=["movie", "tv", "all"]) p.add_argument("--window", default="week", choices=["day", "week"]) p.add_argument("--limit", type=int, default=10) @@ -294,7 +294,7 @@ def cmd_trending(client, args): def cmd_genre_list(client, args): - p = argparse.ArgumentParser(prog="tmdb-cli genre list") + p = argparse.ArgumentParser(prog="tmdb genre list") p.add_argument("--type", required=True, choices=["movie", "tv"]) parsed, _ = p.parse_known_args(args) @@ -322,7 +322,7 @@ def cmd_cert_list(client, args): def cmd_upcoming(client, args): - p = argparse.ArgumentParser(prog="tmdb-cli movie upcoming") + p = argparse.ArgumentParser(prog="tmdb movie upcoming") p.add_argument("--limit", type=int, default=10) parsed, _ = p.parse_known_args(args) if client.dry_run: @@ -341,7 +341,7 @@ def main(): if GLOBAL_FLAGS.get("json", False): warnings.simplefilter("ignore") - parser = argparse.ArgumentParser(prog="tmdb-cli", description="The Movie Database API CLI.") + parser = argparse.ArgumentParser(prog="tmdb", description="The Movie Database API CLI.") sub = parser.add_subparsers(dest="resource") # movie From ff1dab9273564ef4117780041f12e6f840a75877 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 02:13:56 -0400 Subject: [PATCH 02/40] docs(skills): repair catalog cross-references after -cli renames Update every audited reference site to the six renamed skills: root README headings and links (and drop the jira-jql entry ahead of its absorption), references/skill-triggers.md rows for jellyfin and tempest plus four new-name rows for ghost/jira/openlibrary/tmdb, scripts/grandfathered-skills.txt pruned to the three retained *-cli entries, pyproject.toml deptry exclusion jellyfin-cli -> jellyfin, neckbeard routing seams, cli-builder example names, and the lastfm / verification-methodology eval texts mentioning jellyfin. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- README.md | 16 ++++++---------- cli-builder/SKILL.md | 2 +- lastfm/SKILL.md | 2 +- lastfm/evals/evals.json | 2 +- neckbeard/README.md | 2 +- neckbeard/references/routing-table.md | 2 +- neckbeard/references/tracker-discovery.md | 2 +- pyproject.toml | 2 +- references/skill-triggers.md | 8 ++++++-- scripts/grandfathered-skills.txt | 10 ---------- verification-methodology/evals/evals.json | 6 +++--- 11 files changed, 22 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 26a3a04..c355d36 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ Build and maintain web frontends — component architecture, state management, A Guide a person in cultivating creativity in their own work and life: open conversational sessions on creative blocks, habits, environment, motivation, and resilience, or structured development of a concrete project or fledgling idea through a five-phase practice. Do not use for therapy or clinical support, general life coaching, product or stakeholder discovery, or as a study guide for a book. -### [ghost-cli](ghost-cli/SKILL.md) +### [ghost](ghost/SKILL.md) Ghost CMS from the terminal. Manage posts and pages, list tags, and check site info. Admin API key from Ghost Integrations. JWT authentication handled automatically. @@ -256,18 +256,14 @@ Turn an approved requirement or specification into an executable, dependency-awa Convert operational incident and near-miss evidence into verified, owned improvements across product, code, tests, evals, operations, and governance. Separates observed facts from causal hypotheses and unresolved uncertainty; maps escaped-from gaps (requirements, monitoring, authority, migration, adoption); assigns domain-specific follow-up work with owners and verification methods; and requires evidence of the implemented change — not just tickets — for closure. Routes implementation to SRE, QA, verification, agent evals, product lifecycle learning, implementation planning, resilience-and-recovery, and production-readiness. Ships 4 references (discovery brief, evidence/inference taxonomy, escaped-from analysis, follow-up domains, verification and closure), 4 templates (incident-learning record, causal/evidence ledger, follow-up work map, verification and closure record), and 5 evals. -### [jellyfin-cli](jellyfin-cli/SKILL.md) +### [jellyfin](jellyfin/SKILL.md) Jellyfin media server from the terminal. Check server info, browse recently added and library contents, search and inspect media, see next-up episodes, and view statistics. -### [jira-cli](jira-cli/SKILL.md) +### [jira](jira/SKILL.md) Atlassian Jira from the terminal. Search issues with JQL, view details, create issues, add comments, list projects, and transition status. API token from id.atlassian.com. -### [jira-jql](jira-jql/SKILL.md) - -Expert-level Jira Query Language reference covering all operators, functions (date/time, user, sprint/version, issue, custom field, JSM), history operators (WAS/CHANGED), relative dates, performance best practices, role-based ready queries, REST API usage, and troubleshooting. Three companion references: complete function catalog, role-specific query bank (dev, scrum master, PO, power user, admin), and gotchas/troubleshooting guide. - ### [kanban-guru](kanban-guru/SKILL.md) A virtual Kanban expert for engineering teams. Diagnose flow problems, design board configurations, calibrate WIP limits, establish service level expectations, set up multi-portfolio operating models, and navigate Scrum-to-Kanban transitions. Covers all seven cadences, classes of service, Little's Law, flow metrics, and the full practitioner's playbook with rich reference material. @@ -359,7 +355,7 @@ prompt templates for text-only and reference-image-driven workflows. Google's Open Knowledge Format (OKF) v0.1 — create, validate, and consume vendor-neutral AI agent knowledge bundles. Markdown files with YAML frontmatter, organized in directory hierarchies with cross-links and progressive disclosure. Ships a validation script, concept template, example bundle, and detailed references covering the spec, bundle architecture, and real-world use cases. -### [openlibrary-cli](openlibrary-cli/SKILL.md) +### [openlibrary](openlibrary/SKILL.md) Open Library book metadata from the terminal. Search books and authors, get work and edition details, lookup by ISBN. No API key required — the public Open Library API is free for everyone. @@ -567,7 +563,7 @@ Turn technology preferences and architecture-governance choices into explicit, r Operate the Prometheus + OpenTelemetry + Loki observability stack as one unit: scrape config, recording and alerting rules, relabeling, retention, and HA; OpenTelemetry Collector pipelines (receivers, processors, exporters, sampling, trace/span correlation); and Loki ingest, LogQL, retention, and label design. Ships the read-only `telemetry-check` script (rule sanity + scrape-target reachability, `--json`), fixtures, tests, dated references, and 6 evals. Routes strategy to platform-engineering and dashboards to grafana. -### [tempest-cli](tempest-cli/SKILL.md) +### [tempest](tempest/SKILL.md) Hyper-local weather from a WeatherFlow Tempest station. Query current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. A complete reference implementation of the cli-builder patterns in a working, testable project — including the CLI binary and full API field layout reference. @@ -579,7 +575,7 @@ Operate Terraform and OpenTofu safely across the whole infrastructure lifecycle: Build browser-based Three.js and WebGL scenes, animations, and interactive 3D visualizations. -### [tmdb-cli](tmdb-cli/SKILL.md) +### [tmdb](tmdb/SKILL.md) The Movie Database API from the terminal. Search and discover movies and TV by genre, certification, rating, and date range. Check trending, upcoming, and now playing. Free API key from themoviedb.org. diff --git a/cli-builder/SKILL.md b/cli-builder/SKILL.md index fb08119..52caa97 100644 --- a/cli-builder/SKILL.md +++ b/cli-builder/SKILL.md @@ -54,7 +54,7 @@ Capture data shapes → Run tests as-you-go → Fix failures Each API or data source gets its own CLI. Do not combine disparate services into one tool. -**Correct:** `tmdb-cli` (TMDb only), `ghost-cli` (Ghost CMS only) +**Correct:** `tmdb` (TMDb only), `ghost` (Ghost CMS only) **Wrong:** `media-cli` (combines TMDb + Trakt + Radarr) Exception: services from the same vendor sharing auth (e.g. Radarr + Sonarr). diff --git a/lastfm/SKILL.md b/lastfm/SKILL.md index 62b7d47..2e28ed0 100644 --- a/lastfm/SKILL.md +++ b/lastfm/SKILL.md @@ -120,7 +120,7 @@ The core flow for turning liked tracks into recommendations: 4. **Cross-reference** against what they've already scrobbled (recent-tracks) to filter out already-heard material. -5. **Check against the user's own collection** (via Radarr/Sonarr/jellyfin-cli or Spotify library) to see what's already in the library vs genuinely new discovery. +5. **Check against the user's own collection** (via Radarr/Sonarr/jellyfin or Spotify library) to see what's already in the library vs genuinely new discovery. ## Using with --json for Machine Processing diff --git a/lastfm/evals/evals.json b/lastfm/evals/evals.json index 6d64f54..0c53ef6 100644 --- a/lastfm/evals/evals.json +++ b/lastfm/evals/evals.json @@ -48,7 +48,7 @@ { "id": "route-away-from-music-data", "prompt": "Play some Radiohead songs for me right now.", - "expected_output": "Scenario: should-not-trigger. Playback is not a Last.fm data operation — the skill covers metadata, history, charts, discovery, and scrobbling, not controlling a music player. The agent does not load lastfm or invoke the CLI; it routes playback to whatever local player/streaming integration exists (e.g., jellyfin-cli or a Spotify tool) instead.", + "expected_output": "Scenario: should-not-trigger. Playback is not a Last.fm data operation — the skill covers metadata, history, charts, discovery, and scrobbling, not controlling a music player. The agent does not load lastfm or invoke the CLI; it routes playback to whatever local player/streaming integration exists (e.g., jellyfin or a Spotify tool) instead.", "assertions": [ "The skill is not loaded for a playback request", "No lastfm-cli command is run to fulfill playback", diff --git a/neckbeard/README.md b/neckbeard/README.md index 88d18d5..d5885bb 100644 --- a/neckbeard/README.md +++ b/neckbeard/README.md @@ -44,7 +44,7 @@ ordinary bug-fix reproduction requirements. | `references/lifecycle.md` | Platform mechanics for GitHub (reference mode) and enterprise contexts — intake snapshots, CI/review monitoring, terminal states, and post-merge release authority | | `references/delivery-packet.md` | Durable cross-phase handoff: provenance, resumability, gate verdicts, exact-head binding, lifecycle states, and an artifact ownership map | | `references/position-assessment.md` | Entry mode for picking up delivery work that started elsewhere: artifact inventory, phase exit-condition scoring, a position report, and packet bootstrap | -| `references/tracker-discovery.md` | Intake sub-step that detects which tracking system holds the work item and routes tracker operations to the matching tooling skill (`linear`, `jira-cli`, `notion`) | +| `references/tracker-discovery.md` | Intake sub-step that detects which tracking system holds the work item and routes tracker operations to the matching tooling skill (`linear`, `jira`, `notion`) | | `references/evaluation.md` | Evaluation methodology: fixtures, baselines, rubrics, multi-run reporting, claims policy | | `templates/` | Change contract, decision record, evidence ledger, verification plan, evaluation report | | `templates/delivery-packet.md` | Fillable delivery packet template mirroring the nine field groups defined in the reference | diff --git a/neckbeard/references/routing-table.md b/neckbeard/references/routing-table.md index dd57dcd..6b1d33d 100644 --- a/neckbeard/references/routing-table.md +++ b/neckbeard/references/routing-table.md @@ -54,7 +54,7 @@ defaults without qualification. | `agent-evals-and-observability` | Change modifies AI/agent behavior: eval definitions, agent task contracts, grader bindings, trajectory fixtures, prompt templates, or agent observability/telemetry | No agent eval, task contract, grader, trajectory fixture, prompt template, or agent telemetry is created or modified | Contract, ledger | | `opensource-contributions` | **Conditional — public/OSS repos only.** Repository remote is public, an open-source license is present, and a `CONTRIBUTING.md` or equivalent contribution governance file exists (verify via repo remote or `gh api`); contribution norms, agent disclosure, or fork etiquette apply | Repository is private or enterprise-internal (non-public remote, no open-source license); record skip as "non-public repository." Also skip when no contribution-norm question arises even in a public repo | Contract, ledger | | `linear` | The product's tracking system was identified as Linear by [tracker-discovery.md](tracker-discovery.md) during intake, and tracker operations are needed (read/update work items, transitions, comments) | Tracking system is not Linear; or the change needs no tracker operation beyond reading provenance already captured | Contract, ledger | -| `jira-cli` | The tracking system was identified as Jira by [tracker-discovery.md](tracker-discovery.md), and tracker operations are needed | Tracking system is not Jira; or no tracker operation is needed beyond captured provenance | Contract, ledger | +| `jira` | The tracking system was identified as Jira by [tracker-discovery.md](tracker-discovery.md), and tracker operations are needed | Tracking system is not Jira; or no tracker operation is needed beyond captured provenance | Contract, ledger | | `notion` | The tracking system was identified as Notion by [tracker-discovery.md](tracker-discovery.md), and tracker operations are needed | Tracking system is not Notion; or no tracker operation is needed beyond captured provenance | Contract, ledger | ## Test-hardening evidence diff --git a/neckbeard/references/tracker-discovery.md b/neckbeard/references/tracker-discovery.md index d58b8a0..1114a41 100644 --- a/neckbeard/references/tracker-discovery.md +++ b/neckbeard/references/tracker-discovery.md @@ -85,7 +85,7 @@ improvising API calls: |---|---| | GitHub (issues, PRs, releases) | Native mechanics per [lifecycle.md](lifecycle.md) — the documented reference mode | | Linear | `linear` | -| Jira | `jira-cli` | +| Jira | `jira` | | Notion | `notion` | | Other / none of the above | No specialist route: operate only through the system's verified official interface (primary vendor documentation, confirmed endpoint/auth surface), with bounded reads; note the absent specialist in the ledger | diff --git a/pyproject.toml b/pyproject.toml index 36170f1..f9c1018 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,7 +111,7 @@ extend_exclude = [ "life-coach", "raleigh", "forgejo-cli", - "jellyfin-cli", + "jellyfin", "comic-chat", "fireflies", "linear", diff --git a/references/skill-triggers.md b/references/skill-triggers.md index 65ae715..268fa9e 100644 --- a/references/skill-triggers.md +++ b/references/skill-triggers.md @@ -18,8 +18,12 @@ Each skill's `description` field is the canonical routing contract. This conveni | "self-hosted runner", "github actions runner", "CI runner", "set up a runner", "runner registration", "runner won't register", "autoscaling runners", "runner security", "runner group", "ARC", "Actions Runner Controller", "runner scale set", "myoung34/github-runner", "ephemeral runner", "just-in-time runner", "runner container image", "runner custom image", "runner network", "runner troubleshooting", "runner monitoring" | [github-runner](../github-runner/SKILL.md) | | "Grafana", "Grafana dashboard", "Grafana panel", "Grafana variable", "Grafana data source", "Grafana alerting", "contact point", "notification policy", "mute timing", "Grafana provisioning", "dashboard as code", "Grafana API", "Grafana service account", "Grafana RBAC", "Grafana plugin", "Grafana troubleshooting", "duplicate dashboard UID" | [grafana](../grafana/SKILL.md) | | "hugo theme", "hugo cms", "accessible theme", "wcag theme", "theme design", "theme accessibility", "theme UX", "design tokens", "css theme", "theme contrast", "responsive theme", "hugo template", "hugo pipes", "hugo module", "hugo shortcode", "render hook", "tailwindcss hugo", "hugo i18n", "hugo seo", "hugo output format", "hugo site", "hugo static site" | [hugo-theme](../hugo-theme/SKILL.md) | -| "Jellyfin", "Jellyfin media server", "recently added movies", "recently added episodes", "media library", "JELLYFIN_API_KEY" | [jellyfin-cli](../jellyfin-cli/SKILL.md) | -| "weather", "forecast", "temperature", "is it raining", "Tempest" | [tempest-cli](../tempest-cli/SKILL.md) | +| "Ghost", "Ghost CMS", "ghost blog", "create a post on my blog", "blog publishing", "GHOST_ADMIN_KEY" | [ghost](../ghost/SKILL.md) | +| "Jellyfin", "Jellyfin media server", "recently added movies", "recently added episodes", "media library", "JELLYFIN_API_KEY" | [jellyfin](../jellyfin/SKILL.md) | +| "Jira", "Atlassian Jira", "JQL", "ticket PROJ-123", "sprint work", "JIRA_API_TOKEN" | [jira](../jira/SKILL.md) | +| "Open Library", "openlibrary", "book search", "ISBN lookup", "author records", "work details" | [openlibrary](../openlibrary/SKILL.md) | +| "weather", "forecast", "temperature", "is it raining", "Tempest" | [tempest](../tempest/SKILL.md) | +| "TMDb", "The Movie Database", "movie search", "trending movies", "upcoming TV releases", "TMDB_ACCESS_KEY" | [tmdb](../tmdb/SKILL.md) | | "traefik", "reverse proxy", "load balancer", "API gateway", "Let's Encrypt", "ACME", "Docker routing", "traefik.yml", "entry point", "middleware", "TLS termination", "forward auth", "rate limit" | [traefik](../traefik/SKILL.md) | | "reverse-engineer", "understand this codebase", "PRD from code", "architecture document", "architecture health", "coupling analysis", "modularity", "decomposition readiness", "data ownership map", "distributed workflow analysis", "reconciliation path" | [software-architecture-analysis](../software-architecture-analysis/SKILL.md) | | "backend service", "service layer", "domain/application/infrastructure", "unit of work", "domain event implementation", "transactional outbox", "inbox deduplication", "idempotent handler", "event replay handler", "message consumer implementation", "service coexistence", "strangler handoff", "service adapter", "anti-corruption adapter", "dual path authority" | [backend-engineering](../backend-engineering/SKILL.md) | diff --git a/scripts/grandfathered-skills.txt b/scripts/grandfathered-skills.txt index 1d9eaa4..9893c4c 100644 --- a/scripts/grandfathered-skills.txt +++ b/scripts/grandfathered-skills.txt @@ -29,15 +29,11 @@ flaresolverr flaresolverr-cli forgejo-cli frontend-engineering -ghost-cli github-runner go-to-market gutenberg haystack hugo-theme -jellyfin-cli -jira-cli -jira-jql kanban-guru kubernetes langchain @@ -51,11 +47,9 @@ meshcore-packet-capture ml-engineering nous-branding open-knowledge-format -openlibrary-cli opensource-contributions operational-design org-design -peertube platform-engineering product-design-and-ux product-discovery @@ -79,12 +73,8 @@ supabase systematic-debugging technical-documentation technology-radar -tempest-cli three -tmdb-cli traefik -trakt -transistor vercel-eve web-accessibility woodpecker-ci diff --git a/verification-methodology/evals/evals.json b/verification-methodology/evals/evals.json index a486225..1786130 100644 --- a/verification-methodology/evals/evals.json +++ b/verification-methodology/evals/evals.json @@ -4,10 +4,10 @@ "evals": [ { "id": "configured-jellyfin-server-first", - "prompt": "The jellyfin-cli skill is installed, its bundled CLI and valid server configuration are available, and the user asks: ‘What’s new on Jellyfin?’ Verify the answer.", - "expected_output": "A source-faithful investigation that loads jellyfin-cli and uses the skill-resolved bundled CLI to query recently added items on the configured server before considering adjacent integrations or public project sources.", + "prompt": "The jellyfin skill is installed, its bundled CLI and valid server configuration are available, and the user asks: ‘What’s new on Jellyfin?’ Verify the answer.", + "expected_output": "A source-faithful investigation that loads jellyfin and uses the skill-resolved bundled CLI to query recently added items on the configured server before considering adjacent integrations or public project sources.", "assertions": [ - "Loads the jellyfin-cli skill before investigating service content.", + "Loads the jellyfin skill before investigating service content.", "The first content query uses the bundled CLI's documented recent-items command through the path resolved by the skill.", "Does not query Home Assistant, GitHub, web search, or public Jellyfin project activity before attempting the configured server.", "Reports the configured server's results as server state rather than conflating them with project news." From 2619ba395f4696ef7da2a789622389ae1990bc37 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 02:26:59 -0400 Subject: [PATCH 03/40] docs(skills): add When-not-to-use boundaries to renamed skills The quality validator enforces an imperative-verb description plus a negative boundary on every changed SKILL.md. The six renamed skills and cli-builder carried no substantive boundary, so add a tailored When-not-to-use section to each naming concrete alternatives (dashboard administration, sibling platforms, adjacent skills). cli-builder lands at 499 lines, inside the 500-line budget. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- cli-builder/SKILL.md | 5 +++++ ghost/SKILL.md | 4 ++++ jellyfin/SKILL.md | 4 ++++ jira/SKILL.md | 4 ++++ openlibrary/SKILL.md | 4 ++++ tempest/SKILL.md | 4 ++++ tmdb/SKILL.md | 4 ++++ 7 files changed, 29 insertions(+) diff --git a/cli-builder/SKILL.md b/cli-builder/SKILL.md index 52caa97..c112bab 100644 --- a/cli-builder/SKILL.md +++ b/cli-builder/SKILL.md @@ -484,6 +484,11 @@ The default for this repo is **`scripts/` inside the skill** — it follows the ## Agent-Readiness Checklist Use [the agent-readiness checklist](references/agent-readiness-checklist.md) before shipping a CLI. + +## When not to use + +Do not use this skill to design conversational agent tools or MCP servers — [references/mcp-vs-cli.md](references/mcp-vs-cli.md) carries that decision framework — and route general API design questions to [api-design-and-evolution](../api-design-and-evolution/SKILL.md). This skill also does not cover GUI, TUI, or web-app interface design. + ## References - [templates/bash-cli-scaffold.sh](templates/bash-cli-scaffold.sh) — Full bash project template with pre-wired global flags, logging helpers, and subcommand dispatch. Use as a starting point for any bash CLI. diff --git a/ghost/SKILL.md b/ghost/SKILL.md index 546aaf0..701ce72 100644 --- a/ghost/SKILL.md +++ b/ghost/SKILL.md @@ -122,3 +122,7 @@ ghost --verbose site # verbose logging - [scripts/ghost](scripts/ghost) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. - [Ghost Admin API Docs](https://ghost.org/docs/admin-api/) — Official Ghost Admin API documentation. - [Ghost Integrations](https://ghost.org/docs/integrations/) — How to create Custom Integrations and get your Admin API key. + +## When not to use + +Do not use this skill for Ghost site administration that requires the admin dashboard (themes, staff accounts, membership tiers, sending settings), for front-end theme development, or for other publishing platforms — WordPress, Hugo, and Jekyll each have their own tooling. diff --git a/jellyfin/SKILL.md b/jellyfin/SKILL.md index d9bbc87..4ab9bbc 100644 --- a/jellyfin/SKILL.md +++ b/jellyfin/SKILL.md @@ -132,3 +132,7 @@ scripts/jellyfin --dry-run search --query "dune" # preview request without - [scripts/jellyfin](scripts/jellyfin) — The bundled read-only CLI binary with `--json`, `--dry-run`, and lazy authentication. - [Jellyfin API Docs](https://jellyfin.org/docs/general/clients/api) — Official API documentation. - [Jellyfin Downloads](https://jellyfin.org/downloads) — Server download and setup guide. + +## When not to use + +Do not use this skill for playback control or library management (starting streams, editing item metadata, creating users) — every bundled command is read-only; route remote-control automation to Jellyfin's official clients, and Plex or Kodi servers expose their own separate APIs. diff --git a/jira/SKILL.md b/jira/SKILL.md index 5dcf659..a24e4c6 100644 --- a/jira/SKILL.md +++ b/jira/SKILL.md @@ -117,3 +117,7 @@ jira --quiet list # suppress non-essential output - [scripts/jira](scripts/jira) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. - [Jira REST API v3 docs](https://developer.atlassian.com/cloud/jira/platform/rest/v3/) — Official API reference. - [API Token Management](https://id.atlassian.com/manage/api-tokens) — Generate and revoke tokens. + +## When not to use + +Do not use this skill for GitHub or GitLab issue tracking (each platform has its own tooling), for Jira site administration such as project permissions, workflow schemes, or user management, or for writing application code against the Jira REST API — see the Atlassian developer docs for integration development instead. diff --git a/openlibrary/SKILL.md b/openlibrary/SKILL.md index 35a4103..0378aa4 100644 --- a/openlibrary/SKILL.md +++ b/openlibrary/SKILL.md @@ -119,3 +119,7 @@ openlibrary --dry-run isbn 9780451524935 # see what URL would be - [scripts/openlibrary](scripts/openlibrary) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, structured logging. - [Open Library API Docs](https://openlibrary.org/developers/api) — Official API documentation. - [Open Library](https://openlibrary.org) — The open, editable library catalog. + +## When not to use + +Do not use this skill for local library-catalog administration (Koha, Evergreen, MARC batch processing), for licensed commercial book data feeds (ISBNdb, Google Books), or for citation formatting — Open Library is a free public catalog API, not a bibliographic management system. diff --git a/tempest/SKILL.md b/tempest/SKILL.md index 1400f5e..a361efd 100644 --- a/tempest/SKILL.md +++ b/tempest/SKILL.md @@ -155,3 +155,7 @@ tempest current --device-id 67890 --json # flag after subcommand - [references/tempest-api-field-layouts.md](references/tempest-api-field-layouts.md) — Full field index maps for obs_st, obs_air, and obs_sky observation arrays. Read when decoding raw JSON output or building on top of the Tempest API. - [scripts/tempest](scripts/tempest) — The CLI binary itself. Designed following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, idempotent, dual-output via `emit()`, and structured logging. + +## When not to use + +Do not use this skill for weather questions that do not involve a personal WeatherFlow station (a public forecast service serves those better), for aviation METAR/TAF data, or for hardware from other vendors — every endpoint here requires a Tempest account token and talks to WeatherFlow's consumer API. diff --git a/tmdb/SKILL.md b/tmdb/SKILL.md index 767da70..8ef2dc9 100644 --- a/tmdb/SKILL.md +++ b/tmdb/SKILL.md @@ -127,3 +127,7 @@ tmdb --verbose movie search --term "alien" # verbose logging - [scripts/tmdb](scripts/tmdb) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. - [TMDb API v3 Reference](https://developer.themoviedb.org/reference) — Official API documentation. - [TMDb API Settings (get a key)](https://www.themoviedb.org/settings/api) — Free API key registration. + +## When not to use + +Do not use this skill for streaming-availability lookups (TMDb delegates watch-provider data to JustWatch and may lag), for torrent or piracy search, or for tracking what you have already watched — TMDb is a metadata database, not a viewing source; pair it with trakt for personal watch history. From 17032f8fc7fe5fb5a5cce94fd9631380c068b233 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 02:34:57 -0400 Subject: [PATCH 04/40] docs(triggers): cite TMDB_ACCESS_TOKEN env var in tmdb trigger row Align the convenience-index keyword with the env vars the bundled tmdb script actually reads (TMDB_ACCESS_TOKEN or TMDB_API_KEY). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- references/skill-triggers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/references/skill-triggers.md b/references/skill-triggers.md index 268fa9e..48d0983 100644 --- a/references/skill-triggers.md +++ b/references/skill-triggers.md @@ -23,7 +23,7 @@ Each skill's `description` field is the canonical routing contract. This conveni | "Jira", "Atlassian Jira", "JQL", "ticket PROJ-123", "sprint work", "JIRA_API_TOKEN" | [jira](../jira/SKILL.md) | | "Open Library", "openlibrary", "book search", "ISBN lookup", "author records", "work details" | [openlibrary](../openlibrary/SKILL.md) | | "weather", "forecast", "temperature", "is it raining", "Tempest" | [tempest](../tempest/SKILL.md) | -| "TMDb", "The Movie Database", "movie search", "trending movies", "upcoming TV releases", "TMDB_ACCESS_KEY" | [tmdb](../tmdb/SKILL.md) | +| "TMDb", "The Movie Database", "movie search", "trending movies", "upcoming TV releases", "TMDB_ACCESS_TOKEN" | [tmdb](../tmdb/SKILL.md) | | "traefik", "reverse proxy", "load balancer", "API gateway", "Let's Encrypt", "ACME", "Docker routing", "traefik.yml", "entry point", "middleware", "TLS termination", "forward auth", "rate limit" | [traefik](../traefik/SKILL.md) | | "reverse-engineer", "understand this codebase", "PRD from code", "architecture document", "architecture health", "coupling analysis", "modularity", "decomposition readiness", "data ownership map", "distributed workflow analysis", "reconciliation path" | [software-architecture-analysis](../software-architecture-analysis/SKILL.md) | | "backend service", "service layer", "domain/application/infrastructure", "unit of work", "domain event implementation", "transactional outbox", "inbox deduplication", "idempotent handler", "event replay handler", "message consumer implementation", "service coexistence", "strangler handoff", "service adapter", "anti-corruption adapter", "dual path authority" | [backend-engineering](../backend-engineering/SKILL.md) | From 28d990301294239ca7c6291ce72a75a372ff1017 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 03:14:48 -0400 Subject: [PATCH 05/40] docs(jira): absorb jira-jql corpus into reference files Migrate the retired jira-jql skill content into three dense jira/references/ files, preserving full substance: - jql-functions-catalog.md: complete function catalog with fields and operators, including JSM approval functions (approved/pending/ approver/pendingApprovalBy/myApproval family) and SLA functions (breached/running/paused/completed/remaining/withinCalendarHours) - jql-best-practices.md: performance rules, operator precedence (AND binds tighter than OR), the empty-value trap (!= excludes nulls), troubleshooting flows, marketplace extensions - jql-cookbook.md: 50 ready-to-run queries organized by role (developers, scrum masters, product owners/managers, power users, admins) Wire a Reference Files routing table into jira/SKILL.md pointing at all three; add JQL gotchas (empty-value trap, precedence, leading wildcards, search-endpoint duality) and two multi-step pipeline recipes to SKILL.md. Each migrated file carries a Sources footer with verified Atlassian doc URLs plus attribution to the retired skill. Token parity vs source corpus: 53/53 function tokens preserved. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- jira-jql/README.md | 34 -- jira-jql/SKILL.md | 386 ------------------ jira-jql/references/best-practices.md | 93 ----- jira-jql/references/role-queries.md | 156 ------- jira/SKILL.md | 52 ++- jira/references/jql-best-practices.md | 147 +++++++ jira/references/jql-cookbook.md | 271 ++++++++++++ .../references/jql-functions-catalog.md | 167 +++++--- 8 files changed, 584 insertions(+), 722 deletions(-) delete mode 100644 jira-jql/README.md delete mode 100644 jira-jql/SKILL.md delete mode 100644 jira-jql/references/best-practices.md delete mode 100644 jira-jql/references/role-queries.md create mode 100644 jira/references/jql-best-practices.md create mode 100644 jira/references/jql-cookbook.md rename {jira-jql => jira}/references/jql-functions-catalog.md (51%) diff --git a/jira-jql/README.md b/jira-jql/README.md deleted file mode 100644 index 09493e5..0000000 --- a/jira-jql/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Jira Query Language (JQL) — Expert Reference - -The complete reference for writing, debugging, and optimizing JQL queries. Covers every operator, function, and pattern used in Jira issue search. - -## Why Install This Skill - -When your agent loads this skill, it becomes a **JQL expert** who can: - -- **Write any JQL query** — from simple field comparisons to complex date ranges and history operators -- **Use all JQL functions** — date/time, user, sprint/version, issue, custom field, JSM functions -- **Query history** — WAS, WAS IN, CHANGED operators for trend and sprint analysis -- **Optimize performance** — indices, avoiding expensive clauses, best practices -- **Build role-specific queries** — ready-made queries for developers, scrum masters, product owners, and admins -- **Debug query problems** — common gotchas and troubleshooting patterns - -## What You Get - -| Directory | Purpose | -|-----------|---------| -| `SKILL.md` | Core syntax, operators, keywords, functions catalog | -| `references/` | Complete function catalog, role-specific query bank, gotchas & troubleshooting guide | - -## Triggers - -Load this when writing, debugging, or optimizing JQL queries; building saved filters, dashboard gadgets, or automation rules. - -## Requirements - -None — this is a reference skill. No scripts, no API keys. - - -## Quick Start - -Start with the setup and first workflow in SKILL.md, then use the linked resources for the specific task you need to complete. diff --git a/jira-jql/SKILL.md b/jira-jql/SKILL.md deleted file mode 100644 index 57f7106..0000000 --- a/jira-jql/SKILL.md +++ /dev/null @@ -1,386 +0,0 @@ ---- -name: jira-jql -description: >- - Expert-level skill for Jira Query Language (JQL). Use when the user asks about - writing, debugging, optimizing, or understanding JQL queries; needs to filter - Jira issues by complex criteria, date ranges, history, or cross-project conditions; - wants to build saved filters, dashboard gadgets, or automation rules; or needs - guidance on JQL performance, functions, operators, history operators (WAS/CHANGED), - relative dates, role-based query patterns, or the JQL REST API. -license: MIT -compatibility: Compatible with any agent supporting the Agent Skills format -metadata: - source: "Atlassian official docs + community best practices" - spec-version: "1.0" - topics: "jql,jira,query-language,atlassian" -allowed-tools: terminal web_search web_extract ---- - -# Jira Query Language (JQL) — Expert Reference - -JQL is Atlassian's structured query language for searching Jira issues (now called "work items"). Every clause is **Field + Operator + Value**, chained with keywords. - -Use this skill when: -- The user asks for help writing or debugging a JQL query -- They need to find issues across projects, sprints, versions, components -- They want to use history operators (WAS, CHANGED) for trend/sprint analysis -- Performance optimization or saved filter design is needed -- They're building automation rules, REST API calls, or dashboard gadgets with JQL - ---- - -## 1. Core Syntax - -``` -field OPERATOR value [AND|OR field OPERATOR value ...] [ORDER BY field [ASC|DESC]] -``` - -### Operators - -| Operator | Meaning | Example | -|----------|---------|---------| -| `=`, `!=` | Equals, not equals | `assignee = currentUser()` | -| `>`, `<`, `>=`, `<=` | Comparison | `created >= -7d` | -| `IN`, `NOT IN` | Set membership | `status IN ("To Do", "In Progress")` | -| `IS`, `IS NOT` | Null check — only with `EMPTY` or `NULL` | `assignee IS EMPTY` | -| `~`, `!~` | Contains (text search) | `summary ~ "login*"` | -| `WAS`, `WAS NOT` | Historical value | `assignee WAS "jsmith"` | -| `WAS IN`, `WAS NOT IN` | Historical set | `fixVersion WAS IN ("Sprint 1", "Sprint 2")` | -| `CHANGED` | Field transition | `status CHANGED FROM "Open" TO "Done"` | - -### Keywords - -| Keyword | Purpose | -|---------|---------| -| `AND` | Both conditions must be true (binds tighter than OR) | -| `OR` | At least one condition must be true | -| `NOT` | Negates a clause | -| `ORDER BY` | Sorting — add `ASC` or `DESC` (default ASC) | -| `EMPTY` / `NULL` | Used with `IS` / `IS NOT` | - -### Precedence - -**AND binds tighter than OR.** `A OR B AND C` = `A OR (B AND C)`. Always parenthesize OR groups: - -```jql --- Correct -(project = A OR project = B) AND status = Open - --- Wrong — reads as project = A OR (project = B AND status = Open) -project = A OR project = B AND status = Open -``` - ---- - -## 2. Available Fields (System) - -Common indexed fields that JQL accepts: - -`project`, `issuetype`, `status`, `assignee`, `reporter`, `creator`, `priority`, `resolution`, `resolutiondate`, `created`, `updated`, `duedate`, `fixVersion`, `affectedVersion`, `component`, `labels`, `sprint`, `votes`, `watchers`, `workRatio`, `parentEpic`, `issueLinkType`, `statusCategory` - -Custom fields work by name — quote if they contain spaces: `"Story Points"`. - -**Pro tip:** Prefer IDs over names for project/sprint/version when possible — names change, IDs (`project = 1001`) don't. - ---- - -## 3. Functions (Complete Catalog) - -### Date/Time Relative Functions - -All accept optional increment strings in `(+/-)nn(y|M|w|d|h|m)` format. Default unit matches the function's natural period. - -| Function | Default Unit | Example | -|----------|-------------|---------| -| `startOfDay()` / `endOfDay()` | `d` | `created > startOfDay("-1")` = yesterday | -| `startOfWeek()` / `endOfWeek()` | `w` | `due < endOfWeek("+1w")` = end of next week | -| `startOfMonth()` / `endOfMonth()` | `M` | `created > startOfMonth("-1")` = start of last month | -| `startOfYear()` / `endOfYear()` | `y` | `resolutiondate > startOfYear()` | -| `now()` | — | Current timestamp | -| `currentLogin()` | — | When session began | -| `lastLogin()` | — | Previous login | - -### User Functions - -| Function | Fields | Operators | Behavior | -|----------|--------|-----------|----------| -| `currentUser()` | Assignee, Reporter, Voter, Watcher, Creator + custom User | `=`, `!=` | Your identity | -| `membersOf("group")` | Assignee, Reporter, Voter, Watcher, Creator | `IN`, `NOT IN`, `WAS IN`, `WAS NOT IN` | Group members. For teams: `membersOf(id:)` | -| `componentsLeadByUser(user)` | Component | `IN`, `NOT IN` | Omit user = current user | -| `spacesLeadByUser(user)` | Project (Space) | `IN`, `NOT IN` | Omit user = current user | -| `spacesWhereUserHasPermission(p)` | Project | `IN`, `NOT IN` | e.g. `"Edit work items"` | -| `spacesWhereUserHasRole(role)` | Project | `IN`, `NOT IN` | e.g. `"Administrators"` | - -### Sprint/Version Functions - -| Function | Fields | Behavior | -|----------|--------|----------| -| `openSprints()` | Sprint | Active, not yet completed | -| `closedSprints()` | Sprint | Completed sprints | -| `earliestUnreleasedVersion(project)` | AffectedVersion, FixVersion, custom Version | Earliest unreleased in release order | -| `latestReleasedVersion(project)` | Same | Most recently released version | -| `releasedVersions(project)` | Same | All released. Omit project for all | -| `unreleasedVersions(project)` | Same | All unreleased. Omit project for all | - -### Issue Functions - -| Function | Syntax | Description | -|----------|--------|-------------| -| `linkedIssues(key, linkType?)` | `issue in linkedIssues("ABC-44")` | Link type optional — e.g. `"is blocked by"` | -| `parentEpic` (field) | `parentEpic = DEMO-123` | Stories/subtasks in an epic | -| `issueHistory()` | `issue in issueHistory()` | Recently viewed | -| `votedWorkItems()` | `issue in votedWorkItems()` | You voted on these | -| `watchedWorkItems()` | `issue in watchedWorkItems()` | You watch these | -| `updatedBy(user, from?, to?)` | `issue in updatedBy(jsmith, "-8d")` | Updated by user. Rounds < 1d up to 1d | - -### Jira Service Management Functions - -| Function | Field Type | Effect | -|----------|-----------|--------| -| `approved()`, `pending()` | Custom Approval | Approval state | -| `approver(user)`, `pendingApprovalBy(user)` | Custom Approval | Specific approver | -| `myApproval()`, `myPendingApproval()` | Custom Approval | Current user as approver | -| `breached()`, `running()`, `paused()`, `completed()` | SLA | SLA state | -| `remaining()` | SLA | Compare remaining time | -| `withinCalendarHours()` | SLA | Running within calendar | -| `customerDetail("Field", "Value")` | Reporter, Organization | Customer attribute search | -| `organizationDetail("Field", "Value")` | Organization | Org attribute search | -| `organizationMembers("Org")` | Reporter, Assignee | Members of organization | - -### Custom Field Functions - -| Function | Field Type | Use With | -|----------|-----------|----------| -| `cascadeOption(parent, child?)` | Cascading Select | `IN`, `NOT IN` | -| `choiceOption(value1, value2...)` | Multiple Choice / Dropdown | `IN`, `NOT IN` | - -Use `none` keyword to search for empty cascade tiers: `location in cascadeOption("USA", none)`. - ---- - -## 4. History Operators (WAS / CHANGED) - -JQL can search issue *history*, not just current state. This is unique to JQL vs SQL. - -### WAS / WAS NOT / WAS IN / WAS NOT IN - -``` -assignee WAS "jsmith" -- Was assigned to jsmith at any point -fixVersion WAS "Sprint A" -- Was in Sprint A historically -status WAS IN ("In Progress", "Under Review") -- Was any of these statuses -``` - -WAS supports a **predicate clause** for time bounds: - -``` -status WAS "In Progress" DURING (startOfWeek(), endOfWeek()) -assignee WAS "jsmith" BEFORE "2024/01/01" -status WAS "Open" BY currentUser() -``` - -### CHANGED - -``` -status CHANGED FROM "Open" TO "Done" -- Transition took place -status CHANGED FROM "Open" TO "Done" AFTER -1d -- Today -status CHANGED TO "Done" BY currentUser() -- Who did it -resolution CHANGED TO "Fixed" DURING (startOfYear(), endOfYear()) -- Year recap -``` - -**Supported operators for `CHANGED` predicate:** `AFTER`, `BEFORE`, `DURING`, `BY`, `FROM`, `TO` - ---- - -## 5. Relative Date Expressions - -All date fields support ISO 8601 absolute dates AND relative offsets: - -| Expression | Meaning | -|-----------|---------| -| `-1d` | 1 day ago | -| `-2w` | 2 weeks ago | -| `+1M` | 1 month from now | -| `-3h` | 3 hours ago | -| `-1y` | 1 year ago | -| `"2026-05-22"` | Absolute date | -| `"2026/05/22"` | Absolute date (alternative) | - -Common dynamic patterns: - -```jql -created >= -7d -- Last 7 days -duedate >= startOfWeek() AND duedate <= endOfWeek() -- This week -resolutiondate >= startOfDay(-3M) AND resolutiondate < endOfDay(-3M) -- Exactly 3 months ago -created > startOfMonth("-1") -- Since start of last month -updated < -30d -- Zombie tickets (not touched in 30 days) -``` - ---- - -## 6. Best Practices & Performance - -### Write Efficient Queries - -1. **Filter by project first** — narrows the search space immediately -2. **Use `IN` instead of chained `OR`** — `status IN (3 values)` vs `status = X OR status = Y OR status = Z` -3. **Prefer indexed fields** — `project`, `issuetype`, `status`, `assignee` are indexed -4. **Avoid negations** — `!=`, `!~`, `NOT`, `NOT IN` scan wider -5. **No leading wildcards** — `summary ~ "*bug"` forces full-scan -6. **Don't sort in JQL if downstream sorts** — redundant sort wastes time - -### Handle Empty Values Correctly - -`!=` does NOT include empty/null values. Explicitly include EMPTY: - -```jql --- Finds all issues NOT assigned to current user, INCLUDING unassigned -(assignee != currentUser() OR assignee IS EMPTY) - --- NOT this — misses unassigned issues -assignee != currentUser() -``` - -### Organize Saved Filters - -- **Break complex queries into reusable sub-filters** — save sub-queries as filters, then compose: `filter = "Unresolved ABC bugs" AND assignee = currentUser()` -- **Consistent naming convention**: `{Sprint}_{Epic}_{OrderedBy}` — e.g. `CurrentSprint_AudioDevEpic_OrderedByAssignee` -- **Use relative dates in saved filters** — they stay dynamic: `created >= startOfMonth()` - -### Scope-Sort Pattern - -Start broad, narrow iteratively: - -```jql --- Step 1: all open issues -project = PWC AND status = open - --- Step 2: narrow by sprint -project = PWC AND status = open AND fixVersion = "Current Sprint" - --- Step 3: carried-over issues only -project = PWC AND status = open AND fixVersion = "Current Sprint" AND fixVersion WAS "Last Sprint" - --- Step 4: sort by priority then assignee -... ORDER BY priority, assignee -``` - ---- - -## 7. Role-Based Ready Queries - -### Developers - -```jql --- My unresolved issues by priority -assignee = currentUser() AND resolution = Unresolved ORDER BY priority DESC - --- Bugs I reported -reporter = currentUser() AND status != Done - --- My completed work this week -resolution = Fixed AND resolutiondate >= -7d AND assignee = currentUser() - --- Where I'm mentioned in comments -comment ~ currentUser() -``` - -### Scrum Masters - -```jql --- Unassigned in active sprint -sprint IN openSprints() AND assignee IS EMPTY - --- Stale tickets -status NOT IN (Closed, Done) AND updated < -30d - --- Recently completed -status CHANGED TO Done AFTER startOfWeek() - --- Reopened tickets (quality flag) -status CHANGED FROM Done TO "In Progress" - --- Team's in-progress work -assignee in membersOf("Dev Team") AND status = "In Progress" -``` - -### Product Owners - -```jql --- Pre-release readiness -fixVersion = earliestUnreleasedVersion() AND status != Done - --- Critical/Highest unresolved bugs -priority IN (Critical, Highest) AND resolution = Unresolved - --- Due this sprint -duedate >= startOfMonth() AND duedate <= endOfMonth() AND resolution = Unresolved - --- Pending approvals (JSM) -approvals = pending() -``` - -### Cross-Project Portfolio - -```jql -project in ("Project Mercury", "PTC") AND issuetype in ("Epic", "Task") AND created >= -180d -``` - ---- - -## 8. Gotchas & Known Limitations - -- **Standard JQL has no aggregation** — no COUNT, SUM, AVG. Use dashboard gadgets or marketplace apps. -- **Can't check linked issue status** — `issueLinkType = "is blocked by"` finds links but can't check if the blocker is resolved. Needs ScriptRunner. -- **No recursive hierarchy traversal** — epics+stories+subtasks need separate queries. -- **`updatedBy()` rounds < 1 day up to 1 day** — `updatedBy(jsmith, "-1h")` becomes 1 day. -- **`membersOf()` does NOT support project roles** — only groups and teams. -- **`IS EMPTY` works for fields that exist** — can't find issues where a field was *never* created. -- **Atlassian is renaming "issue" to "work item"** — old terms (`project`, `issue`, `fixVersion`) still work; no migration needed. -- **Starting a text search with `*` is very expensive** — put wildcards after the first few chars. - -### Marketplace Extensions for Advanced Needs - -| Extension | What It Adds | -|-----------|-------------| -| JQL Tricks Plugin | 50+ extra functions | -| JQL Search Extensions (Cloud) | Find comments, attachments, subtasks, epics | -| JQL Booster Pack (Server/DC) | 15+ user-related functions | -| ScriptRunner (Adaptavist) | Custom Groovy JQL functions — most powerful | - ---- - -## 9. JQL in REST API - -Query via Jira REST API v3: - -```bash -curl -u email:token \ - "https://your-domain.atlassian.net/rest/api/3/search?jql=project=PWC+AND+status=Open&fields=summary,assignee" -``` - -Returns structured JSON. Use `jql` parameter, URL-encode when needed. Also supports `startAt`, `maxResults`, `fields`, `expand` params. - ---- - -## 10. Edge Cases & Troubleshooting - -**Query is valid but slow:** Check for leading wildcards, unindexed custom fields, or missing project filter. - -**Query returns 0 results unexpectedly:** Verify field names haven't changed (esp. custom fields), check for case sensitivity in values (depends on Jira config), and ensure you're in the right project scope. - -**"Filter not found" when using `filter =`:** The user doesn't have permission to that saved filter. - -**`CHANGED` returns nothing:** Ensure the field actually has tracking enabled. Some custom fields don't log history. - -**Jira says "Field 'X' does not exist":** The field name is wrong, disabled for this project, or requires a marketplace app. - ---- - -## Key Reference URLs - -- **Official JQL Functions:** https://support.atlassian.com/jira-software-cloud/docs/jql-functions/ -- **JQL Operators:** https://support.atlassian.com/jira-software-cloud/docs/jql-operators/ -- **JQL Fields:** https://support.atlassian.com/jira-software-cloud/docs/jql-fields/ -- **JQL Keywords:** https://support.atlassian.com/jira-software-cloud/docs/jql-keywords/ -- **JQL Performance KB:** https://confluence.atlassian.com/jirakb/understanding-jql-performance-720416549.html -- **Free Atlassian University Intro to JQL:** https://www.youtube.com/watch?v=BcHKXSiOHqw diff --git a/jira-jql/references/best-practices.md b/jira-jql/references/best-practices.md deleted file mode 100644 index cedf954..0000000 --- a/jira-jql/references/best-practices.md +++ /dev/null @@ -1,93 +0,0 @@ -# JQL Best Practices, Gotchas & Troubleshooting - -## Performance Optimization - -### Do's -- **Filter by project first** — narrows the search space immediately -- **Use `IN` over chained `OR`** — `status IN ("X", "Y")` vs `status = X OR status = Y` -- **Prefer indexed fields** — `project`, `issuetype`, `status`, `assignee` are always indexed -- **Use IDs for stable entities** — `project = 1001` survives renames; `project = "Old Name"` breaks -- **Break complex queries into saved filters** — reference with `filter = "Saved Filter Name"` - -### Don'ts -- **Don't lead with wildcards** — `summary ~ "*bug"` forces full-text scan across all issues -- **Don't overuse negations** — `!=`, `!~`, `NOT` scan wider than positive conditions -- **Don't sort in JQL when downstream sorts** — redundant sorting wastes server cycles -- **Don't mix AND/OR without parentheses** — AND binds tighter than OR, results will surprise - -## Common Mistakes - -| Mistake | Example | Fix | -|---------|---------|-----| -| Missing EMPTY on negation | `assignee != currentUser()` | `(assignee != currentUser() OR assignee IS EMPTY)` | -| AND/OR precedence | `A OR B AND C` | `(A OR B) AND C` | -| Name vs ID fragility | `project = "My Project"` | `project = 1001` | -| Searching by renamed sprint | `sprint = "Sprint 1"` | Use sprint ID | -| Status but no resolution | `status = Done` | Add `resolution IS NOT EMPTY` or `resolution = Fixed` | -| Missing timezone offsets | `created > startOfDay()` | Jira uses the user's configured timezone | -| Forgetting sprint scope | `sprint IS EMPTY` | Also check `sprint NOT IN openSprints()` for backlog | - -## Gotchas - -### Core Platform -- **Atlassian is renaming "issue" to "work item"** — old terms (project, issue, fixVersion) still work. No migration needed. New docs use "work item" but queries are backward-compatible. -- **JQL has NO aggregation** — COUNT, SUM, AVG don't exist. Must use dashboard gadgets (pie chart, statistics), marketplace apps (eazyBI, ScriptRunner), or Jira REST API with external processing. -- **No recursive hierarchy traversal** — can't grab epics + their stories + their subtasks in one query. Run three separate JQL statements or use marketplace plugins. -- **`IS EMPTY` only works for fields that exist** — can't find issues that *never had a value* for a field. -- **`!=` excludes nulls** — always pair with `OR field IS EMPTY` if you want truly everything except the value. - -### Function-Specific -- **`membersOf()` does NOT support project roles** — only Jira groups and teams (by teamId). -- **`updatedBy()` rounds to 1 day minimum** — `updatedBy(jsmith, "-1h")` becomes `-1d`. -- **`votedWorkItems()` / `watchedWorkItems()` capped at 32,000** — if you watch more, results truncate silently. -- **`cascadeOption(none)` is keyword-based** — to literally search for a value "none", wrap in quotes: `cascadeOption("\"none\"")`. - -### Jira Cloud-Specific -- **Saved filters can share names** — Jira doesn't prevent duplicates. Bad for dashboard gadgets. -- **Auto-suggest list depends on permissions** — if you can't see a field in autocomplete, you may lack permission for it. -- **JQL AI assistant exists** — button left of JQL bar in Cloud. Early-stage; useful for beginners but misses nuance. -- **Some custom fields don't log history** — `CHANGED` and `WAS` won't work on fields without history tracking enabled. - -### ScriptRunner (if available) -- **Powerful but heavier** — `issueFunction` can create custom JQL expressions but adds latency -- **Common use:** `issueFunction in hasLinkType("Epic-Story Link")`, `issueFunction in commented("by user after -1d")` - -## Troubleshooting Flow - -**"No results, query looks valid"** -1. Check field name spelling (custom fields especially) -2. Verify project/sprint/version existence -3. Check case sensitivity — values may be case-sensitive depending on Jira config -4. Try `ORDER BY created DESC` to confirm query works but returns no matches - -**"Query is very slow"** -1. Remove leading wildcards from text searches -2. Add project filter first -3. Replace `OR` chains with `IN` -4. Remove negations if possible -5. Reduce result set with tighter date/status filters - -**"Field doesn't exist"** -1. Field may be disabled for this project -2. May be a custom field from an uninstalled marketplace app -3. Check if the user running it has permission to that field - -**"CHANGED returns nothing"** -1. The field may not track history -2. Verify date range — `CHANGED TO "Done" AFTER startOfDay()` may be too narrow -3. Check that the transition actually happened (some workflows skip statuses) - -**"WAS operator returns no results"** -1. The field must have a trackable history (most system fields do) -2. Verify the value was ever set — `WAS` returns nothing if the field was always current value - -## Marketplace Extensions - -| Plugin | Hosting | What It Adds | -|--------|---------|-------------| -| JQL Tricks Plugin | Server/DC | 50+ extra functions | -| JQL Search Extensions | Cloud | Find issues, comments, attachments, subtasks, versions, epics, links | -| JQL Booster Pack | Server/DC | 15+ user-related functions, archived version filtering | -| JQL Functions Collection | Server/DC | String and date format functions | -| Groups & Organizations JQL | Server/DC | Match multi-group custom field values | -| ScriptRunner | Cloud/Server/DC | Custom Groovy JQL functions — most powerful and flexible | diff --git a/jira-jql/references/role-queries.md b/jira-jql/references/role-queries.md deleted file mode 100644 index 685702b..0000000 --- a/jira-jql/references/role-queries.md +++ /dev/null @@ -1,156 +0,0 @@ -# JQL — Role-Based Ready Queries - -## Developers - -```jql --- My plate, sorted by urgency -assignee = currentUser() AND resolution = Unresolved ORDER BY priority DESC - --- Bugs I reported that haven't been fixed -reporter = currentUser() AND status != Done - --- Where I'm mentioned (standup prep) -comment ~ currentUser() - --- My completed work this week -resolution = Fixed AND resolutiondate >= -7d AND assignee = currentUser() - --- This week's deadlines -duedate >= startOfWeek() AND duedate <= endOfWeek() - --- My blocked tickets -issueLinkType = "is blocked by" AND assignee = currentUser() - --- Subtasks of a specific story -parent = "PROJ-123" - --- Watched but not closed -watcher = currentUser() AND status != Closed - --- Full-text search across summary, description, comments -text ~ "error message here" - --- Recent unplanned work -created >= -3d AND assignee = currentUser() AND resolution = Unresolved -``` - -## Scrum Masters - -```jql --- Unassigned in active sprint -sprint IN openSprints() AND assignee IS EMPTY - --- Zombie tickets (not touched in 30 days) -status NOT IN (Closed, Done) AND updated < -30d - --- Recently completed this sprint -status CHANGED TO Done AFTER startOfWeek() - --- Reopened tickets (quality regression) -status CHANGED FROM Done TO "In Progress" - --- Volatility — new issues created this sprint -sprint IN openSprints() AND created >= -1w - --- Team's in-flight work -assignee in membersOf("Dev Team") AND status = "In Progress" - --- Carried-over issues (current sprint + was in previous) -fixVersion = "Current Sprint" AND fixVersion WAS "Last Sprint" - --- Issues blocked (any) -issueLinkType = "is blocked by" - --- Sprint capacity check -sprint IN openSprints() AND assignee in membersOf("Dev Team") -``` - -## Product Owners / Managers - -```jql --- Pre-launch readiness -fixVersion = earliestUnreleasedVersion() AND status != Done - --- Firefighting view -priority IN (Critical, Highest) AND resolution = Unresolved - --- Upcoming due dates -duedate >= startOfMonth() AND duedate <= endOfMonth() AND resolution = Unresolved - --- Pending approvals (JSM) -approvals = pending() - --- Epics without stories attached (requires ScriptRunner) -issuetype = Epic AND issueFunction not in hasLinkType("Epic-Story Link") - --- Component-level tech debt -component = "Backend" AND status != Done ORDER BY priority DESC - --- Recently reported bugs by component -issuetype = Bug AND created >= -14d ORDER BY created DESC - --- Cross-project view for portfolio -project in ("Project Mercury", "PTC") AND issuetype in ("Epic", "Task") AND status = "To Do" AND created >= -180d - --- Status-category aggregation -statusCategory = "In Progress" OR statusCategory = "To Do" - --- Feature completeness by version -fixVersion = "v2.0" AND status != Done ORDER BY component, priority -``` - -## Power Users - -```jql --- Issues where custom cascading select has specific values -location in cascadeOption("USA", "New York") - --- Issues assigned to any admin -assignee in membersOf("jira-administrators") - --- Issues where I was the previous assignee -assignee WAS currentUser() - --- Resolved by current user this year (retrospective) -resolution CHANGED TO "Fixed" BY currentUser() DURING (startOfYear(), endOfYear()) - --- Issues updated by specific user in last week -issue in updatedBy(jsmith, "-8d") - --- Issues linked through specific link type -issue in linkedIssues("PROJ-123", "is duplicated by") - --- Issues that were in a specific sprint historically -sprint WAS "Sprint 5" - --- Epics with their children -parentEpic = "PROJ-EPIC-1" - --- Everything that changed status in last 24h -status CHANGED AFTER -1d - --- All subtask issue types -issuetype in subtaskWorkTypes() - --- Issues that breached SLA (JSM) -SLA = breached() -``` - -## Automation / Admin Queries - -```jql --- Issues from users not in a group (for access reviews) -reporter NOT IN membersOf("internal-users") - --- Issues in projects user has no role in (permission audit) -project NOT IN spacesWhereUserHasRole("Developers") - --- Old unassigned tickets (automation: assign or close) -assignee IS EMPTY AND created < -90d AND resolution = Unresolved - --- Bulk transition candidates -status = "In Progress" AND updated < -14d - --- Stale sprints (automation: warn or move) -sprint IN closedSprints() AND resolution = Unresolved -``` diff --git a/jira/SKILL.md b/jira/SKILL.md index a24e4c6..0c0cc9f 100644 --- a/jira/SKILL.md +++ b/jira/SKILL.md @@ -1,9 +1,12 @@ --- name: jira -description: 'Interact with Atlassian Jira from the terminal: search issues, view - details, create issues, add comments, list projects, and transition status. Use - when the user mentions Jira, a ticket key (e.g. PROJ-123), or asks about issues, - bugs, tasks, projects, or sprint work.' +description: 'Interact with Atlassian Jira from the terminal: search issues with + JQL, view details, create issues, add comments, list projects, and transition + status. Includes a full JQL language reference (functions, operators, history + queries, performance tuning). Use when the user mentions Jira, a ticket key + (e.g. PROJ-123), asks about issues, bugs, tasks, projects, or sprint work, + or needs to write, debug, or optimize JQL queries. Do not use for GitHub or + GitLab issue tracking, Jira site administration, or generic ticketing systems.' license: MIT compatibility: Requires JIRA_EMAIL and JIRA_API_TOKEN env vars (free from id.atlassian.com/manage/api-tokens), Python 3.8+, and the `requests` library. Also requires JIRA_SERVER (defaults to @@ -112,6 +115,47 @@ jira --quiet list # suppress non-essential output - **Rate limits** — Jira Cloud has rate limits. The API returns 429 if exceeded. The CLI does not auto-retry. - **Project keys are case-sensitive** in some contexts, but the Jira API generally accepts uppercase or lowercase. +### JQL gotchas + +- **`!=` excludes empty values** — `assignee != currentUser()` silently drops unassigned issues. Write `(assignee != currentUser() OR assignee IS EMPTY)` to include them. +- **AND binds tighter than OR** — `A OR B AND C` parses as `A OR (B AND C)`. Always parenthesize OR groups. +- **No leading wildcards** — `summary ~ "*bug"` forces a full scan and is very slow; put wildcards after the first few characters. +- **Filter by project first** — the single biggest JQL performance lever on large instances. +- **JQL has no aggregation** — no COUNT/SUM/AVG in the query language itself. +- **History operators need history tracking** — `WAS`/`CHANGED` return nothing for custom fields without history enabled. +- **Search endpoint duality** — this CLI uses the classic `/rest/api/3/search` endpoint with offset pagination (`startAt`, `maxResults`). Atlassian's enhanced `/rest/api/3/search/jql` replaces it with a `nextPageToken` model and no offset; the classic endpoint is being deprecated, so expect migration. Mixing the two pagination models is a common source of truncated or erroring result pages. + +## Multi-Step Pipeline Recipes + +### Sprint hygiene sweep + +Find stalled sprint work, then bulk-review each ticket: + +```bash +jira list --jql 'sprint IN openSprints() AND updated < -14d AND status != Done' --json \ + | jq -r '.issues[].key' \ + | while read -r key; do jira view "$key"; done +``` + +The `--json` output shape from `list` is `{"total": N, "issues": [{"key", "summary", "status", "assignee", "issuetype", "priority"}]}` — pipe through `jq -r '.issues[].key'` to feed follow-up commands. + +### My-week digest + +```bash +jira list --jql 'assignee = currentUser() AND updated >= startOfWeek()' --max 50 --json \ + | jq -r '.issues[] | "\(.key)\t\(.status)\t\(.summary)"' +``` + +More ready-to-run queries live in [references/jql-cookbook.md](references/jql-cookbook.md), organized by role (developers, scrum masters, product owners, admins). + +## Reference Files + +| File | Topic | Read when | +|------|-------|-----------| +| [references/jql-functions-catalog.md](references/jql-functions-catalog.md) | Every JQL function with fields/operators — date/time, user, sprint/version, issue, custom field, plus JSM approval and SLA functions | Writing or debugging a query that uses functions; checking which operators a function supports | +| [references/jql-best-practices.md](references/jql-best-practices.md) | Performance rules, operator precedence, the empty-value trap, common mistakes, troubleshooting flow, marketplace extensions | A query is slow, returns wrong/zero results, or mixes AND/OR | +| [references/jql-cookbook.md](references/jql-cookbook.md) | 50 ready-to-run JQL queries organized by role (developers, scrum masters, product owners/managers, power users, admins) | Building dashboards, saved filters, automation rules, or sprint reviews | + ## References - [scripts/jira](scripts/jira) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. diff --git a/jira/references/jql-best-practices.md b/jira/references/jql-best-practices.md new file mode 100644 index 0000000..1520ddc --- /dev/null +++ b/jira/references/jql-best-practices.md @@ -0,0 +1,147 @@ +# JQL Best Practices, Performance & Troubleshooting + +Performance rules, the traps JQL springs on unwary writers, and a troubleshooting flow for queries that misbehave. Pairs with [jql-functions-catalog.md](jql-functions-catalog.md) for the function reference and [jql-cookbook.md](jql-cookbook.md) for ready-to-run queries. + +## Operator Precedence + +**AND binds tighter than OR.** `A OR B AND C` parses as `A OR (B AND C)` — almost never what you meant. Always parenthesize OR groups: + +```text +-- Correct +(project = A OR project = B) AND status = Open + +-- Wrong — reads as project = A OR (project = B AND status = Open) +project = A OR project = B AND status = Open +``` + +Mixing AND and OR without parentheses is listed in the common-mistakes table below for a reason: results will surprise you. + +## Performance Optimization + +### Do's + +- **Filter by project first** — narrows the search space immediately +- **Use `IN` over chained `OR`** — `status IN ("X", "Y")` vs `status = X OR status = Y` +- **Prefer indexed fields** — `project`, `issuetype`, `status`, `assignee` are always indexed +- **Use IDs for stable entities** — `project = 1001` survives renames; `project = "Old Name"` breaks when the name changes +- **Break complex queries into saved filters** — save the sub-query once, then compose with `filter = "Saved Filter Name"` +- **Keep relative dates in saved filters** — they stay dynamic instead of freezing at creation time + +### Don'ts + +- **Don't lead with wildcards** — `summary ~ "*bug"` forces a full-text scan across all issues; starting a text search with `*` is very expensive, put wildcards after the first few characters +- **Don't overuse negations** — `!=`, `!~`, `NOT IN`, `NOT` scan wider than positive conditions +- **Don't sort in JQL when downstream sorts** — redundant sorting wastes server cycles +- **Don't mix AND/OR without parentheses** — see precedence rule above + +## The Empty-Value Trap + +Negation does **not** include empty values. `!=` excludes nulls, so a plain negation silently drops unassigned issues. Explicitly include EMPTY: + +```jql +-- Finds all issues NOT assigned to current user, INCLUDING unassigned +(assignee != currentUser() OR assignee IS EMPTY) + +-- NOT this — misses unassigned issues entirely +assignee != currentUser() +``` + +The same trap applies to every negated comparison (`!=`, `NOT IN`) on optional fields. If you want "everything except X", write `(field != X OR field IS EMPTY)`. + +## Common Mistakes + +| Mistake | Example | Fix | +|---------|---------|-----| +| Missing EMPTY on negation | `assignee != currentUser()` | `(assignee != currentUser() OR assignee IS EMPTY)` | +| AND/OR precedence | `A OR B AND C` | `(A OR B) AND C` | +| Name vs ID fragility | `project = "My Project"` | `project = 1001` (IDs survive renames) | +| Searching by renamed sprint | `sprint = "Sprint 1"` | Use the sprint ID | +| Status but no resolution | `status = Done` | Add `resolution IS NOT EMPTY` or `resolution = Fixed` | +| Missing timezone offsets | `created > startOfDay()` | Jira evaluates dates in the user's configured timezone | +| Forgetting sprint scope | `sprint IS EMPTY` | Also check `sprint NOT IN openSprints()` to catch backlog items | + +## Gotchas + +### Core Platform + +- **Atlassian is renaming "issue" to "work item"** — old terms (project, issue, fixVersion) still work; no migration needed. New docs say "work item", but existing queries are backward-compatible. +- **JQL has NO aggregation** — COUNT, SUM, AVG do not exist. Use dashboard gadgets (pie chart, statistics), marketplace apps such as eazyBI or ScriptRunner, or pull via REST API and aggregate externally. +- **No recursive hierarchy traversal** — you cannot fetch epics + their stories + their subtasks in one query. Run separate statements per level, or use marketplace plugins. +- **`IS EMPTY` only works for fields that exist** — it cannot find issues where a field was *never* given a value. +- **`!=` excludes nulls** — pair it with `OR field IS EMPTY` whenever you want truly everything except a value. + +### Function-Specific + +- **`membersOf()` does NOT support project roles** — only Jira groups and teams (by team id). +- **`updatedBy()` rounds to a 1-day minimum** — `updatedBy(jsmith, "-1h")` behaves as `-1d`. +- **`votedWorkItems()` / `watchedWorkItems()` cap at 32,000** — beyond that, results truncate silently. +- **`cascadeOption(none)` is keyword-based** — to literally match a value of "none", quote-escape it: `cascadeOption("\"none\"")`. + +### Jira Cloud-Specific + +- **Saved filters can share names** — Jira does not prevent duplicates, which makes name-based dashboard gadget references fragile. +- **Auto-suggest depends on permissions** — if a field never appears in autocomplete, you may simply lack permission to it. +- **A JQL AI assistant exists in Cloud** — button left of the JQL bar. Early-stage; useful for beginners but misses nuance. +- **Some custom fields don't log history** — `CHANGED` and `WAS` silently return nothing on fields without history tracking. + +### ScriptRunner (if installed) + +- **Powerful but heavier** — `issueFunction` adds latency versus native JQL. +- **Common uses:** `issueFunction in hasLinkType("Epic-Story Link")`, `issueFunction in commented("by user after -1d")`. + +## Troubleshooting Flow + +**Query returns 0 results unexpectedly** + +1. Check field-name spelling (custom fields especially) +2. Verify the project/sprint/version actually exists +3. Check value case sensitivity — it depends on your Jira configuration +4. Run `ORDER BY created DESC` alone to confirm the query executes and genuinely has no matches +5. Remember the empty-value trap: negations exclude empty fields + +**Query is valid but slow** + +1. Remove leading wildcards from text searches +2. Add a project filter first +3. Replace `OR` chains with `IN` +4. Remove negations where possible +5. Tighten date/status bounds to shrink the candidate set + +**Jira says "Field 'X' does not exist"** + +1. The field may be disabled for this project +2. It may be a custom field owned by an uninstalled marketplace app +3. The querying user may lack permission to that field + +**"Filter not found" when using `filter =`** + +- The user lacks permission to that saved filter (or the filter was deleted). + +**`CHANGED` returns nothing** + +1. The field may not track history +2. Widen the date range — `CHANGED TO "Done" AFTER startOfDay()` may be too narrow +3. Confirm the transition actually happened; some workflows skip statuses + +**`WAS` operator returns no results** + +1. Most system fields have trackable history, but verify this one does +2. `WAS` matches past values — if the field always held its current value there is no history to match + +## Marketplace Extensions for Advanced Needs + +| Plugin | Hosting | What It Adds | +|--------|---------|-------------| +| JQL Tricks Plugin | Server/DC | 50+ extra functions | +| JQL Search Extensions | Cloud | Find comments, attachments, subtasks, epics, links | +| JQL Booster Pack | Server/DC | 15+ user-related functions, archived version filtering | +| JQL Functions Collection | Server/DC | String and date format functions | +| Groups & Organizations JQL | Server/DC | Match multi-group custom field values | +| ScriptRunner (Adaptavist) | Cloud/Server/DC | Custom Groovy JQL functions — most powerful and flexible | + +Attribution: adapted from the retired jira-jql skill, sourced from Atlassian official documentation and community best practices. + +## Sources + +- Advanced searching (JQL): https://support.atlassian.com/jira-software-cloud/docs/use-advanced-search-with-jira-query-language-jql/ +- Search endpoint used to run JQL over REST: https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/ diff --git a/jira/references/jql-cookbook.md b/jira/references/jql-cookbook.md new file mode 100644 index 0000000..257d791 --- /dev/null +++ b/jira/references/jql-cookbook.md @@ -0,0 +1,271 @@ +# JQL Cookbook — Role-Based Ready Queries + +Fifty ready-to-run JQL queries organized by role. Every query is copy-pasteable; substitute your own project keys, group names, and issue keys. Pairs with [jql-functions-catalog.md](jql-functions-catalog.md) for function semantics and [jql-best-practices.md](jql-best-practices.md) for performance rules. + +## Developers (10 queries) + +```jql +-- 1. My plate, sorted by urgency +assignee = currentUser() AND resolution = Unresolved ORDER BY priority DESC +``` + +```jql +-- 2. Bugs I reported that haven't been fixed +reporter = currentUser() AND status != Done +``` + +```jql +-- 3. Where I'm mentioned (standup prep) +comment ~ currentUser() +``` + +```jql +-- 4. My completed work this week +resolution = Fixed AND resolutiondate >= -7d AND assignee = currentUser() +``` + +```jql +-- 5. This week's deadlines +duedate >= startOfWeek() AND duedate <= endOfWeek() +``` + +```jql +-- 6. My blocked tickets +issueLinkType = "is blocked by" AND assignee = currentUser() +``` + +```jql +-- 7. Subtasks of a specific story +parent = "PROJ-123" +``` + +```jql +-- 8. Watched but not closed +watcher = currentUser() AND status != Closed +``` + +```jql +-- 9. Full-text search across summary, description, and comments +text ~ "error message here" +``` + +```jql +-- 10. Recent unplanned work +created >= -3d AND assignee = currentUser() AND resolution = Unresolved +``` + +## Scrum Masters (9 queries) + +```jql +-- 11. Unassigned in active sprint +sprint IN openSprints() AND assignee IS EMPTY +``` + +```jql +-- 12. Zombie tickets (not touched in 30 days) +status NOT IN (Closed, Done) AND updated < -30d +``` + +```jql +-- 13. Recently completed this sprint +status CHANGED TO Done AFTER startOfWeek() +``` + +```jql +-- 14. Reopened tickets (quality regression flag) +status CHANGED FROM Done TO "In Progress" +``` + +```jql +-- 15. Volatility — new issues created into the current sprint +sprint IN openSprints() AND created >= -1w +``` + +```jql +-- 16. Team's in-flight work +assignee in membersOf("Dev Team") AND status = "In Progress" +``` + +```jql +-- 17. Carried-over issues (in current sprint, was in previous) +fixVersion = "Current Sprint" AND fixVersion WAS "Last Sprint" +``` + +```jql +-- 18. Issues blocked by anything +issueLinkType = "is blocked by" +``` + +```jql +-- 19. Sprint capacity check +sprint IN openSprints() AND assignee in membersOf("Dev Team") +``` + +## Product Owners / Managers (10 queries) + +```jql +-- 20. Pre-release readiness +fixVersion = earliestUnreleasedVersion() AND status != Done +``` + +```jql +-- 21. Firefighting view — critical unresolved bugs +priority IN (Critical, Highest) AND resolution = Unresolved +``` + +```jql +-- 22. Due within the calendar month +duedate >= startOfMonth() AND duedate <= endOfMonth() AND resolution = Unresolved +``` + +```jql +-- 23. Pending approvals awaiting action (JSM) +approvals = pending() +``` + +```jql +-- 24. Epics without stories attached (requires ScriptRunner) +issuetype = Epic AND issueFunction not in hasLinkType("Epic-Story Link") +``` + +```jql +-- 25. Component-level tech debt +component = "Backend" AND status != Done ORDER BY priority DESC +``` + +```jql +-- 26. Recently reported bugs for triage review +issuetype = Bug AND created >= -14d ORDER BY created DESC +``` + +```jql +-- 27. Cross-project portfolio view +project in ("Project Mercury", "PTC") AND issuetype in ("Epic", "Task") AND status = "To Do" AND created >= -180d +``` + +```jql +-- 28. Status-category rollup across workflows +statusCategory = "In Progress" OR statusCategory = "To Do" +``` + +```jql +-- 29. Feature completeness for a release +fixVersion = "v2.0" AND status != Done ORDER BY component, priority +``` + +## Power Users (12 queries) + +```jql +-- 30. Cascading select matches a specific path +location in cascadeOption("USA", "New York") +``` + +```jql +-- 31. Issues assigned to any administrator +assignee in membersOf("jira-administrators") +``` + +```jql +-- 32. Issues where I was the previous assignee +assignee WAS currentUser() +``` + +```jql +-- 33. Resolved by me this year (retrospective input) +resolution CHANGED TO "Fixed" BY currentUser() DURING (startOfYear(), endOfYear()) +``` + +```jql +-- 34. Issues updated by a specific user in the last week +issue in updatedBy(jsmith, "-8d") +``` + +```jql +-- 35. Issues linked through a specific link type +issue in linkedIssues("PROJ-123", "is duplicated by") +``` + +```jql +-- 36. Issues that were in a specific sprint historically +sprint WAS "Sprint 5" +``` + +```jql +-- 37. Children of an epic +parentEpic = "PROJ-EPIC-1" +``` + +```jql +-- 38. Everything that changed status in the last 24 hours +status CHANGED AFTER -1d +``` + +```jql +-- 39. All subtask issue types +issuetype in subtaskWorkTypes() +``` + +```jql +-- 40. Issues that breached SLA (JSM) +SLA = breached() +``` + +```jql +-- 41. Approval requests assigned to me as approver (JSM) +approvals = myPendingApproval() +``` + +## Automation / Admin Queries (9 queries) + +```jql +-- 42. Reports from users outside an internal group (access reviews) +reporter NOT IN membersOf("internal-users") +``` + +```jql +-- 43. Projects where the user holds no Developers role (permission audit) +project NOT IN spacesWhereUserHasRole("Developers") +``` + +```jql +-- 44. Old unassigned tickets (automation: assign or close) +assignee IS EMPTY AND created < -90d AND resolution = Unresolved +``` + +```jql +-- 45. Bulk transition candidates (stalled in progress) +status = "In Progress" AND updated < -14d +``` + +```jql +-- 46. Stale sprints still holding unresolved work (automation: warn or move) +sprint IN closedSprints() AND resolution = Unresolved +``` + +```jql +-- 47. Approvals decided by a specific user (audit trail, JSM) +approvals = approver(jsmith) +``` + +```jql +-- 48. SLA clocks at risk of breach soon (JSM monitoring) +SLA != completed() AND SLA <= remaining("-4h") +``` + +```jql +-- 49. Requests from one customer organization (JSM triage split) +reporter in organizationMembers("Atlassian") AND resolution = Unresolved +``` + +```jql +-- 50. Backlog hygiene: never-scheduled and long untouched +(sprint IS EMPTY OR sprint NOT IN openSprints()) AND updated < -60d AND resolution = Unresolved +``` + +Attribution: adapted from the retired jira-jql skill, sourced from Atlassian official documentation and community best practices. + +## Sources + +- Advanced searching (JQL): https://support.atlassian.com/jira-software-cloud/docs/use-advanced-search-with-jira-query-language-jql/ +- JQL functions reference: https://support.atlassian.com/jira-software-cloud/docs/jql-functions/ +- Search endpoint used to run JQL over REST: https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/ diff --git a/jira-jql/references/jql-functions-catalog.md b/jira/references/jql-functions-catalog.md similarity index 51% rename from jira-jql/references/jql-functions-catalog.md rename to jira/references/jql-functions-catalog.md index dd3e5cb..e1aba22 100644 --- a/jira-jql/references/jql-functions-catalog.md +++ b/jira/references/jql-functions-catalog.md @@ -1,13 +1,15 @@ -# JQL Functions — Complete Official Catalog +# JQL Functions — Complete Catalog -Full reference: https://support.atlassian.com/jira-software-cloud/docs/jql-functions/ +Every JQL function with its supported fields, operators, and worked syntax. Pairs with [jql-best-practices.md](jql-best-practices.md) for performance rules and [jql-cookbook.md](jql-cookbook.md) for ready-to-run queries. -## Date/Time Functions +Increment strings follow the `(+/-)nn(y|M|w|d|h|m)` pattern everywhere a date function accepts an offset; if the unit qualifier is omitted it defaults to the function's natural period. -All accept optional increment: `(+/-)nn(y|M|w|d|h|m)`. If the unit qualifier is omitted, it defaults to the natural period. +## Date and Time Functions + +All accept an optional increment string `(+/-)nn(y|M|w|d|h|m)`. If the unit qualifier is omitted, it defaults to the natural period shown below. **Supported fields:** Created, Due, Resolved, Updated, custom Date/Time fields -**Supported operators:** `=, !=, >, >=, <, <=, WAS*, WAS IN*, WAS NOT*, WAS NOT IN*, CHANGED*` (* predicate only) +**Supported operators:** `=, !=, >, >=, <, <=, WAS*, WAS IN*, WAS NOT*, WAS NOT IN*, CHANGED*` (* predicate position only) **Unsupported operators:** `~, !~, IS, IS NOT, IN, NOT IN` | Function | Syntax | Notes | @@ -21,153 +23,220 @@ All accept optional increment: `(+/-)nn(y|M|w|d|h|m)`. If the unit qualifier is | `startOfYear()` | `created > startOfYear()` | January 1st | | `endOfYear()` | `due < endOfYear()` | December 31st | | `now()` | `updated < now()` | Current exact time | -| `currentLogin()` | `updated > currentLogin()` | When session started | +| `currentLogin()` | `updated > currentLogin()` | When the session started | | `lastLogin()` | `created > lastLogin()` | Previous login timestamp | +Relative offsets work directly on date fields without a function: `created >= -7d`, `updated < -30d`. + ## User Functions ### `currentUser()` -Your identity. Only works for logged-in users (not anonymous). + +Your identity. Only works for logged-in users (not anonymous access). + - **Fields:** Assignee, Reporter, Voter, Watcher, Creator, custom User fields - **Operators:** `=`, `!=` -- **Unsupported:** everything else ### `membersOf(group)` + Members of a group or team. -- **Syntax:** `membersOf("group-name")` or `membersOf(id:)` for teams + +- **Syntax:** `membersOf("group-name")`, or `membersOf(id:)` for teams - **Fields:** Assignee, Reporter, Voter, Watcher, Creator, custom User fields - **Operators:** `IN, NOT IN, WAS IN, WAS NOT IN` -- **Does NOT support project roles** +- Does **NOT** support project roles — groups and teams only ### `componentsLeadByUser(user)` -Components led by a user. Omit user → current user. + +Components led by a user. Omit the user argument to mean the current user. + - **Fields:** Component - **Operators:** `IN, NOT IN` ### `spacesLeadByUser(user)` -Projects led by a user. Omit user → current user. + +Projects led by a user. Omit the user argument to mean the current user. + - **Fields:** Project (Space) - **Operators:** `IN, NOT IN` ### `spacesWhereUserHasPermission(permission)` -Projects where you have a specific permission. + +Projects where you hold a specific permission, e.g. `"Edit work items"`. + - **Fields:** Project - **Operators:** `IN, NOT IN` - Only available for logged-in users ### `spacesWhereUserHasRole(rolename)` -Projects where you have a specific role. + +Projects where you have a specific role, e.g. `"Administrators"`. + - **Fields:** Project - **Operators:** `IN, NOT IN` -## Sprint/Version Functions +## Sprint and Version Functions ### `openSprints()` + Active sprints that have started but not yet completed. + - **Fields:** Sprint - **Operators:** `IN, NOT IN` -- Issues can belong to both open AND closed sprints simultaneously +- Issues can belong to open AND closed sprints simultaneously ### `closedSprints()` + Completed sprints. + - **Fields:** Sprint - **Operators:** `IN, NOT IN` ### `earliestUnreleasedVersion(project)` + Earliest unreleased version, ordered by the Releases page order (bottom = earliest). -- **Fields:** AffectedVersion, FixVersion, custom Version + +- **Fields:** AffectedVersion, FixVersion, custom Version fields - **Operators:** `=, !=` ### `latestReleasedVersion(project)` + Most recently released version. -- **Fields:** Same + +- **Fields:** AffectedVersion, FixVersion, custom Version fields - **Operators:** `=, !=` ### `releasedVersions(project)` -All released versions. Omit project for all projects. -- **Fields:** AffectedVersion, FixVersion, custom Version + +All released versions. Omit the project argument to search across all projects. + +- **Fields:** AffectedVersion, FixVersion, custom Version fields - **Operators:** `IN, NOT IN` ### `unreleasedVersions(project)` -All unreleased versions. Omit project for all projects. -- **Fields:** AffectedVersion, FixVersion, custom Version + +All unreleased versions. Omit the project argument to search across all projects. + +- **Fields:** AffectedVersion, FixVersion, custom Version fields - **Operators:** `IN, NOT IN` ## Issue Functions ### `linkedIssues(key, linkType?)` -Issues linked to a specific issue. -- **Syntax:** `issue in linkedIssues("ABC-44")` or `issue in linkedIssues("ABC-44", "is blocked by")` + +Issues linked to a specific issue. The link type argument is optional. + +```jql +issue in linkedIssues("ABC-44") +issue in linkedIssues("ABC-44", "is blocked by") +``` + - **Fields:** Issue - **Operators:** `IN, NOT IN` ### `issueHistory()` / `votedWorkItems()` / `watchedWorkItems()` -Recently viewed / voted on / watched issues. + +Recently viewed, voted-on, and watched issues respectively. + - **Operators:** `IN, NOT IN` -- `votedWorkItems()` and `watchedWorkItems()` return up to 32,000 IDs +- `votedWorkItems()` and `watchedWorkItems()` return up to 32,000 issue IDs ### `updatedBy(user, dateFrom?, dateTo?)` -Issues updated by a specific user (includes creating, updating fields, creating/deleting comments, editing comments). -- **Syntax:** `issue in updatedBy(jsmith, "-8d")` or `issue in updatedBy(jsmith, "2024/01/01", "2024/06/01")` -- **Minimum granularity:** 1 day (smaller values rounded up) -### `parentEpic` (field, not function) -Find stories/subtasks in a specific epic. -- **Syntax:** `parentEpic = DEMO-123` or `parentEpic in (DEMO-1, SAMPLE-4)` +Issues updated by a specific user — includes creating the issue, updating fields, creating/deleting comments, and editing comments. + +```jql +issue in updatedBy(jsmith, "-8d") +issue in updatedBy(jsmith, "2024/01/01", "2024/06/01") +``` + +- **Operators:** `IN, NOT IN` (used with the `issue` field) +- Minimum granularity is 1 day; smaller values such as `-1h` round up to `-1d` + +### `parentEpic` (field, not a function) + +Find stories/subtasks belonging to a specific epic. + +```jql +parentEpic = DEMO-123 +parentEpic in (DEMO-1, SAMPLE-4) +``` + - **Fields:** Issue - **Operators:** `=, !=, IN, NOT IN` -- Only for company-managed projects +- Company-managed projects only ## Custom Field Functions ### `cascadeOption(parentOption, childOption?)` + Cascading Select custom fields. -- **Syntax:** `location in cascadeOption("USA", "New York")` -- Use `none` keyword for empty: `location in cascadeOption("USA", none)` + +```jql +location in cascadeOption("USA", "New York") +``` + +Use the `none` keyword to match an empty tier: `location in cascadeOption("USA", none)`. + - **Operators:** `IN, NOT IN` ### `choiceOption(valueOption...)` + Multiple Choice or Dropdown custom fields. + - **Operators:** `IN, NOT IN` ### `standardWorkTypes()` / `subtaskWorkTypes()` -Filter by standard vs subtask issue types. + +Filter by standard versus subtask issue types. + - **Fields:** Type - **Operators:** `IN, NOT IN` ## Jira Service Management Functions +These require Jira Service Management and operate on the Approval and SLA custom fields. + ### Approval Functions + | Function | Syntax | Effect | |----------|--------|--------| | `approved()` | `approvals = approved()` | All approved requests | -| `pending()` | `approvals = pending()` | Has pending approval step | +| `pending()` | `approvals = pending()` | Has a pending approval step | | `approver(user)` | `approvals = approver(jsmith)` | Specific user is an approver (pending or completed) | -| `pendingApprovalBy(user)` | `approvals = pendingApprovalBy(jsmith)` | User has pending approval | -| `pendingBy(user)` | `approvals = pendingBy(jsmith)` | User is approver, may/may not have decided | -| `myApproval()` | `approvals = myApproval()` | Current user is approver | -| `myPendingApproval()` | `approvals = myPendingApproval()` | Current user has pending approval | -| `myPending()` | `approvals = myPending()` | Current user is approver for pending step | +| `pendingApprovalBy(user)` | `approvals = pendingApprovalBy(jsmith)` | User has a pending approval | +| `pendingBy(user)` | `approvals = pendingBy(jsmith)` | User is an approver, may or may not have decided | +| `myApproval()` | `approvals = myApproval()` | Current user is an approver | +| `myPendingApproval()` | `approvals = myPendingApproval()` | Current user has a pending approval | +| `myPending()` | `approvals = myPending()` | Current user is the approver for a pending step | ### SLA Functions + | Function | Operators | Effect | |----------|-----------|--------| | `breached()` | `=, !=` | SLA missed its goal | | `completed()` | `=, !=` | SLA cycle complete | | `running()` | `=, !=` | SLA clock running | -| `paused()` | `=, !=` | SLA paused (out of calendar hours etc.) | -| `remaining()` | `=, !=, >, <, >=, <=` | Time remaining comparison | +| `paused()` | `=, !=` | SLA paused (out of calendar hours, etc.) | +| `remaining()` | `=, !=, >, <, >=, <=` | Compare remaining time | | `withinCalendarHours()` | `=, !=` | Running within calendar hours | -### Organization Functions +### Organization and Customer Functions + | Function | Fields | Syntax | |----------|--------|--------| | `customerDetail("Field", "Value")` | Reporter, Assignee, Voter, Watcher | `reporter in customerDetail("Region", "APAC")` | | `organizationDetail("Field", "Value")` | Organization | `organization in organizationDetail("Support level", "Platinum")` | | `organizationMembers("OrgName")` | Reporter, Assignee, Voter, Watcher | `reporter in organizationMembers("Atlassian")` | -### `customerDetail()` / `organizationDetail()` -Used with multi-select dropdowns: chain multiple `AND` clauses. -- Returns up to 32,000 records -- Includes deleted/deactivated customers — exclude with `AND reporter NOT IN inactiveUsers()` +`customerDetail()` and `organizationDetail()` pair with multi-select dropdown fields; chain multiple `AND` clauses for combined matches. Both return up to 32,000 records and include deleted/deactivated customers — exclude them with `AND reporter NOT IN inactiveUsers()`. + - **Operators:** `IN, NOT IN` + +Attribution: adapted from the retired jira-jql skill, sourced from Atlassian official documentation and community best practices. + +## Sources + +- JQL functions reference: https://support.atlassian.com/jira-software-cloud/docs/jql-functions/ +- JQL fields reference: https://support.atlassian.com/jira-software-cloud/docs/jql-fields/ +- Search endpoint used to run JQL over REST: https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/ From 5a097d5693a1dbf600031cf93a53641e70c3503e Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 03:16:55 -0400 Subject: [PATCH 06/40] chore(catalog): regenerate after jira-jql absorption Refresh all three generated artifacts so they reflect the jira-jql removal and the renamed skill set: - .claude-plugin/marketplace.json (154 plugins) - .codex-plugin/plugin.json (154 skills) - llms.txt (154 skills) The codex generator also rewrites .agents/plugins/marketplace.json; it remains byte-identical to origin/main (single-entry pointer artifact carrying no skill names). All three check modes exit 0 against the current tree. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/marketplace.json | 35 ++++++++++++--------------------- .codex-plugin/plugin.json | 13 ++++++------ llms.txt | 13 ++++++------ 3 files changed, 25 insertions(+), 36 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index f9242ad..d1ba47a 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -483,10 +483,10 @@ "description": "Guide a person in cultivating creativity in their own work and life: open conversational sessions on creative blocks, habits, environment, motivation, and resilience, or structured development of a concrete project or fledgling idea through a five-phase practice. Do not use for therapy or clinical support, general life coaching, product or stakeholder discovery, or as a study guide for a book." }, { - "name": "ghost-cli", + "name": "ghost", "source": "./", "skills": [ - "./ghost-cli" + "./ghost" ], "strict": false, "description": "Manage Ghost CMS content from the terminal — create and list posts, pages, and tags, and fetch site info via the Ghost Admin API (v5/v6). Use when the user asks about ghost, cms, blog, blogging, posts, pages, tags, publishing, or site configuration." @@ -564,31 +564,22 @@ "description": "Convert operational incident and near-miss evidence into durable product, engineering, test, evaluation, and governance improvements with verified closure. Separate observed facts from causal hypotheses and unresolved uncertainty; map follow-up work across code, tests, skills, operations, product, and governance; track ownership, verification, and closure for every finding. Do not use to assign blame or produce a generic postmortem template; do not close learning because tickets were created — require evidence the intended change occurred." }, { - "name": "jellyfin-cli", + "name": "jellyfin", "source": "./", "skills": [ - "./jellyfin-cli" + "./jellyfin" ], "strict": false, "description": "Query your Jellyfin media server from the terminal — recently added media, search, item details, next-up episodes, library browsing, server info, and stats. Use when the user asks about Jellyfin, media server, movies, TV shows, next episodes, or their media library." }, { - "name": "jira-cli", + "name": "jira", "source": "./", "skills": [ - "./jira-cli" + "./jira" ], "strict": false, - "description": "Interact with Atlassian Jira from the terminal: search issues, view details, create issues, add comments, list projects, and transition status. Use when the user mentions Jira, a ticket key (e.g. PROJ-123), or asks about issues, bugs, tasks, projects, or sprint work." - }, - { - "name": "jira-jql", - "source": "./", - "skills": [ - "./jira-jql" - ], - "strict": false, - "description": "Expert-level skill for Jira Query Language (JQL). Use when the user asks about writing, debugging, optimizing, or understanding JQL queries; needs to filter Jira issues by complex criteria, date ranges, history, or cross-project conditions; wants to build saved filters, dashboard gadgets, or automation rules; or needs guidance on JQL performance, functions, operators, history operators (WAS/CHANGED), relative dates, role-based query patterns, or the JQL REST API." + "description": "Interact with Atlassian Jira from the terminal: search issues with JQL, view details, create issues, add comments, list projects, and transition status. Includes a full JQL language reference (functions, operators, history queries, performance tuning). Use when the user mentions Jira, a ticket key (e.g. PROJ-123), asks about issues, bugs, tasks, projects, or sprint work, or needs to write, debug, or optimize JQL queries. Do not use for GitHub or GitLab issue tracking, Jira site administration, or generic ticketing systems." }, { "name": "kanban-guru", @@ -780,10 +771,10 @@ "description": "Google's Open Knowledge Format (OKF) v0.1 — an open, vendor-neutral spec for representing knowledge as markdown files with YAML frontmatter, designed for AI agent consumption. Use when the user mentions OKF, Open Knowledge Format, Google's knowledge format, LLM wiki bundles, agent knowledge packs, creating OKF bundles, validating OKF documents, or converting knowledge into the OKF standard." }, { - "name": "openlibrary-cli", + "name": "openlibrary", "source": "./", "skills": [ - "./openlibrary-cli" + "./openlibrary" ], "strict": false, "description": "Search books, authors, and works on Open Library from the terminal. Look up books by ISBN, search titles and authors, and fetch detailed work/author records via the public Open Library API. No API key required." @@ -1248,10 +1239,10 @@ "description": "Operate the observability stack that deploys as one unit: Prometheus scrape configuration, recording and alerting rules, relabeling, retention, and high availability; OpenTelemetry Collector pipelines (receivers, processors, exporters, sampling, trace/span correlation); and Loki ingest, LogQL, retention, and label design — with a bundled read-only telemetry-check script for Prometheus rule sanity and scrape-target reachability. Use when running, tuning, or troubleshooting a Prometheus, OpenTelemetry Collector, or Loki deployment, or reviewing the collection/ingest/retention layer. Do not use for observability strategy, SLI/SLO design, or paging policy (that is platform-engineering) or Grafana dashboards, panels, and Grafana-side alerting (that is grafana)." }, { - "name": "tempest-cli", + "name": "tempest", "source": "./", "skills": [ - "./tempest-cli" + "./tempest" ], "strict": false, "description": "Query hyper-local weather from a WeatherFlow Tempest station: current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. Use when the user asks about the weather, temperature, rain, wind, humidity, forecast, or wants conditions from their own station rather than a generic weather service." @@ -1275,10 +1266,10 @@ "description": "Build browser-based Three.js and WebGL scenes, animations, and interactive 3D visualizations with a small vanilla JavaScript starting point." }, { - "name": "tmdb-cli", + "name": "tmdb", "source": "./", "skills": [ - "./tmdb-cli" + "./tmdb" ], "strict": false, "description": "Search and discover movies, TV shows, and trending content via The Movie Database (TMDb) API v3. Use when the user asks about movies, TV, film, cinema, genres, certifications, ratings, cast, upcoming releases, or trending media." diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 0814903..ecd1ca1 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -71,7 +71,7 @@ "./forward-deployed-engineering", "./frontend-engineering", "./genius-life", - "./ghost-cli", + "./ghost", "./github-runner", "./go-to-market", "./grafana", @@ -80,9 +80,8 @@ "./hugo-theme", "./implementation-planning", "./incident-learning", - "./jellyfin-cli", - "./jira-cli", - "./jira-jql", + "./jellyfin", + "./jira", "./kanban-guru", "./kubernetes", "./langchain", @@ -104,7 +103,7 @@ "./notion", "./nous-branding", "./open-knowledge-format", - "./openlibrary-cli", + "./openlibrary", "./opensource-contributions", "./operational-design", "./org-design", @@ -156,10 +155,10 @@ "./technical-documentation", "./technology-radar", "./telemetry", - "./tempest-cli", + "./tempest", "./terraform", "./three", - "./tmdb-cli", + "./tmdb", "./traefik", "./trakt", "./transistor", diff --git a/llms.txt b/llms.txt index 816cc8c..3157664 100644 --- a/llms.txt +++ b/llms.txt @@ -55,7 +55,7 @@ - [forward-deployed-engineering](forward-deployed-engineering/SKILL.md): Guide embedded technical engagements from ambiguous stakeholder need through discovery, framing, hypothesis, build, evaluation, deployment, adoption, measurement, and generalization while preserving evidence, decision rights, and field learning. Use when one accountable technical lead must carry continuity across customer or stakeholder discovery, implementation, production fit, adoption, and measurable outcomes. Do not use for a bounded repository change, product investment governance, ongoing reliability or platform ownership, an isolated specialist task, or advisory work that ends before implementation and adoption. - [frontend-engineering](frontend-engineering/SKILL.md): Build and maintain web frontends — component architecture, state management, API integration, responsive layout, client-side performance, and frontend testing patterns. Framework agnostic, focused on web frontend implementation. Do not use for backend service implementation, data engineering, or platform infrastructure work. - [genius-life](genius-life/SKILL.md): Guide a person in cultivating creativity in their own work and life: open conversational sessions on creative blocks, habits, environment, motivation, and resilience, or structured development of a concrete project or fledgling idea through a five-phase practice. Do not use for therapy or clinical support, general life coaching, product or stakeholder discovery, or as a study guide for a book. -- [ghost-cli](ghost-cli/SKILL.md): Manage Ghost CMS content from the terminal — create and list posts, pages, and tags, and fetch site info via the Ghost Admin API (v5/v6). Use when the user asks about ghost, cms, blog, blogging, posts, pages, tags, publishing, or site configuration. +- [ghost](ghost/SKILL.md): Manage Ghost CMS content from the terminal — create and list posts, pages, and tags, and fetch site info via the Ghost Admin API (v5/v6). Use when the user asks about ghost, cms, blog, blogging, posts, pages, tags, publishing, or site configuration. - [github-runner](github-runner/SKILL.md): Deploy, manage, and troubleshoot self-hosted GitHub Actions runners. Covers systemd service, Docker containers, Kubernetes (Actions Runner Controller), and the Scale Set Client. Use when setting up a CI runner, debugging registration failures, designing autoscaling, or hardening runner security. - [go-to-market](go-to-market/SKILL.md): Plan and execute go-to-market strategy — positioning and messaging frameworks (April Dunford's positioning, message hierarchy), customer acquisition strategy (paid, organic, PLG, SLG), brand architecture (brand house vs house of brands), growth modeling (CAC/LTV by channel, cohort analysis), market entry strategy (beachhead, land-and-expand), and competitive response (pricing wars, feature races, brand defense). Do not use for sales execution and pipeline management, product strategy, or visual brand identity design. - [grafana](grafana/SKILL.md): Operate, configure, provision, secure, and troubleshoot Grafana OSS, Enterprise, and Cloud, including dashboards, folders, data sources, annotations, alert rules, contact points, notification policies, silences, mute timings, service accounts, RBAC, plugins, APIs, and as-code workflows. Use for Grafana product work and Grafana-side integrations. Do not use for defining SLOs or paging policy, operating Prometheus/Loki/Tempo/InfluxDB backends, generic Docker/Kubernetes/Terraform/reverse-proxy work, plugin development, or authorized security assessments; use the corresponding specialist skill. @@ -64,9 +64,8 @@ - [hugo-theme](hugo-theme/SKILL.md): Build, customize, and debug advanced Hugo CMS themes — template architecture, asset pipeline (CSS/JS/image processing), shortcodes and render hooks, page bundles, cover images, Hugo Modules, performance, SEO, and CI/CD. Use when working on a Hugo theme or site template layer. - [implementation-planning](implementation-planning/SKILL.md): Plan the implementation of an approved requirement or specification: produce an executable, dependency-aware delivery plan covering work breakdown, dependency mapping, critical path, ownership, parallelism and sequencing, rollout strategy, rollback and recovery paths, and verification against the original requirement. Supports cross-team, cross-repository, migration, and staged-rollout scenarios. Do not use for pre-approval discovery or needs-finding, authoring a specification from scratch, coding or implementation, the neckbeard issue-to-PR delivery flow itself, or any work whose prerequisite decision has not been approved — planning unapproved work is an explicit stop condition. - [incident-learning](incident-learning/SKILL.md): Convert operational incident and near-miss evidence into durable product, engineering, test, evaluation, and governance improvements with verified closure. Separate observed facts from causal hypotheses and unresolved uncertainty; map follow-up work across code, tests, skills, operations, product, and governance; track ownership, verification, and closure for every finding. Do not use to assign blame or produce a generic postmortem template; do not close learning because tickets were created — require evidence the intended change occurred. -- [jellyfin-cli](jellyfin-cli/SKILL.md): Query your Jellyfin media server from the terminal — recently added media, search, item details, next-up episodes, library browsing, server info, and stats. Use when the user asks about Jellyfin, media server, movies, TV shows, next episodes, or their media library. -- [jira-cli](jira-cli/SKILL.md): Interact with Atlassian Jira from the terminal: search issues, view details, create issues, add comments, list projects, and transition status. Use when the user mentions Jira, a ticket key (e.g. PROJ-123), or asks about issues, bugs, tasks, projects, or sprint work. -- [jira-jql](jira-jql/SKILL.md): Expert-level skill for Jira Query Language (JQL). Use when the user asks about writing, debugging, optimizing, or understanding JQL queries; needs to filter Jira issues by complex criteria, date ranges, history, or cross-project conditions; wants to build saved filters, dashboard gadgets, or automation rules; or needs guidance on JQL performance, functions, operators, history operators (WAS/CHANGED), relative dates, role-based query patterns, or the JQL REST API. +- [jellyfin](jellyfin/SKILL.md): Query your Jellyfin media server from the terminal — recently added media, search, item details, next-up episodes, library browsing, server info, and stats. Use when the user asks about Jellyfin, media server, movies, TV shows, next episodes, or their media library. +- [jira](jira/SKILL.md): Interact with Atlassian Jira from the terminal: search issues with JQL, view details, create issues, add comments, list projects, and transition status. Includes a full JQL language reference (functions, operators, history queries, performance tuning). Use when the user mentions Jira, a ticket key (e.g. PROJ-123), asks about issues, bugs, tasks, projects, or sprint work, or needs to write, debug, or optimize JQL queries. Do not use for GitHub or GitLab issue tracking, Jira site administration, or generic ticketing systems. - [kanban-guru](kanban-guru/SKILL.md): A virtual Kanban expert who can diagnose flow problems, design board configurations, set up multi-portfolio operating models, calibrate WIP limits, establish service level expectations, and guide Scrum-to-Kanban transitions. Load this when your team is struggling with throughput, cycle times are unpredictable, multiple stakeholders compete for the same engineers, or you're wondering if Kanban is right for you. - [kubernetes](kubernetes/SKILL.md): Operate, troubleshoot, secure, upgrade, and automate Kubernetes clusters and workloads safely across upstream Kubernetes, k3s, RKE2, MicroK8s, k0s, Talos, OpenShift/OKD, kind, Minikube, Rancher-managed clusters, EKS, AKS, and GKE. Use when a task involves kubectl, Kubernetes APIs, Pods, Deployments, StatefulSets, Services, Ingress or Gateway API, CRDs, RBAC, NetworkPolicy, storage, scheduling, autoscaling, cluster lifecycle, or the bundled agent-first k8s-cli. - [langchain](langchain/SKILL.md): Expert skill for building LLM applications with LangChain — LCEL chains, RAG pipelines, agent orchestration, LangGraph integration, LangSmith observability, and production deployment via LangServe. Use when working with LangChain or comparing LLM application frameworks. @@ -88,7 +87,7 @@ - [notion](notion/SKILL.md): Operate Notion from a terminal or agent: retrieve pages, query databases, search pages and databases, and update page properties — with a bundled notion-cli script that is read-only by default and gates every create or update behind a --dry-run/--yes confirmation. Use when an agent needs to read Notion content, answer questions from a team wiki or database, or make a confirmed edit. Do not use for building Notion integrations or block-level page composition beyond property updates (that is Notion API application development), or for other knowledge bases (that is their own tooling). - [nous-branding](nous-branding/SKILL.md): Generate images and content consistent with the Nous Research brand identity. Use when creating visuals in the Nous / Theia / Hermes ecosystem: a "cyber-classical" style blending neo-classical statuary, cyberpunk/industrial grunge, and retro anime illustration. Covers official brand color palette, typography (Inter/IBM Plex Sans, JetBrains Mono, heavy distressed display faces), the Nous Girl mascot, texture system, and image prompt construction. Ships reference images for palette, mascot, and brand collage that can be used as img2img inputs. - [open-knowledge-format](open-knowledge-format/SKILL.md): Google's Open Knowledge Format (OKF) v0.1 — an open, vendor-neutral spec for representing knowledge as markdown files with YAML frontmatter, designed for AI agent consumption. Use when the user mentions OKF, Open Knowledge Format, Google's knowledge format, LLM wiki bundles, agent knowledge packs, creating OKF bundles, validating OKF documents, or converting knowledge into the OKF standard. -- [openlibrary-cli](openlibrary-cli/SKILL.md): Search books, authors, and works on Open Library from the terminal. Look up books by ISBN, search titles and authors, and fetch detailed work/author records via the public Open Library API. No API key required. +- [openlibrary](openlibrary/SKILL.md): Search books, authors, and works on Open Library from the terminal. Look up books by ISBN, search titles and authors, and fetch detailed work/author records via the public Open Library API. No API key required. - [opensource-contributions](opensource-contributions/SKILL.md): Make good open source contributions — check CONTRIBUTING.md first, follow project norms, be a good citizen. Covers bug reports, feature requests, and pull requests with a defensible default posture when the project hasn't documented expectations. - [operational-design](operational-design/SKILL.md): Design and improve operational processes and organizational scaling — process design, operational metrics, compliance and audit, vendor management, and team topology. Covers value stream mapping, BPMN, bottleneck analysis, scaling from 10 to 100 to 1000 people, KPI design, balanced scorecard, SOC 2, ISO 27001, GDPR readiness, RFP processes, SLA design, vendor scorecards, team topologies, Conway's Law, and Dunbar's Number. Do not use for engineering delivery, financial modeling, or technology evaluation. - [org-design](org-design/SKILL.md): CHRO methodology — organizational design (team topologies, span of control, reporting structures), talent strategy (make-vs-buy, skill taxonomies, succession planning), compensation frameworks (market benchmarking, equity design, leveling), culture architecture (values codification, rituals, psychological safety), organizational health metrics (eNPS, retention risk, engagement surveys), DEI strategy (inclusive design, equitable systems, belonging). @@ -140,10 +139,10 @@ - [technical-documentation](technical-documentation/SKILL.md): Create and review technical documentation, including READMEs, agent-facing instructions, API references, and CLI help. Use when documentation must help someone complete real work. Do not use for marketing copy, brand messaging, or long-form editorial content. - [technology-radar](technology-radar/SKILL.md): Build and maintain technology radars for adoption, trial, assessment, and hold decisions, and choose proportionate architecture-governance paths for technology portfolios. Use when governing technology choices, build-versus-buy decisions, architecture standards, exceptions, or engineering portfolio risk. Do not use for enterprise capability or target-state architecture, writing ADRs, implementing systems, security engineering, or operational incident/runbook work. - [telemetry](telemetry/SKILL.md): Operate the observability stack that deploys as one unit: Prometheus scrape configuration, recording and alerting rules, relabeling, retention, and high availability; OpenTelemetry Collector pipelines (receivers, processors, exporters, sampling, trace/span correlation); and Loki ingest, LogQL, retention, and label design — with a bundled read-only telemetry-check script for Prometheus rule sanity and scrape-target reachability. Use when running, tuning, or troubleshooting a Prometheus, OpenTelemetry Collector, or Loki deployment, or reviewing the collection/ingest/retention layer. Do not use for observability strategy, SLI/SLO design, or paging policy (that is platform-engineering) or Grafana dashboards, panels, and Grafana-side alerting (that is grafana). -- [tempest-cli](tempest-cli/SKILL.md): Query hyper-local weather from a WeatherFlow Tempest station: current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. Use when the user asks about the weather, temperature, rain, wind, humidity, forecast, or wants conditions from their own station rather than a generic weather service. +- [tempest](tempest/SKILL.md): Query hyper-local weather from a WeatherFlow Tempest station: current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. Use when the user asks about the weather, temperature, rain, wind, humidity, forecast, or wants conditions from their own station rather than a generic weather service. - [terraform](terraform/SKILL.md): Operate Terraform and OpenTofu across the whole infrastructure lifecycle: module structure, state backends and locking, plan/apply workflow, drift detection, remote state, upgrade and refactor flows, and evidence-based diagnostics. Use when running or inspecting terraform plans, applies, state files, imports, or state surgery, or when the bundled tfops script should handle the task. Do not use for IaC methodology or cloud design decisions - those route up to platform-engineering. - [three](three/SKILL.md): Build browser-based Three.js and WebGL scenes, animations, and interactive 3D visualizations with a small vanilla JavaScript starting point. -- [tmdb-cli](tmdb-cli/SKILL.md): Search and discover movies, TV shows, and trending content via The Movie Database (TMDb) API v3. Use when the user asks about movies, TV, film, cinema, genres, certifications, ratings, cast, upcoming releases, or trending media. +- [tmdb](tmdb/SKILL.md): Search and discover movies, TV shows, and trending content via The Movie Database (TMDb) API v3. Use when the user asks about movies, TV, film, cinema, genres, certifications, ratings, cast, upcoming releases, or trending media. - [traefik](traefik/SKILL.md): Deploy, configure, and troubleshoot Traefik v3 reverse proxy — covers all providers, routing, TLS/ACME, middlewares, and production patterns with YAML examples. Load when setting up or debugging a Traefik instance. - [trakt](trakt/SKILL.md): Discover trending, anticipated, and popular movies and TV shows via the Trakt.tv API from the terminal. No authentication required for read-only discovery. Use when the user asks about what to watch, trending movies, popular shows, or media discovery. - [transistor](transistor/SKILL.md): Manage Transistor.fm podcast hosting from the terminal: view shows, list episodes, check analytics, and get subscriber counts. Use when the user mentions Transistor, podcast hosting, podcast analytics, show management, or episode tracking. From 3ff46ee934e011a0e39101a0a5ef8d92aa5a89ff Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 06:09:06 -0400 Subject: [PATCH 07/40] docs(jira): re-expand JQL depth sections and neutralize org example Add jql-history-and-dates.md covering WAS/CHANGED predicate walkthrough, relative-date expression tables, and saved-filter composition/naming conventions, per foundation scrutiny depth directives. Swap cookbook query 49's organizationMembers("Atlassian") for the YOUR_ORG placeholder and fix the matching catalog example. All claims cited to live-verified Atlassian docs. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- jira/references/jql-cookbook.md | 2 +- jira/references/jql-functions-catalog.md | 2 +- jira/references/jql-history-and-dates.md | 176 +++++++++++++++++++++++ 3 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 jira/references/jql-history-and-dates.md diff --git a/jira/references/jql-cookbook.md b/jira/references/jql-cookbook.md index 257d791..c496b3c 100644 --- a/jira/references/jql-cookbook.md +++ b/jira/references/jql-cookbook.md @@ -254,7 +254,7 @@ SLA != completed() AND SLA <= remaining("-4h") ```jql -- 49. Requests from one customer organization (JSM triage split) -reporter in organizationMembers("Atlassian") AND resolution = Unresolved +reporter in organizationMembers("YOUR_ORG") AND resolution = Unresolved ``` ```jql diff --git a/jira/references/jql-functions-catalog.md b/jira/references/jql-functions-catalog.md index e1aba22..944a1fb 100644 --- a/jira/references/jql-functions-catalog.md +++ b/jira/references/jql-functions-catalog.md @@ -227,7 +227,7 @@ These require Jira Service Management and operate on the Approval and SLA custom |----------|--------|--------| | `customerDetail("Field", "Value")` | Reporter, Assignee, Voter, Watcher | `reporter in customerDetail("Region", "APAC")` | | `organizationDetail("Field", "Value")` | Organization | `organization in organizationDetail("Support level", "Platinum")` | -| `organizationMembers("OrgName")` | Reporter, Assignee, Voter, Watcher | `reporter in organizationMembers("Atlassian")` | +| `organizationMembers("OrgName")` | Reporter, Assignee, Voter, Watcher | `reporter in organizationMembers("YOUR_ORG")` | `customerDetail()` and `organizationDetail()` pair with multi-select dropdown fields; chain multiple `AND` clauses for combined matches. Both return up to 32,000 records and include deleted/deactivated customers — exclude them with `AND reporter NOT IN inactiveUsers()`. diff --git a/jira/references/jql-history-and-dates.md b/jira/references/jql-history-and-dates.md new file mode 100644 index 0000000..d33f595 --- /dev/null +++ b/jira/references/jql-history-and-dates.md @@ -0,0 +1,176 @@ +# JQL History Predicates, Date Expressions & Saved Filters + +Deep-dive on the three areas that trip up even experienced JQL writers: history operators (`WAS`, `CHANGED` and their predicates), relative-date expressions, and saved-filter composition. Pairs with [jql-functions-catalog.md](jql-functions-catalog.md), [jql-best-practices.md](jql-best-practices.md), and [jql-cookbook.md](jql-cookbook.md). + +A JQL clause is a field followed by an operator followed by one or more values or functions (`project = "TEST"`); clauses join with keywords like `AND`/`OR`. Without parentheses a statement evaluates left-to-right, which is why parenthesizing OR groups matters. + +## History Operators: the WAS Family + +**Field restriction first:** `WAS`, `WAS IN`, `WAS NOT`, and `WAS NOT IN` work with **Assignee, Fix Version, Priority, Reporter, Resolution, and Status only**. On any other field they error out; custom-field history is simply not addressable through these operators. + +### What WAS actually matches + +`status WAS "In Progress"` finds issues that currently have OR previously had that value. Two subtle matching rules: + +1. It matches the value name **as it was configured at the time of the change** — if your workflow renamed "In Progress" to "Active" last quarter, `status WAS "In Progress"` still finds historical states recorded under the old name. +2. It also matches the value's numeric ID — `status WAS "Resolved"` and `status WAS "4"` hit the same issues when 4 was Resolved's ID. + +### Optional predicates + +Every WAS-family operator accepts optional predicates: + +| Predicate | Form | Meaning | +|-----------|------|---------| +| `AFTER` | `AFTER "date"` | change happened after the date | +| `BEFORE` | `BEFORE "date"` | change happened before the date | +| `BY` | `BY "user"` / `BY (user1,user2)` | user who made the change | +| `DURING` | `DURING ("date1","date2")` | change inside the window | +| `ON` | `ON "date"` | change on that exact date | + +The `BY` user may be a username or an Atlassian account ID (`status WAS "Resolved" BY abcde-12345-fedcba BEFORE "2019/02/02"`). Dates use the standard JQL date format (`"2019/02/02"`) or any expression from the relative-date section below — `DURING (startOfYear(), endOfYear())` is valid. + +### Walkthrough: build a "reopened bugs" query step by step + +Goal: bugs that went backwards from Done back to In Progress. + +```jql +-- Step 1: base form — did status ever hold "Done"? +issuetype = Bug AND status WAS Done + +-- Step 2: add the transition direction with CHANGED (below): +issuetype = Bug AND status CHANGED FROM Done TO "In Progress" + +-- Step 3: bound it to this year so the scan stays cheap: +issuetype = Bug AND status CHANGED FROM Done TO "In Progress" DURING (startOfYear(), endOfYear()) +``` + +Each predicate composes: `priority CHANGED BY freddo BEFORE endOfWeek() AFTER startOfWeek()` chains two time bounds around a user bound. + +### The other WAS operators + +| Operator | Equivalent longhand | Example | +|----------|--------------------|---------| +| `WAS IN ("Resolved","Closed")` | `status WAS "Resolved" OR status WAS "Closed"` | `status WAS IN ("Resolved","In Progress")` | +| `WAS NOT "X"` | never held X | `status WAS NOT "In Progress" BEFORE "2011/02/02"` | +| `WAS NOT IN (...)` | `WAS NOT A AND WAS NOT B` | `status WAS NOT IN ("Resolved","In Progress")` | + +### The 10,000-change truncation + +If an issue has more than 10,000 changes, WAS-family queries search **only its most recent changes**. Ancient history on hyper-active issues is invisible to JQL — use the issue view or export for those. This is silent: you get results, just not complete ones. + +## The CHANGED Operator + +`CHANGED` finds issues whose field value *changed* (not what it changed to — that is what `FROM`/`TO` refine). + +Predicates: everything WAS takes, **plus** `FROM "oldvalue"` and `TO "newvalue"`: + +| Predicate | Purpose | +|-----------|---------| +| `FROM "oldvalue"` | previous value equals | +| `TO "newvalue"` | new value equals | +| `AFTER` / `BEFORE` / `DURING` / `ON` | time bounds | +| `BY "user"` | who performed the change | + +Same six-field restriction applies (Assignee, Fix Version, Priority, Reporter, Resolution, Status). + +Canonical patterns: + +```jql +-- Any assignee change at all: +assignee CHANGED + +-- Regression detector: went backwards from In Progress to Open: +status CHANGED FROM "In Progress" TO "Open" + +-- Priority churn by one user this week: +priority CHANGED BY freddo AFTER startOfWeek() BEFORE endOfWeek() + +-- Resolved-by-me-this-year (cookbook #33): +resolution CHANGED TO "Fixed" BY currentUser() DURING (startOfYear(), endOfYear()) +``` + +**Prerequisites and failure mode:** `CHANGED` and the WAS family return nothing for fields without history tracking — most system fields track, some custom fields do not. If a `CHANGED` query returns zero rows, confirm transitions actually occurred and widen the date window before assuming the data is missing. Note the docs' own quirk: the >10,000-changes truncation paragraph under CHANGED still says "the WAS operator" — same limit, shared implementation. + +## Relative Dates and Expressions + +### Direct offsets on date fields + +Date fields accept an increment string directly: `(+/-)nn(y|M|w|d|h)` — years, months (capital M!), weeks, days, hours. No function call needed: + +```jql +created >= -7d /* last seven days */ +updated < -30d /* untouched for a month */ +duedate <= 2w /* due within two weeks */ +``` + +Case matters: `-1m` is minutes, `-1M` is months. If you drop the unit entirely the default depends on context (days for the bare-number legacy form). + +### Function forms + +| Expression | Evaluates to | Typical use | +|------------|--------------|-------------| +| `startOfDay()` | today 00:00 local | `created > startOfDay()` | +| `endOfDay()` | today 23:59 local | `due < endOfDay("+1")` | +| `startOfWeek()` | week start (Sunday default) | `created >= startOfWeek()` | +| `endOfWeek()` | week end (Saturday default) | `due <= endOfWeek()` | +| `startOfMonth()` / `endOfMonth()` | month boundaries | `resolved >= startOfMonth("-1M")` | +| `startOfYear()` / `endOfYear()` | Jan 1 / Dec 31 | retrospective windows | +| `now()` | exact current timestamp | `updated < now()` | + +Offsets compose inside functions: `startOfWeek("+1d")` shifts to Monday on Sunday-default sites; `endOfMonth("+15d")` lands mid-next-month. Full parameter tables per function live in [jql-functions-catalog.md](jql-functions-catalog.md). + +### Timezone trap + +Jira evaluates dates in the querying user's timezone. A dashboard shared across regions shows different rows for the same `startOfDay()` query near midnight boundaries. For cross-timezone automation prefer explicit dates over day-grain relatives. + +### Keep relatives in saved filters + +Relative expressions re-evaluate at every run — exactly what you want in a saved filter. Freezing an absolute date into a filter meant as "this week" is a classic mistake: the filter silently stops matching next week. + +## Saved Filters: Composition and Naming Conventions + +Saved filters turn long JQL into reusable, shareable building blocks. From the filter lifecycle: save a search, manage/update/copy/delete it, star favorites, subscribe yourself or others to scheduled email delivery, share with colleagues (or outside the organization via links), export results (RSS, Excel), and drive dashboard gadgets. + +### Composing queries with `filter =` + +```jql +filter = "My Team Open Bugs" AND priority in (High, Highest) +filter = 10203 AND updated >= -7d -- numeric filter IDs also work +``` + +Sub-queries compose once and get reused everywhere; fix logic in one place instead of pasting the same clause into twenty dashboards. Performance-wise this does not make Jira faster by itself (Jira expands the filter), but it makes the optimization advice in [jql-best-practices.md](jql-best-practices.md) applyable from a single edit point. + +### Naming conventions that survive contact with reality + +Jira does **not** enforce unique filter names — two people can each own "Open Bugs", and name-based references resolve ambiguously. Conventions that keep dashboards and subscriptions maintainable: + +1. **Prefix by owning team or domain** — `platform-api-stale-prs`, `mobile-crash-triage`. Collisions become visible instead of silent. +2. **Encode scope and cadence** — `weekly-security-review`, `sprint-current-blocked`. Readers of a subscription email should know cadence without opening the filter. +3. **Never rename a filter others reference** — dashboard gadgets and subscriptions bind by filter identity, but humans navigate by name; renames strand both. Copy-and-deprecate instead. +4. **Prefer the numeric ID in scripts** — `filter = 10203` survives renames exactly like project IDs do; reserve name-based references for interactive use. +5. **Keep one canonical "definition" filter per recurring question** — then derive variants (`... AND assignee IS EMPTY`) rather than duplicating the whole query. + +Sharing rules matter before composition works: a gadget or subscription breaks with "Filter not found" for any viewer lacking permission to the underlying filter — grant the audience access to the filter itself, not just the dashboard. + +## Quick Pitfall Reference + +| Symptom | Likely cause | +|---------|--------------| +| `WAS` errors on a custom field | History operators limited to Assignee/Fix Version/Priority/Reporter/Resolution/Status | +| Old history missing on a busy issue | >10,000 changes truncated to recent-only search | +| `CHANGED` returns nothing | No history tracking on the field, or transitions never actually happened | +| Month offset behaved like minutes | `-1m` (minutes) vs `-1M` (months) case sensitivity | +| Same filter shows different rows per region | Day-grain relatives evaluate in each user's timezone | +| Gadget says "Filter not found" | Viewer lacks permission to the referenced saved filter | + +Attribution: adapted in part from the retired jira-jql skill, sourced from Atlassian official documentation. + +## Sources + +- JQL operators (WAS/CHANGED/predicate reference): https://support.atlassian.com/jira-software-cloud/docs/jql-operators/ +- Advanced searching overview (clause structure, precedence, bounded JQL): https://support.atlassian.com/jira-software-cloud/docs/use-advanced-search-with-jira-query-language-jql/ +- What is advanced search (precedence, reserved words, bounded/unbounded): https://support.atlassian.com/jira-software-cloud/docs/what-is-advanced-search-in-jira-cloud/ +- JQL functions (date functions, increment syntax): https://support.atlassian.com/jira-software-cloud/docs/jql-functions/ +- Save your search as a filter: https://support.atlassian.com/jira-software-cloud/docs/save-your-search-as-a-filter/ +- JQL optimization recommendations: https://support.atlassian.com/jira-software-cloud/docs/jql-optimization-recommendations/ +- Search endpoint that runs JQL over REST (startAt/maxResults envelope): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/ From 32474b2848716835c90858a1ebb48a9b9182745b Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 06:09:16 -0400 Subject: [PATCH 08/40] docs(jira): add REST v3 references for auth, search duality, issues Two new reference files distilled from live Atlassian docs research: rest-auth-and-search.md (basic-auth email+token vs OAuth/PAT split, rate-limit headers, error envelopes, legacy /search offset paging vs enhanced /search/jql nextPageToken model with CHANGE-2046 deprecation status) and rest-issues-and-transitions.md (issue CRUD shapes, the GET-transitions-then-POST flow with screen-field requirements and resolution semantics, ADF document model). Both carry Sources footers. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- jira/references/rest-auth-and-search.md | 194 ++++++++++++++++ .../references/rest-issues-and-transitions.md | 214 ++++++++++++++++++ 2 files changed, 408 insertions(+) create mode 100644 jira/references/rest-auth-and-search.md create mode 100644 jira/references/rest-issues-and-transitions.md diff --git a/jira/references/rest-auth-and-search.md b/jira/references/rest-auth-and-search.md new file mode 100644 index 0000000..3c75a00 --- /dev/null +++ b/jira/references/rest-auth-and-search.md @@ -0,0 +1,194 @@ +# Jira Cloud REST API v3 — Auth, Search & Pagination + +Ground truth for authenticating against Jira Cloud and for running searches under both pagination models. Every claim traces to the Atlassian sources in the footer. + +## Authentication + +### Basic auth: email + API token (the default for scripts and CLIs) + +Atlassian's recommended method "for personal scripts, bots, and ad-hoc execution of the REST APIs": + +1. Create an API token at `https://id.atlassian.com/manage/api-tokens` (shown once; cannot be recovered later). +2. Build the string `email:api_token` — email is your **Atlassian account email address**, never a password. +3. Base64-encode it and send it as a proactively-supplied header: + +``` +Authorization: Basic base64(email:token) +``` + +`requests` does this via `auth=(email, token)` tuple — no manual base64 needed. + +Facts worth knowing: + +- API tokens work even when the org has two-factor authentication or SAML enabled. +- Since December 15 2024 new tokens expire after one year by default (configurable 1 day–1 year); tokens created before that were retroactively given expiry from March 13 2025. Expired tokens surface as 401s. +- Password authentication is fully deprecated; there is no password fallback. +- Jira does not send an auth challenge — clients must send the header unprompted. +- Tokens can optionally carry OAuth-style scopes; scoped tokens are used against `https://api.atlassian.com/ex/jira/{cloudId}` instead of the site URL. Scopeless tokens keep working against `https://your-domain.atlassian.net`. + +### The other methods, and when they apply + +| Method | Applies to | Header | Base URL | +|--------|-----------|--------|----------| +| Basic + API token | personal scripts, bots | `Basic base64(email:token)` | site URL | +| OAuth 2.0 (3LO) | integrations acting for users, distributable apps | `Bearer ACCESS_TOKEN` | `https://api.atlassian.com/ex/jira/{cloudId}` | +| Forge / Connect apps | apps on those platforms | built-in JWT/requestJira | varies | + +### PATs: Data Center yes, Cloud no + +Personal Access Tokens (`Authorization: Bearer `) exist only on **Data Center/Server** (Jira 8.14+). Jira Cloud has no PAT feature; its equivalent is the API-token model above. Point anyone asking about "Bearer tokens for Jira" at DC docs or to 3LO for Cloud. + +### CAPTCHA lockout symptom + +After repeated failed logins Jira may trigger CAPTCHA, which blocks REST auth entirely. Symptom: response header `X-Seraph-LoginReason: AUTHENTICATION_DENIED` — login rejected "without even checking the password". Fix by logging in through the browser once, not by retrying the script. + +## Rate Limits + +Three systems, all surfacing as HTTP 429 with these headers: + +``` +Retry-After: +X-RateLimit-Limit: +X-RateLimit-Remaining: 0 +X-RateLimit-Reset: +RateLimit-Reason: jira-quota-global-based +``` + +`RateLimit-Reason` values: `jira-quota-global-based`, `jira-quota-tenant-based` (hourly point quotas), `jira-burst-based` (per-second buckets; defaults ~100 req/s GET/POST), `jira-per-issue-on-write` (20 writes/2s per issue). Honor `Retry-After`; back off with jitter rather than tight retries. + +## Error Envelopes + +Standard error collection everywhere in v3: + +```json +{ "errorMessages": ["..."], "errors": {"field_name": "message"}, "status": 400 } +``` + +| Status | Meaning | +|--------|---------| +| 400 | Malformed request or bad JQL ("Field 'X' does not exist..."); `errors` map names offending fields | +| 401 | Credentials rejected/expired (or CAPTCHA-gated — check `X-Seraph-LoginReason`) | +| 403 | Authenticated but lacking permission | +| 404 | Resource absent or invisible | +| 422 | Validation failure on create/edit payloads | +| 429 | Rate limited — see headers above | + +## Search Endpoints: the Duality + +This is the single most important operational fact about Jira Cloud search in the current era: **two search endpoints with incompatible pagination models coexist**, and the legacy one is being removed. + +### Legacy: `GET|POST /rest/api/3/search` — offset paging + +Status: documented as "**Currently being removed**" and marked deprecated in the OpenAPI spec. Announced 31 October 2024 with removal promised "after May 1, 2025" (CHANGE-2046); sunset has proceeded gradually since. + +Request parameters: `jql`, `startAt` (default 0), `maxResults` (default 50), `validateQuery` (`strict` default | `warn` | `none`), `fields`, `expand`, `properties`, `fieldsByKeys`, `failFast`. + +Response envelope (`SearchResults`): + +```json +{ + "issues": [{"id": "10002", "key": "ED-1", "fields": {}}], + "startAt": 0, + "maxResults": 50, + "total": 1, + "warningMessages": [] +} +``` + +Loop shape: + +```python +start_at = 0 +while True: + page = get("/search", params={"jql": jql, "startAt": start_at, "maxResults": 100}) + yield from page["issues"] + start_at += len(page["issues"]) + if start_at >= page.get("total", 0) or not page["issues"]: + break +``` + +Caveats: `total` can change between pages, so always tolerate an empty page; there is no `isLast` field on this envelope; deep offsets re-scan everything before them. + +### Enhanced: `GET|POST /rest/api/3/search/jql` — token paging + +The replacement, non-deprecated. Request body/params: `jql`, `nextPageToken`, `maxResults` (default 50, ceiling 5,000 — though real-world pages often cap near 100 even when more are requested, so follow the token instead of assuming page sizes), `fields` (**default is `id` only**, unlike every other endpoint), `expand`, `properties`, `fieldsByKeys`, `failFast`, `reconcileIssues`. + +Response envelope (`SearchAndReconcileResults`): + +```json +{ + "isLast": false, + "issues": [{"id": "10002", "key": "ED-1"}], + "nextPageToken": "CAEaAggB", + "warnings": [] +} +``` + +Key differences from legacy: + +| Aspect | Legacy `/search` | Enhanced `/search/jql` | +|--------|------------------|------------------------| +| Offset param | `startAt` | none — opaque `nextPageToken` | +| Total count | `total` present | absent | +| Last-page signal | none (compute from total) | `isLast` boolean; `nextPageToken` omitted on final page | +| Default fields | all navigable | `id` only — pass explicit `fields` | +| Warnings key | `warningMessages` | `warnings` | +| JQL restriction | unbounded allowed | **bounded queries required** — bare `order by key desc` returns 400 | +| `orderBy` cap | none | max 7 fields | +| Consistency | immediate-ish | eventual; optional `reconcileIssues` (≤50 ids) for read-after-write | + +"Bounded" means at least one real condition: `assignee = currentUser() order by key` is bounded; `order by created DESC` alone is not. + +Loop shape: + +```python +body = {"jql": jql, "maxResults": 100, "fields": ["summary", "status"]} +while True: + page = post("/search/jql", json_data=body) + yield from page["issues"] + if page.get("isLast") or "nextPageToken" not in page: + break + body["nextPageToken"] = page["nextPageToken"] +``` + +Token continuation is sequential-only: you cannot fetch pages in parallel, and you must carry the exact previous token forward. + +### Which model fails how — symptoms of mixing them up + +- Passing `startAt` to `/search/jql`: parameter ignored/rejected; you silently loop over page one forever if your loop advances the offset instead of the token. +- Reading `total` off `/search/jql`: `KeyError` — the field does not exist; use `/search/approximate-count` first if you need a count. +- Expecting populated `fields` from `/search/jql` without asking: you get `id`/`key` only. +- Sending an unbounded query to `/search/jql`: immediate `400`. +- Calling legacy `/search` after removal completes: connection-level failure/404-class errors; before that, responses still work but the endpoint is formally dead-ended. + +### Approximate counts + +Need "how many?" without fetching? `POST /rest/api/3/search/approximate-count` with body `{"jql": "project = HSP"}` returns `{"count": 153}`. Works regardless of which search endpoint you use for rows; approximate because it skips permission filtering per row. + +## Endpoint Cheat Sheet + +| Operation | Call | +|-----------|------| +| Current user | `GET /rest/api/3/myself` → `{accountId, displayName, emailAddress?, timeZone}` | +| Search (legacy) | `GET/POST /rest/api/3/search` — deprecated, offset paging | +| Search (current) | `GET/POST /rest/api/3/search/jql` — token paging | +| Count matches | `POST /rest/api/3/search/approximate-count` | +| Get issue | `GET /rest/api/3/issue/{key}?fields=summary,status,...` | +| Create issue | `POST /rest/api/3/issue` — `{fields: {...}}` | +| Edit issue | `PUT /rest/api/3/issue/{key}` — fields at top level, `notifyUsers=false` to silence mail | +| Delete issue | `DELETE /rest/api/3/issue/{key}?deleteSubtasks=true` | +| List transitions | `GET /rest/api/3/issue/{key}/transitions` | +| Apply transition | `POST /rest/api/3/issue/{key}/transitions` — `{"transition": {"id": "..."}}` | +| Add comment | `POST /rest/api/3/issue/{key}/comment` — ADF body | +| Projects | `GET /rest/api/3/project/search` (paginated; plain `/project` is a deprecated bare array) | + +## Sources + +- REST API v3 intro (auth modes, error collection): https://developer.atlassian.com/cloud/jira/platform/rest/v3/intro/ +- Issue search group (both endpoints, approximate-count): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/ +- Basic auth for REST APIs: https://developer.atlassian.com/cloud/jira/platform/basic-auth-for-rest-apis/ +- Deprecation changelog entry CHANGE-2046: https://developer.atlassian.com/changelog/#CHANGE-2046 +- Rate limiting: https://developer.atlassian.com/cloud/jira/platform/rate-limiting/ +- Manage API tokens: https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/ +- OAuth 2.0 (3LO) apps: https://developer.atlassian.com/cloud/jira/platform/oauth-2-3lo-apps/ +- PATs (Data Center/Server only): https://confluence.atlassian.com/enterprise/using-personal-access-tokens-1026032365.html diff --git a/jira/references/rest-issues-and-transitions.md b/jira/references/rest-issues-and-transitions.md new file mode 100644 index 0000000..a427521 --- /dev/null +++ b/jira/references/rest-issues-and-transitions.md @@ -0,0 +1,214 @@ +# Jira Cloud REST API v3 — Issues, Transitions & ADF + +Field-level semantics for the issue lifecycle: reading, creating, editing, commenting, and — the part everyone gets wrong — transitioning. Plus the Atlassian Document Format rules that decide whether your payload is accepted at all. + +## Reading Issues + +`GET /rest/api/3/issue/{issueIdOrKey}` + +- Query params: `fields`, `fieldsByKeys`, `expand`, `properties`, `updateHistory`, `failFast`. +- Default response includes all navigable fields. Trim with `?fields=summary,status,assignee` — faster pages, less noise. +- Non-matching keys get a case-insensitive lookup plus moved-issue check; a found match returns directly (no redirect). +- Response top level: `{id, key, self, fields:{...}}`. + +Field access patterns: + +```json +{ + "fields": { + "summary": "Main order flow broken", + "status": {"name": "In Progress", "statusCategory": {"key": "in-flight"}}, + "issuetype": {"name": "Bug"}, + "priority": {"name": "High"}, + "assignee": {"accountId": "5b10a2844c20165700ede21g", "displayName": "Mia Krystof"}, + "reporter": {"accountId": "...", "displayName": "..."}, + "created": "2019-04-05T10:30:00.000+1000", + "updated": "2024-01-11T08:15:00.000+0000", + "description": {"type": "doc", "version": 1, "content": []} + } +} +``` + +Gotchas: + +- `assignee`/`reporter` may be `null` (unassigned) — null-check before `.displayName`. +- `description` is an ADF object, not text (see ADF section). +- User identity is `accountId` everywhere; `username`/`userKey` were removed in the GDPR migration (April 2019). Email visibility depends on each user's privacy settings. + +## Creating Issues + +`POST /rest/api/3/issue` with body root keys `fields`, `update`, `historyMetadata`, `properties`, `transition`. Only `fields` matters for basic creation. + +```json +{ + "fields": { + "project": { "key": "EX" }, + "summary": "Order entry fails when selecting supplier.", + "issuetype": { "name": "Bug" }, + "description": { + "type": "doc", "version": 1, + "content": [ { "type": "paragraph", "content": [ { "type": "text", "text": "Steps to reproduce..." } ] } ] + }, + "priority": { "name": "High" }, + "labels": ["bugfix"], + "parent": { "key": "PROJ-123" } + } +} +``` + +Rules: + +- Project by `{"key": ...}` or `{"id": ...}`; issuetype by name or id. +- `description`, `environment`, and any `textarea`-type custom fields **require ADF objects**; plain strings are rejected. Single-line `textfield` custom fields take plain strings. +- Users are addressed as `assignee: {"accountId": "..."}` or `{"id": ""}`. +- Success: 201 with `{id, key, self}` (+ optional transition echo). +- Failure: 400/422 with the error collection; `errors` map names the offending field ("Project 'XYZ' does not exist or you do not have permission..."). + +## Editing Issues + +`PUT /rest/api/3/issue/{issueIdOrKey}` + +```json +{ "fields": { "summary": "Completed orders still displaying in pending", + "labels": ["bugfix", "triage"] } } +``` + +- Fields sit under `fields` (or granular ops under `update`, e.g. array field manipulation). +- Success is **204 No Content** — an empty response body means it worked; don't parse for confirmation JSON. +- Transitions are ignored on this endpoint; changing status requires the transitions endpoint below. +- Suppress notification emails with query param `notifyUsers=false` (bulk edits especially). + +Deleting: `DELETE /rest/api/3/issue/{key}` refuses when subtasks exist unless you pass `deleteSubtasks=true`; success is 204. + +## Comments + +`GET /rest/api/3/issue/{key}/comment?startAt=0&maxResults=50` → `{comments: [...], startAt, maxResults, total}` (offset paging, like legacy search). + +Each comment: `{id, author:{accountId, displayName}, body: , created, updated, updateAuthor}`. + +Add one: `POST /rest/api/3/issue/{key}/comment` with `{"body": {ADF doc}}`. The body must be an ADF object — a bare string fails with a 400/500-class error naming the wrong type. + +## Transitions: GET First, Then POST + +This is the highest-friction endpoint pair in Jira integration work. Two calls are always required because **transition IDs differ per workflow, per project, and per current status**, and names alone are ambiguous across workflows. + +### Step 1 — discover available transitions + +`GET /rest/api/3/issue/{key}/transitions?expand=transitions.fields` + +```json +{ + "transitions": [ + { + "id": "31", + "name": "Done", + "hasScreen": true, + "isGlobal": false, + "isConditional": false, + "to": { "name": "Done", "statusCategory": {"key": "completed"} }, + "fields": { + "resolution": { "required": true, "allowedValues": [{"name": "Done"}, {"name": "Fixed"}] }, + "comment": { "required": false } + } + } + ] +} +``` + +Reading this shape: + +- `id` is the string you POST back. Never hardcode it across projects. +- `to.statusCategory.key` (`to-do` / `in-flight` / `completed`) is the stable way to find "the Done-ish transition" without knowing its display name. +- With `expand=transitions.fields`, `fields` lists what the target screen demands and each field's `required` flag plus `allowedValues`. +- Asking for a nonexistent or status-invalid transition yields an **empty list**, not an error. + +### Step 2 — apply the transition + +`POST /rest/api/3/issue/{key}/transitions` + +Minimal payload: + +```json +{ "transition": { "id": "31" } } +``` + +Setting fields during the move (resolution, assignee, comments ride along): + +```json +{ + "transition": { "id": "31" }, + "fields": { "resolution": { "name": "Fixed" } }, + "update": { + "comment": [ { "add": { "body": { "type": "doc", "version": 1, + "content": [ { "type": "paragraph", "content": [ { "type": "text", "text": "Shipped in build 47" } ] } ] } } } ] + } +} +``` + +Semantics that bite: + +- Success is **204 No Content**. +- If the target screen marks a field `required: true` and you omit it — classically `resolution` on a Done transition — you get **400** with `"errors": {"resolution": "..."}` naming the missing field. Pre-read the fields map from step 1 instead of guessing. +- `resolution` values come from `allowedValues`; sending `{"name": "Done"}` where the site expects `"Fixed"` fails validation. +- Transition names repeat across workflows; IDs do not. Resolve by ID after discovery, optionally filtering by `to.statusCategory.key`. + +### Worked recipe: bulk-close stalled sprint issues + +```python +# 1) find candidates (legacy search shown; see rest-auth-and-search.md for token paging) +issues = search('sprint in openSprints() AND updated < -14d AND resolution = Unresolved') + +# 2) per issue: discover + apply +for issue in issues: + trans = get(f"/issue/{issue['key']}/transitions")["transitions"] + done = next(t for t in trans if t["to"]["statusCategory"]["key"] == "completed") + post(f"/issue/{issue['key']}/transitions", + json={"transition": {"id": done["id"]}, + "fields": {"resolution": {"name": "Done"}}}) +``` + +Rate-limit note: writes count toward per-issue windows (20 per 2s) — sleep briefly between issues in loops. + +## Atlassian Document Format (ADF) + +The document model for every rich-text field in v3 payloads: issue `description`/`environment`, comment bodies, textarea custom fields. Plain-text strings are rejected for these fields. + +### Minimal document + +```json +{ "type": "doc", "version": 1, + "content": [ + { "type": "paragraph", + "content": [ { "type": "text", "text": "Hello world" } ] } + ] } +``` + +Structure invariants: exactly one root `doc` with `version: 1`; content is an ordered tree of block nodes (`paragraph`, `heading`, `bulletList`/`orderedList` > `listItem` > `paragraph`, `codeBlock`, `panel`, `table`, `blockquote`); inline content is `text` nodes carrying optional `marks`. + +### Formatting quick reference + +| Effect | Node/mark shape | +|--------|-----------------| +| Bold | `{"type":"text","text":"world","marks":[{"type":"strong"}]}` | +| Italic | `marks: [{"type":"em"}]` | +| Code | `marks: [{"type":"code"}]` | +| Link | `marks: [{"type":"link","attrs":{"href":"https://..."}}]` | +| Bullet list | `bulletList` node whose `listItem`s contain paragraphs | +| Mention | inline node `{"type":"mention","attrs":{"id":""}}` | + +### Practical guidance + +- Building from user input? Wrap each line/paragraph as its own `paragraph` node; escape nothing manually — text goes in the `text` property verbatim. +- Extracting? Walk `content[]` recursively collecting `text` nodes' `text` values joined by newlines (the bundled CLI's `view` does this for descriptions). +- Round-tripping rich content through plain text loses formatting permanently; if fidelity matters, fetch the ADF and re-post the same structure. + +## Sources + +- Issues group (get/create/edit/delete, transitions GET+POST): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issues/ +- Issue comments group: https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-comments/ +- Issue links group (link payload shapes): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-links/ +- Projects group (paginated vs deprecated bare-array): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-projects/ +- Myself resource: https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-myself/ +- REST v3 intro (error collection schema): https://developer.atlassian.com/cloud/jira/platform/rest/v3/intro/ +- ADF structure reference: https://developer.atlassian.com/cloud/jira/platform/apis/document/structure/ +- GDPR accountId migration guide: https://developer.atlassian.com/cloud/jira/platform/deprecation-notice-user-privacy-api-migration-guide/ From 9843ff7f4fe1edb7e773a510ae9ffe6f5c6b49eb Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 06:09:25 -0400 Subject: [PATCH 09/40] feat(jira): extend CLI with count/transitions and pagination fixes Script: add a 'count' subcommand (POST /search/approximate-count) and a 'transitions' discovery subcommand, plus --resolution on transition (satisfies Done-screen requirements in one call), offset-paginated fetches for --max > 50 with empty-page termination when total shrinks, parsed API error envelopes (errors map + errorMessages) instead of raw dumps, and 429 handling surfacing Retry-After/RateLimit-Reason. Existing flags, --json/--dry-run, and subcommands preserved; stdlib+requests only. Tests: 20-case offline suite (help output, argument errors, dry-run plans, mocked-client pagination/error/transition logic) passing pytest and unittest runners including the proxy-trap zero-egress rerun. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- jira/scripts/jira | 151 +++++++++++++++-- jira/scripts/test_jira.py | 336 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 477 insertions(+), 10 deletions(-) create mode 100644 jira/scripts/test_jira.py diff --git a/jira/scripts/jira b/jira/scripts/jira index 09aea36..1c09ed5 100755 --- a/jira/scripts/jira +++ b/jira/scripts/jira @@ -129,12 +129,31 @@ class JiraClient: die("Forbidden (403). Your account may not have access to this resource.") if resp.status_code == 404: return None + if resp.status_code == 429: + retry_after = resp.headers.get("Retry-After", "unknown") + reason = resp.headers.get("RateLimit-Reason", "burst or quota limit") + die("Rate limited (429): {}. Retry-After: {}s".format(reason, retry_after)) if resp.status_code >= 400: try: detail = resp.json() except Exception: detail = resp.text[:200] - die(f"API error ({resp.status_code}): {detail}") + msg = "API error ({})".format(resp.status_code) + if isinstance(detail, dict): + parts = [] + errs = detail.get("errors") + if isinstance(errs, dict) and errs: + parts.extend("{}: {}".format(k, v) for k, v in errs.items()) + msgs = detail.get("errorMessages") + if isinstance(msgs, list) and msgs: + parts.extend(str(m) for m in msgs) + if parts: + msg += ": " + "; ".join(parts) + else: + msg += ": {}".format(detail) + else: + msg += ": {}".format(detail) + die(msg) try: return resp.json() @@ -163,6 +182,42 @@ class JiraClient: "maxResults": max_results }) + def search_issues_paged(self, jql: str, fields: str = "summary,status,issuetype,assignee,priority", + total_max: int = 100, page_size: int = 50) -> List[Dict]: + """Fetch up to total_max issues, following legacy offset pagination. + + The classic /search endpoint pages via startAt/maxResults with a + `total` field (deprecated by Atlassian but still the endpoint this CLI + uses; see references/rest-auth-and-search.md for the /search/jql + nextPageToken replacement). `total` may change between pages, so an + empty page always terminates the loop. + """ + collected: List[Dict] = [] + start_at = 0 + while len(collected) < total_max: + page_size_eff = min(page_size, total_max - len(collected)) + data = self._get("/search", params={ + "jql": jql, + "fields": fields, + "startAt": start_at, + "maxResults": page_size_eff, + }) + if not data: + break + issues = data.get("issues", []) + if not issues: + break + collected.extend(issues) + start_at += len(issues) + total = data.get("total") + if total is None or start_at >= int(total): + break + return collected[:total_max] + + def approximate_count(self, jql: str) -> Any: + """POST /search/approximate-count — fast match count without fetching rows.""" + return self._post("/search/approximate-count", json_data={"jql": jql}) + def get_issue(self, issue_key: str) -> Any: return self._get(f"/issue/{issue_key}") @@ -205,9 +260,14 @@ class JiraClient: def get_transitions(self, issue_key: str) -> Any: return self._get(f"/issue/{issue_key}/transitions") - def transition_issue(self, issue_key: str, transition_id: str) -> Any: - return self._post(f"/issue/{issue_key}/transitions", - json_data={"transition": {"id": transition_id}}) + def transition_issue(self, issue_key: str, transition_id: str, + resolution: Optional[str] = None) -> Any: + payload: Dict[str, Any] = {"transition": {"id": str(transition_id)}} + if resolution: + # Done-style transitions often require a resolution on their screen; + # omitting it yields 400 with errors.resolution naming the field. + payload["fields"] = {"resolution": {"name": resolution}} + return self._post(f"/issue/{issue_key}/transitions", json_data=payload) # === Command Handlers === @@ -252,7 +312,12 @@ def cmd_list(client: JiraClient, args: List[str]) -> None: "max_results": parsed.max}) return - data = client.search_issues(jql, max_results=parsed.max) + if parsed.max > 50: + # Multi-page fetch via offset pagination; single-page path stays identical. + paged = client.search_issues_paged(jql, total_max=parsed.max) + data = {"issues": paged, "total": len(paged)} + else: + data = client.search_issues(jql, max_results=parsed.max) issues = data.get("issues", []) if data else [] if not issues: @@ -429,15 +494,19 @@ def cmd_transition(client: JiraClient, args: List[str]) -> None: parser = argparse.ArgumentParser(prog="jira transition") parser.add_argument("issue_key", help="Issue key (e.g. PROJ-123)") parser.add_argument("--to", required=True, help="Transition by name or ID") + parser.add_argument("--resolution", help='Set a resolution during the ' + 'transition (often required for Done; e.g. "Fixed")') parsed, _ = parser.parse_known_args(args) if client.dry_run: emit(f"[dry-run] Would transition {parsed.issue_key} to '{parsed.to}'", {"dry_run": True, "command": "transition", - "issue_key": parsed.issue_key, "to": parsed.to}) + "issue_key": parsed.issue_key, "to": parsed.to, + "resolution": parsed.resolution}) return - # First, get available transitions + # First, get available transitions — IDs are workflow- and status-specific, + # so names alone are ambiguous across projects. trans_data = client.get_transitions(parsed.issue_key) if not trans_data: die(f"No transitions available for {parsed.issue_key}") @@ -457,10 +526,63 @@ def cmd_transition(client: JiraClient, args: List[str]) -> None: for t in transitions]) die(f"Transition '{parsed.to}' not found. Available: {available}") - result = client.transition_issue(parsed.issue_key, match) - emit(f"🔄 {parsed.issue_key} transitioned to '{parsed.to}'", + result = client.transition_issue(parsed.issue_key, match, + resolution=parsed.resolution) + _ = result # 204 No Content on success + res_note = f" (resolution: {parsed.resolution})" if parsed.resolution else "" + emit(f"🔄 {parsed.issue_key} transitioned to '{parsed.to}'{res_note}", {"status": "transitioned", "issue_key": parsed.issue_key, - "transition": parsed.to}) + "transition": parsed.to, "resolution": parsed.resolution}) + + +def cmd_transitions(client: JiraClient, args: List[str]) -> None: + """List available transitions for an issue (discovery half of the flow).""" + parser = argparse.ArgumentParser(prog="jira transitions") + parser.add_argument("issue_key", help="Issue key (e.g. PROJ-123)") + parsed, _ = parser.parse_known_args(args) + + if client.dry_run: + emit(f"[dry-run] Would list transitions for {parsed.issue_key}", + {"dry_run": True, "command": "transitions", + "issue_key": parsed.issue_key}) + return + + data = client.get_transitions(parsed.issue_key) + transitions = data.get("transitions", []) if data else [] + if not transitions: + emit(f"No transitions available for {parsed.issue_key}.", + {"transitions": []}) + return + + lines = [] + out = [] + for t in transitions: + to_status = t.get("to", {}).get("name", "?") + category = t.get("to", {}).get("statusCategory", {}).get("key", "?") + lines.append(f" {t.get('id', '?'):>5} {t.get('name', '?'):20} -> {to_status} [{category}]") + out.append({"id": t.get("id"), "name": t.get("name"), + "to_status": to_status, "status_category": category}) + emit("Available transitions:\n" + "\n".join(lines), + {"transitions": out}) + + +def cmd_count(client: JiraClient, args: List[str]) -> None: + """Approximate match count via /search/approximate-count.""" + parser = argparse.ArgumentParser(prog="jira count") + parser.add_argument("--jql", required=True, help="JQL query to count") + parser.add_argument("--project", help="Shortcut: prefix with project=KEY AND") + parsed, _ = parser.parse_known_args(args) + + jql = f"project={parsed.project} AND ({parsed.jql})" if parsed.project else parsed.jql + + if client.dry_run: + emit(f"[dry-run] Would count issues matching: {jql}", + {"dry_run": True, "command": "count", "jql": jql}) + return + + result = client.approximate_count(jql) + count = result.get("count", 0) if isinstance(result, dict) else 0 + emit(f"{count} issue(s) match", {"jql": jql, "count": count}) # === Main === @@ -483,6 +605,12 @@ def main() -> None: sub.add_parser("me", help="Get current user profile") sub.add_parser("projects", help="List projects") + p_transitions = sub.add_parser("transitions", help="List available transitions for an issue") + p_transitions.add_argument("issue_key", help="Issue key (e.g. PROJ-123)") + + p_count = sub.add_parser("count", help="Count issues matching a JQL query (fast approximate count)") + p_count.add_argument("--jql", required=True, help="JQL query") + p_list = sub.add_parser("list", help="Search issues") p_list.add_argument("--jql", default="", help="JQL query") p_list.add_argument("--max", type=int, default=20, help="Max results") @@ -505,6 +633,7 @@ def main() -> None: p_trans = sub.add_parser("transition", help="Transition an issue") p_trans.add_argument("issue_key", help="Issue key") p_trans.add_argument("--to", required=True, help="Transition name or ID") + p_trans.add_argument("--resolution", help='Resolution to set during transition (e.g. "Fixed")') args = parser.parse_args(filtered_argv[1:]) if not args.command: @@ -530,6 +659,8 @@ def main() -> None: "create": cmd_create, "comment": cmd_comment, "transition": cmd_transition, + "transitions": cmd_transitions, + "count": cmd_count, } handler = cmd_map.get(args.command) if not handler: diff --git a/jira/scripts/test_jira.py b/jira/scripts/test_jira.py new file mode 100644 index 0000000..faca0a7 --- /dev/null +++ b/jira/scripts/test_jira.py @@ -0,0 +1,336 @@ +"""Offline tests for the bundled jira CLI (scripts/jira). + +Four test classes per skill-builder contract: + 1. --help output + 2. argument-error paths + 3. --dry-run behavior + 4. mocked-client logic (requests mocked at the client-call site) + +Zero network calls in every test; the proxy-trap rerun proves egress-freedom. +""" + +import contextlib +import importlib.machinery +import importlib.util +import io +import json +import pathlib +import unittest +from unittest import mock + +import requests + +SCRIPT = pathlib.Path(__file__).resolve().parent / "jira" +LOADER = importlib.machinery.SourceFileLoader("jira_cli", str(SCRIPT)) +SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER) +jira_cli = importlib.util.module_from_spec(SPEC) +LOADER.exec_module(jira_cli) + + +def run_cli(*argv, env_email="ops@example.com", env_token="test-token-123", + clear_env=False): + """Invoke main() with argv[0] prepended; returns (exit_code, stdout, stderr). + + clear_env=True strips JIRA_EMAIL/JIRA_API_TOKEN to exercise lazy-auth paths. + """ + out, err = io.StringIO(), io.StringIO() + code = 0 + if clear_env: + env = {} + else: + env = {"JIRA_EMAIL": env_email or "", "JIRA_API_TOKEN": env_token or ""} + with mock.patch.dict("os.environ", env, clear=True): + with mock.patch.object(jira_cli.sys, "argv", ["jira", *argv]): + with mock.patch.object(jira_cli.sys, "stdout", out), \ + mock.patch.object(jira_cli.sys, "stderr", err), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + jira_cli.main() + except SystemExit as exc: + code = exc.code if isinstance(exc.code, int) else 0 + return code, out.getvalue(), err.getvalue() + + +class FakeResponse: + def __init__(self, status_code=200, payload=None, text="", headers=None): + self.status_code = status_code + self._payload = payload + self.text = text or (json.dumps(payload) if payload is not None else "") + self.headers = headers or {} + + def json(self): + if self._payload is None: + raise ValueError("no json") + return self._payload + + +class SearchPageFactory: + """Builds legacy-offset search pages for pagination tests.""" + + def __init__(self, total, page_size=2, key_prefix="PROJ"): + self.total = total + self.page_size = page_size + self.key_prefix = key_prefix + + def issue(self, n): + return {"key": f"{self.key_prefix}-{n}", + "fields": {"summary": f"issue {n}", + "status": {"name": "Open"}, + "issuetype": {"name": "Task"}, + "assignee": None, + "priority": {"name": "Medium"}}} + + def page(self, start_at): + issues = [self.issue(n) for n in range( + start_at + 1, min(start_at + self.page_size, self.total) + 1)] + return {"issues": issues, "startAt": start_at, + "maxResults": self.page_size, "total": self.total} + + +# === Class 1: help output === + + +class HelpOutputTests(unittest.TestCase): + def test_help_lists_all_subcommands(self): + code, out, _ = run_cli("--help") + self.assertEqual(code, 0) + for noun in ("me", "list", "view", "projects", "create", + "comment", "transition", "transitions", "count"): + self.assertIn(noun, out) + + def test_subcommand_help_mentions_flags(self): + _, out, _ = run_cli("list", "--help") + for flag in ("--jql", "--max", "--project"): + self.assertIn(flag, out) + + +# === Class 2: argument errors === + + +class ArgumentErrorTests(unittest.TestCase): + def test_missing_required_jql_on_count(self): + code, _, err = run_cli("count") + self.assertEqual(code, 2) + self.assertIn("--jql", err) + + def test_transition_requires_to_flag(self): + code, _, err = run_cli("transition", "PROJ-1") + self.assertEqual(code, 2) + self.assertIn("--to", err) + + def test_no_command_prints_help_and_exits(self): + code, out, _ = run_cli() + self.assertEqual(code, 1) + self.assertIn("usage:", out) + + def test_unknown_subcommand_fails(self): + code, _, err = run_cli("frobnicate") + self.assertEqual(code, 2) + self.assertIn("invalid choice", err) + + +# === Class 3: dry-run behavior === + + +class DryRunTests(unittest.TestCase): + def test_dry_run_count_emits_plan_json_without_credentials(self): + # No JIRA_EMAIL/JIRA_API_TOKEN set at all — lazy auth must not fire. + code, out, _ = run_cli("--json", "--dry-run", "count", + "--jql", "status=Open", clear_env=True) + self.assertEqual(code, 0) + plan = json.loads(out) + self.assertTrue(plan["dry_run"]) + self.assertEqual(plan["command"], "count") + self.assertIn("status=Open", plan["jql"]) + + def test_dry_run_create_reports_payload_shape(self): + code, out, _ = run_cli("--json", "--dry-run", "create", + "--project", "PROJ", "--summary", "Test issue") + self.assertEqual(code, 0) + plan = json.loads(out) + self.assertEqual(plan["command"], "create") + self.assertEqual(plan["project"], "PROJ") + + def test_dry_run_never_touches_network(self): + with mock.patch.object(requests, "request") as req: + code, out, _ = run_cli("--dry-run", "list", "--project", "PROJ") + self.assertEqual(code, 0) + req.assert_not_called() + + +# === Class 4: mocked client logic === + + +class MockedClientTests(unittest.TestCase): + def setUp(self): + self.flags = dict(jira_cli.GLOBAL_FLAGS) + jira_cli.GLOBAL_FLAGS.update(json=True, dry_run=False, quiet=False) + # Module-level ENV_* constants are captured at import; pin them here so + # run_cli's env dict is the single source of auth truth in tests. + self._env = (jira_cli.ENV_EMAIL, jira_cli.ENV_TOKEN) + jira_cli.ENV_EMAIL = "ops@example.com" + jira_cli.ENV_TOKEN = "test-token-123" + + def tearDown(self): + jira_cli.ENV_EMAIL, jira_cli.ENV_TOKEN = self._env + jira_cli.GLOBAL_FLAGS.clear() + jira_cli.GLOBAL_FLAGS.update(self.flags) + + def test_list_single_page_parses_issue_rows(self): + page = SearchPageFactory(total=2, page_size=50) + resp = FakeResponse(200, page.page(0)) + with mock.patch.object(requests, "request", return_value=resp) as req: + code, out, _ = run_cli("list", "--project", "PROJ", "--json") + self.assertEqual(code, 0) + req.assert_called_once() + data = json.loads(out) + self.assertEqual(data["total"], 2) + self.assertEqual([i["key"] for i in data["issues"]], + ["PROJ-1", "PROJ-2"]) + self.assertEqual(data["issues"][0]["assignee"], "Unassigned") + + def test_list_multipage_follows_offset_pagination(self): + factory = SearchPageFactory(total=5, page_size=2) + responses = [FakeResponse(200, factory.page(s)) for s in (0, 2, 4)] + + captured = [] + + def fake_request(method, url, **kwargs): + captured.append(kwargs["params"]["startAt"]) + return responses[len(captured) - 1] + + with mock.patch.object(requests, "request", side_effect=fake_request): + code, out, _ = run_cli("list", "--project", "PROJ", "--max", "100", + "--json") + self.assertEqual(code, 0) + self.assertEqual(captured, [0, 2, 4]) + data = json.loads(out) + self.assertEqual(data["total"], 5) + + def test_list_multipage_stops_on_empty_page_when_total_shrinks(self): + # Page 1 full, then `total` drops below startAt -> empty page terminates. + factory = SearchPageFactory(total=4, page_size=2) + first = factory.page(0) + second = {"issues": [], "startAt": 2, "maxResults": 2, "total": 2} + responses = [FakeResponse(200, first), FakeResponse(200, second)] + + def fake_request(method, url, **kwargs): + return responses.pop(0) + + with mock.patch.object(requests, "request", side_effect=fake_request): + code, out, _ = run_cli("list", "--max", "60", "--json") + self.assertEqual(code, 0) + data = json.loads(out) + self.assertEqual(len(data["issues"]), 2) + + def test_error_envelope_is_parsed_into_message(self): + resp = FakeResponse(400, {"errorMessages": [], + "errors": {"resolution": "Resolution is required"}}) + with mock.patch.object(requests, "request", return_value=resp): + code, _, err = run_cli("comment", "PROJ-1", "-m", "hi") + self.assertEqual(code, 1) + self.assertIn("resolution: Resolution is required", err) + + def test_rate_limit_surfaces_retry_after(self): + resp = FakeResponse(429, {"errorMessages": []}, + headers={"Retry-After": "42", + "RateLimit-Reason": "jira-burst-based"}) + with mock.patch.object(requests, "request", return_value=resp): + code, _, err = run_cli("me") + self.assertEqual(code, 1) + self.assertIn("429", err) + self.assertIn("42", err) + self.assertIn("jira-burst-based", err) + + def test_auth_failure_names_env_vars(self): + resp = FakeResponse(401, {"errorMessages": ["Unauthorized"]}) + with mock.patch.object(requests, "request", return_value=resp): + code, _, err = run_cli("me") + self.assertEqual(code, 1) + self.assertIn("JIRA_EMAIL", err) + self.assertIn("401", err) + + def test_transitions_listing_formats_rows(self): + payload = {"transitions": [ + {"id": "31", "name": "Done", + "to": {"name": "Done", + "statusCategory": {"key": "completed"}}}, + {"id": "11", "name": "Start Progress", + "to": {"name": "In Progress", + "statusCategory": {"key": "in-flight"}}}, + ]} + resp = FakeResponse(200, payload) + with mock.patch.object(requests, "request", return_value=resp): + code, out, _ = run_cli("transitions", "PROJ-1", "--json") + self.assertEqual(code, 0) + data = json.loads(out) + self.assertEqual([t["id"] for t in data["transitions"]], ["31", "11"]) + + def test_transition_resolves_name_to_id_and_posts_resolution(self): + listing = FakeResponse(200, {"transitions": [ + {"id": "31", "name": "Done", + "to": {"name": "Done", + "statusCategory": {"key": "completed"}}}]}) + + posted = [] + + def fake_request(method, url, **kwargs): + if method == "GET": + return listing + posted.append((url, kwargs.get("json"))) + return FakeResponse(204, None, text="") + + with mock.patch.object(requests, "request", side_effect=fake_request): + code, out, _ = run_cli("transition", "PROJ-1", "--to", "Done", + "--resolution", "Fixed", "--json") + self.assertEqual(code, 0) + self.assertEqual(len(posted), 1) + url, body = posted[0] + self.assertIn("/issue/PROJ-1/transitions", url) + self.assertEqual(body["transition"], {"id": "31"}) + self.assertEqual(body["fields"]["resolution"], {"name": "Fixed"}) + + def test_transition_unknown_name_lists_available_options(self): + listing = FakeResponse(200, {"transitions": [ + {"id": "11", "name": "Start Progress", + "to": {"name": "In Progress", + "statusCategory": {"key": "in-flight"}}}]}) + with mock.patch.object(requests, "request", return_value=listing): + code, _, err = run_cli("transition", "PROJ-1", "--to", "Done") + self.assertEqual(code, 1) + self.assertIn("Available: 11=Start Progress", err) + + def test_count_uses_approximate_count_endpoint(self): + resp = FakeResponse(200, {"count": 153}) + + with mock.patch.object(requests, "request", return_value=resp) as req: + code, out, _ = run_cli("count", "--jql", "status=Open", "--json") + self.assertEqual(code, 0) + url = req.call_args.kwargs["url"] + self.assertIn("/search/approximate-count", url) + self.assertEqual(req.call_args.kwargs["json"], {"jql": "status=Open"}) + self.assertEqual(json.loads(out)["count"], 153) + + def test_view_extracts_plain_text_from_adf_description(self): + adf = {"type": "doc", "version": 1, "content": [ + {"type": "paragraph", "content": [ + {"type": "text", "text": "First paragraph"}]}, + {"type": "paragraph", "content": [ + {"type": "text", "text": "Second paragraph"}]}, + ]} + payload = {"key": "PROJ-9", "fields": { + "summary": "Broken flow", "status": {"name": "Open"}, + "issuetype": {"name": "Bug"}, "assignee": None, + "description": adf}} + with mock.patch.object(requests, "request", + return_value=FakeResponse(200, payload)): + code, out, _ = run_cli("view", "PROJ-9", "--json") + self.assertEqual(code, 0) + data = json.loads(out) + self.assertIn("First paragraph\nSecond paragraph", + data["description"]) + self.assertEqual(data["assignee"], "Unassigned") + + +if __name__ == "__main__": + unittest.main() From 1aab3e991cce5547bc82f4a98cecf57c372fb706 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 06:10:12 -0400 Subject: [PATCH 10/40] docs(jira): lastfm-model SKILL.md rewrite, evals manifest, README Rewrite SKILL.md (199/500 body lines) with Setup (basic-auth token mechanics), intent-grouped commands incl. new count/transitions, three pipeline recipes, jq guidance, Known Gotchas grounded in researched API behavior (search-endpoint duality with CHANGE-2046 status, transition screen-field requirements, ADF, CAPTCHA lockouts, rate-limit headers, accountId GDPR migration), When-to-use / When-not-to-use boundary, and a routing table covering all six reference files. Add schema-v1 evals/evals.json: six cases (read-only search, stalled-sprint bulk-close pipeline, transition-id gotcha, approximate-count, auth setup/expiry, and a GitHub-issue should-not-trigger negative probe). Passes validate-evals and the paired fake smoke. Refresh README to human format with What You Get table listing real paths. Regenerate catalogs for the description change; sync root README blurb. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- jira/README.md | 49 +++++++---- jira/SKILL.md | 187 +++++++++++++++++++++++++++--------------- jira/evals/evals.json | 71 ++++++++++++++++ 3 files changed, 221 insertions(+), 86 deletions(-) create mode 100644 jira/evals/evals.json diff --git a/jira/README.md b/jira/README.md index 0a5d87e..30f0ad7 100644 --- a/jira/README.md +++ b/jira/README.md @@ -1,39 +1,52 @@ # Jira Issue Tracker from the Terminal -Interact with Atlassian Jira Cloud via the REST API v3. Search issues, view details, create issues, add comments, list projects, and transition status. +Interact with Atlassian Jira Cloud via the REST API v3: search issues with JQL, view details, create issues, add comments, count matches, list projects, and transition status — plus a full JQL language reference built in. ## Why Install This Skill -When your agent loads this skill, it can **manage your entire Jira workflow** without opening a browser. That means: +When your agent loads this skill, it can **run your entire Jira workflow** without opening a browser: -- **Search issues by project, JQL, or assignee** — find anything in your tracker -- **View full issue details** — description, status, assignee, comments -- **Create and update issues** — new tasks, bugs, stories from the terminal -- **Add comments** — update threads without the web UI -- **Transition status** — move tickets through workflows -- **List projects** — see what's available +- **Search anything** — by project, assignee, or arbitrary JQL; results over 50 auto-paginate +- **Count before diving in** — fast approximate counts instead of fetching every ticket +- **Create, comment, edit** — with Atlassian Document Format handled for you +- **Transition safely** — discovers valid workflow transitions per issue before changing status, and can set resolutions in the same call +- **Write better queries** — a 50-query cookbook by role, complete function catalog, performance rules, and history-operator/date-expression deep dives + +The skill also knows where the bodies are buried: the legacy-vs-enhanced search endpoint split (offset paging vs `nextPageToken`), transition screens that silently require resolution fields, the `!=` empty-value trap, and rate-limit headers worth honoring. ## What You Get -| Directory | Purpose | -|-----------|---------| -| `SKILL.md` | Complete command reference with setup and examples | -| `scripts/jira` | CLI tool for Jira REST API v3 | +| Path | Purpose | +|------|---------| +| `SKILL.md` | Command reference: setup, intent-grouped commands, pipeline recipes, jq guidance, known gotchas | +| `scripts/jira` | CLI tool for Jira REST API v3 (`--json`, `--dry-run`, lazy auth) | +| `scripts/test_jira.py` | Offline test suite for the CLI (help/errors/dry-run/mocked client logic) | +| `references/rest-auth-and-search.md` | Auth models, rate limits, error envelopes, search pagination duality | +| `references/rest-issues-and-transitions.md` | Issue CRUD shapes, transitions GET→POST flow, ADF document model | +| `references/jql-functions-catalog.md` | Every JQL function with fields and operators, incl. JSM approvals & SLAs | +| `references/jql-best-practices.md` | Performance rules, precedence, empty-value trap, troubleshooting flows | +| `references/jql-cookbook.md` | 50 ready-to-run JQL queries organized by role | +| `references/jql-history-and-dates.md` | WAS/CHANGED walkthrough, relative-date tables, saved-filter naming | +| `evals/evals.json` | Behavioral eval cases covering read-only use, pipelines, gotchas | ## Quick Start ```bash -export JIRA_EMAIL="your-email@example.com" -export JIRA_API_TOKEN="your-api-token" +export JIRA_EMAIL="you@company.com" +export JIRA_API_TOKEN="YOUR_API_TOKEN" # free from https://id.atlassian.com/manage/api-tokens export JIRA_SERVER="https://your-domain.atlassian.net" -``` -API token from id.atlassian.com (free). +jira me # verify auth works +jira list --project PROJ # newest tickets +jira count --jql 'issuetype = Bug AND resolution = Unresolved' +``` ## Triggers -Load this when managing Jira issues, searching tickets, creating bugs, or tracking project work. +Load this when managing Jira issues, searching or counting tickets, creating bugs, transitioning sprint work, writing/debugging/optimizing JQL, or designing saved filters and dashboards on an Atlassian Jira Cloud site. ## Requirements -Python 3.8+ with `requests` library. +- Python 3.8+ with the `requests` library +- A free Atlassian account + API token (`JIRA_EMAIL`, `JIRA_API_TOKEN`; optional `JIRA_SERVER`) +- `jq` recommended for processing `--json` output diff --git a/jira/SKILL.md b/jira/SKILL.md index 0c0cc9f..ed245b6 100644 --- a/jira/SKILL.md +++ b/jira/SKILL.md @@ -1,16 +1,18 @@ --- name: jira description: 'Interact with Atlassian Jira from the terminal: search issues with - JQL, view details, create issues, add comments, list projects, and transition + JQL, view details, create issues, add comments, count matches with fast + approximate-count, list projects, discover valid transitions, and change status. Includes a full JQL language reference (functions, operators, history - queries, performance tuning). Use when the user mentions Jira, a ticket key + predicates, date expressions, saved filters, performance tuning) plus REST + auth/pagination guidance. Use when the user mentions Jira, a ticket key (e.g. PROJ-123), asks about issues, bugs, tasks, projects, or sprint work, or needs to write, debug, or optimize JQL queries. Do not use for GitHub or GitLab issue tracking, Jira site administration, or generic ticketing systems.' license: MIT -compatibility: Requires JIRA_EMAIL and JIRA_API_TOKEN env vars (free from id.atlassian.com/manage/api-tokens), - Python 3.8+, and the `requests` library. Also requires JIRA_SERVER (defaults to - your-domain.atlassian.net). +compatibility: Requires JIRA_EMAIL and JIRA_API_TOKEN env vars (free API token from + id.atlassian.com/manage/api-tokens), Python 3.8+, and the `requests` library. + JIRA_SERVER defaults to your-domain.atlassian.net format. metadata: tags: jira, atlassian, issue-tracking, project-management, api-client sources: https://developer.atlassian.com/cloud/jira/platform/rest/v3/, https://id.atlassian.com/manage/api-tokens @@ -18,7 +20,7 @@ metadata: # jira — Jira Issue Tracker from the Terminal -Interact with Atlassian Jira Cloud via the REST API v3. Search issues, view details, create issues, add comments, list projects, and transition status. +Interact with Atlassian Jira Cloud via the REST API v3. Search issues, view details, create issues, add comments, count matches, list projects, and transition status. ## Setup @@ -26,142 +28,191 @@ Interact with Atlassian Jira Cloud via the REST API v3. Search issues, view deta 2. Set environment variables: ```bash -export JIRA_EMAIL="your-email@example.com" -export JIRA_API_TOKEN="your-api-token" -export JIRA_SERVER="https://your-domain.atlassian.net" # defaults to this format +export JIRA_EMAIL="your-email@example.com" # Atlassian account email +export JIRA_API_TOKEN="YOUR_API_TOKEN" # from id.atlassian.com +export JIRA_SERVER="https://your-domain.atlassian.net" ``` -`--help` and `--dry-run` work without credentials. +Auth is HTTP Basic over `base64(email:token)` — your **email address**, never a password (passwords are deprecated for API use). Cloud has no Personal Access Tokens; Bearer PATs are Data Center only. Tokens now expire after at most one year. `--help` and `--dry-run` work without credentials. ## Essential Commands -### me — Current user profile +### me / projects — identity and scope ```bash -jira me # your account info -jira me --json # machine-readable +jira me # verify auth; your accountId, timezone +jira projects --json # all accessible projects ``` -### list — Search issues +### list — search issues ```bash jira list # recent issues jira list --project PROJ # by project jira list --jql 'assignee=currentuser() AND status=Open' # custom JQL -jira list --project PROJ --max 5 --json # top 5 as JSON +jira list --project PROJ --max 120 --json # >50 auto-pages via startAt offsets ``` -The `--jql` flag accepts any valid JQL. The `--project` flag is a shortcut for `project=KEY`. - -### view — Issue details +### view — issue details ```bash -jira view PROJ-123 # full details +jira view PROJ-123 # summary, status, assignee, description jira view PROJ-123 --json # machine-readable ``` -Shows: summary, type, status, priority, assignee, reporter, timestamps, and description (plain text extracted from Atlassian Document Format). +Descriptions arrive as Atlassian Document Format (ADF); the CLI extracts plain text for display. -### projects — List projects +### count — fast match total ```bash -jira projects # all accessible projects -jira projects --json # machine-readable +jira count --jql 'issuetype = Bug AND resolution = Unresolved' # {"count": N} ``` -### create — Create an issue +Uses `POST /search/approximate-count` — no fetching rows. JQL itself has no COUNT/aggregation. + +### create — new issues ```bash jira create --project PROJ --summary "Fix login bug" # Task (default) jira create --project PROJ --summary "Crash on startup" --type Bug jira create --project PROJ --summary "Add dark mode" --type Story --priority High -jira create --project PROJ --summary "Test" --dry-run # preview +jira create --project PROJ --summary "Test" --dry-run # preview payload ``` -### comment — Add a comment +Descriptions are sent as ADF documents. Rich formatting beyond plain paragraphs needs raw ADF JSON — see references/rest-issues-and-transitions.md. + +### comment — add to threads ```bash -jira comment PROJ-123 -m "Fixed in latest build" # add comment +jira comment PROJ-123 -m "Fixed in latest build" jira comment PROJ-123 -m "Looking into it" --dry-run ``` -### transition — Change issue status +### transitions + transition — status changes ```bash -jira transition PROJ-123 --to "In Progress" # by name -jira transition PROJ-123 --to "Done" # by name -jira transition PROJ-123 --to "31" # by ID +jira transitions PROJ-123 # LIST valid transition IDs first +jira transition PROJ-123 --to "In Progress" # then apply by name or ID +jira transition PROJ-123 --to Done --resolution Done jira transition PROJ-123 --to "In Review" --dry-run ``` -The CLI looks up available transitions for the issue and matches by name or ID. If the transition doesn't exist, it shows available options. +Always run `transitions` first when unsure: IDs differ per workflow and current status, and names repeat across workflows. `--resolution` satisfies Done-style screens that require one; omitting it yields `400` with an error naming the missing field. ## Global Flags All flags work in any position: ```bash -jira --json list --project PROJ # flag before subcommand -jira list --project PROJ --json # flag after subcommand -jira --dry-run create --project PROJ --summary "Test" # preview -jira --quiet list # suppress non-essential output +jira --json list --project PROJ # machine output anywhere +jira --dry-run create --project PROJ --summary "Test" # offline preview +jira --quiet list # suppress non-essential output ``` -## Known Gotchas - -- **Authentication** uses HTTP Basic Auth with email + API token. This is the email address tied to your Atlassian account, not a username. -- **Atlassian Document Format (ADF)** — Issue descriptions and comments use ADF (JSON structure), not plain text or markdown. The CLI extracts plain text from ADF, but creating issues with rich formatting requires ADF JSON via `--description`. -- **Transitions are workflow-specific** — Available transitions depend on the issue's current status and the project's workflow. The CLI lists available options when an invalid transition is requested. -- **Rate limits** — Jira Cloud has rate limits. The API returns 429 if exceeded. The CLI does not auto-retry. -- **Project keys are case-sensitive** in some contexts, but the Jira API generally accepts uppercase or lowercase. - -### JQL gotchas - -- **`!=` excludes empty values** — `assignee != currentUser()` silently drops unassigned issues. Write `(assignee != currentUser() OR assignee IS EMPTY)` to include them. -- **AND binds tighter than OR** — `A OR B AND C` parses as `A OR (B AND C)`. Always parenthesize OR groups. -- **No leading wildcards** — `summary ~ "*bug"` forces a full scan and is very slow; put wildcards after the first few characters. -- **Filter by project first** — the single biggest JQL performance lever on large instances. -- **JQL has no aggregation** — no COUNT/SUM/AVG in the query language itself. -- **History operators need history tracking** — `WAS`/`CHANGED` return nothing for custom fields without history enabled. -- **Search endpoint duality** — this CLI uses the classic `/rest/api/3/search` endpoint with offset pagination (`startAt`, `maxResults`). Atlassian's enhanced `/rest/api/3/search/jql` replaces it with a `nextPageToken` model and no offset; the classic endpoint is being deprecated, so expect migration. Mixing the two pagination models is a common source of truncated or erroring result pages. +`--json` emits one JSON object per command on stdout — pipe to jq for structure. ## Multi-Step Pipeline Recipes ### Sprint hygiene sweep -Find stalled sprint work, then bulk-review each ticket: +Find stalled sprint work, review each ticket, close what's finished: ```bash -jira list --jql 'sprint IN openSprints() AND updated < -14d AND status != Done' --json \ +jira list --jql 'sprint IN openSprints() AND updated < -14d AND resolution = Unresolved' --json \ | jq -r '.issues[].key' \ - | while read -r key; do jira view "$key"; done + | while read -r key; do jira view "$key"; jira transitions "$key"; done +# after human review, per key: +jira transition "$key" --to Done --resolution Done ``` -The `--json` output shape from `list` is `{"total": N, "issues": [{"key", "summary", "status", "assignee", "issuetype", "priority"}]}` — pipe through `jq -r '.issues[].key'` to feed follow-up commands. +The `list --json` shape is `{"total": N, "issues": [{"key", "summary", "status", "assignee", "issuetype", "priority"}]}`. -### My-week digest +### Bulk-close with safe discovery + +Transition IDs are workflow-specific — resolve before writing: + +```bash +for key in $(jira list --jql 'status = "In Progress" AND updated < -30d' --json | jq -r '.issues[].key'); do + tid=$(jira transitions "$key" --json | jq -r '.transitions[] | select(.status_category=="completed") | .id' | head -1) + [ -n "$tid" ] && jira transition "$key" --to "$tid" --resolution Done +done +``` + +### Weekly digest via jq ```bash jira list --jql 'assignee = currentUser() AND updated >= startOfWeek()' --max 50 --json \ | jq -r '.issues[] | "\(.key)\t\(.status)\t\(.summary)"' ``` -More ready-to-run queries live in [references/jql-cookbook.md](references/jql-cookbook.md), organized by role (developers, scrum masters, product owners, admins). +More ready-to-run queries live in [references/jql-cookbook.md](references/jql-cookbook.md), organized by role. + +## Using --json with jq + +```bash +jira list --project PROJ --json | jq '.issues[] | {key, status, assignee}' +jira count --jql 'project = PROJ' --json | jq .count +jira transitions PROJ-123 --json | jq -r '.transitions[] | "\(.id)=\(.name) -> \(.to_status)"' +``` + +## Known Gotchas + +- **Search endpoint duality** — this CLI uses the classic `/rest/api/3/search` with offset pagination (`startAt`, `maxResults`, `total`). Atlassian's enhanced `/rest/api/3/search/jql` replaces it with an opaque `nextPageToken` (+ `isLast`), no `startAt`, no `total`, ids-only default fields, and it rejects unbounded JQL (`order by key desc` alone → 400). The classic endpoint is deprecated ("currently being removed", announced Oct 2024, removal promised after May 1 2025), so expect forced migration; mixing the two pagination models is the classic source of infinite-page-one loops. +- **Pagination caps** — legacy pages default to `maxResults=50`; `total` can shrink between pages, so always tolerate empty pages instead of trusting a stale total. +- **Transitions need GET first** — transition IDs (`"31"`, `"711"`) belong to one workflow/status; asking for an invalid one returns an *empty list*, not an error. Done-style screens frequently require `resolution`; missing required fields come back as `400` with `"errors": {"resolution": "..."}` naming them. +- **ADF everywhere** — descriptions, comments, and environment fields take ADF JSON objects in v3 payloads; bare strings are rejected. +- **Authentication** uses HTTP Basic with email + API token. CAPTCHA lockouts (repeated bad logins) block REST auth entirely; symptom header: `X-Seraph-LoginReason: AUTHENTICATION_DENIED`. Fix in the browser, not by retrying. +- **Rate limits** return 429 with `Retry-After` and `RateLimit-Reason` headers; the CLI surfaces both but does not auto-retry. Writes also cap at ~20/2s per issue. +- **Project keys are case-sensitive** in some contexts, though the API generally accepts either case. +- **accountId, not username** — user fields accept Atlassian account IDs (GDPR migration); usernames were removed from the API. + +### JQL gotchas + +- **`!=` excludes empty values** — `assignee != currentUser()` silently drops unassigned issues. Write `(assignee != currentUser() OR assignee IS EMPTY)`. +- **AND binds tighter than OR** — `A OR B AND C` parses as `A OR (B AND C)`. Always parenthesize OR groups; without parentheses evaluation is left-to-right. +- **No leading wildcards** — `summary ~ "*bug"` forces a full scan; put wildcards after the first characters. +- **Filter by project first** — the biggest performance lever on large instances (official optimization guidance). +- **History operators have a field whitelist** — `WAS`/`CHANGED` work only on Assignee, Fix Version, Priority, Reporter, Resolution, Status, and silently return nothing on fields without history tracking. +- **Relative dates are case-sensitive** — `-1m` is minutes, `-1M` is months; day-grain expressions evaluate in each user's timezone. +- **JQL has no aggregation** — no COUNT/SUM; use `jira count` (approximate-count endpoint) or dashboard gadgets. + +## When to use + +- Any Jira Cloud interaction from the terminal: search, view, create, comment, transition +- Writing, debugging, or optimizing JQL queries — full language reference included +- Sprint reviews, triage sweeps, bulk status hygiene, dashboards and saved-filter design + +## When not to use + +Do not use this skill for GitHub or GitLab issue tracking (use those platforms' own tooling such as `gh`), for Jira site administration like permission schemes or workflow editing (admin UI territory), for Confluence content, or for building server-side integrations against the Jira API (use official Atlassian SDK docs instead). ## Reference Files | File | Topic | Read when | |------|-------|-----------| -| [references/jql-functions-catalog.md](references/jql-functions-catalog.md) | Every JQL function with fields/operators — date/time, user, sprint/version, issue, custom field, plus JSM approval and SLA functions | Writing or debugging a query that uses functions; checking which operators a function supports | -| [references/jql-best-practices.md](references/jql-best-practices.md) | Performance rules, operator precedence, the empty-value trap, common mistakes, troubleshooting flow, marketplace extensions | A query is slow, returns wrong/zero results, or mixes AND/OR | -| [references/jql-cookbook.md](references/jql-cookbook.md) | 50 ready-to-run JQL queries organized by role (developers, scrum masters, product owners/managers, power users, admins) | Building dashboards, saved filters, automation rules, or sprint reviews | +| [references/rest-auth-and-search.md](references/rest-auth-and-search.md) | Basic-auth/token mechanics vs OAuth/PATs, rate-limit headers, error envelopes, legacy-vs-enhanced search pagination duality | Setting up credentials, handling 429/401s, paginating large searches, or migrating off `/search` | +| [references/rest-issues-and-transitions.md](references/rest-issues-and-transitions.md) | GET/POST/PUT issue shapes, transitions GET→POST flow with screen-field requirements, ADF document model | Creating/editing issues programmatically, resolving transition failures, formatting rich text | +| [references/jql-functions-catalog.md](references/jql-functions-catalog.md) | Every JQL function with supported fields/operators — date/time, user, sprint/version, custom field, JSM approvals & SLAs | Checking which operators/functions a query can use | +| [references/jql-best-practices.md](references/jql-best-practices.md) | Operator precedence, performance rules, the empty-value trap, troubleshooting flows, marketplace extensions | A query is slow, wrong, or mixes AND/OR | +| [references/jql-cookbook.md](references/jql-cookbook.md) | 50 ready-to-run queries organized by role (developers, scrum masters, POs/managers, power users, admins) | Building filters, automation rules, sprint reviews | +| [references/jql-history-and-dates.md](references/jql-history-and-dates.md) | WAS/CHANGED predicate walkthrough, relative-date expression tables, saved-filter composition and naming conventions | History queries, date math, or designing reusable saved filters | -## References +## Available Scripts -- [scripts/jira](scripts/jira) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. -- [Jira REST API v3 docs](https://developer.atlassian.com/cloud/jira/platform/rest/v3/) — Official API reference. -- [API Token Management](https://id.atlassian.com/manage/api-tokens) — Generate and revoke tokens. +| Script | Purpose | Invocation | +|---|---|---| +| `scripts/jira` | The CLI this skill drives: `me`, `list`, `view`, `projects`, `create`, `comment`, `count`, `transitions`, `transition` — all with `--json`/`--dry-run`, lazy auth, offset-pagination fetches above 50 results, parsed API error messages, and 429 Retry-After surfacing. Run it for every Jira data question above. | `scripts/jira list --project PROJ --json` | +| `scripts/test_jira.py` | Offline pytest/unittest suite covering help text, argument errors, dry-run plans, pagination loops, error envelopes, and transition resolution logic — zero network. Run after modifying `scripts/jira`. | `.venv/bin/python3 -m pytest -p no:cacheprovider --strict-markers scripts/test_jira.py` | -## When not to use +## Prerequisites -Do not use this skill for GitHub or GitLab issue tracking (each platform has its own tooling), for Jira site administration such as project permissions, workflow schemes, or user management, or for writing application code against the Jira REST API — see the Atlassian developer docs for integration development instead. +- Python 3.8+ with `requests` (stdlib otherwise); invoke as `python3 scripts/jira ...` if not executable directly +- `JIRA_EMAIL` + `JIRA_API_TOKEN` exported for any non-dry-run command (token from https://id.atlassian.com/manage/api-tokens); `JIRA_SERVER` defaults to `https://your-domain.atlassian.net` +- `jq` recommended for `--json` post-processing + +## Limitations + +- Targets Jira **Cloud** REST v3; Data Center sites authenticate differently (Bearer PAT) and expose older API surfaces +- The classic search endpoint this CLI uses is deprecated upstream; expect eventual forced migration to `/search/jql` token paging +- Rich-text creation beyond plain paragraphs requires hand-built ADF JSON +- No auto-retry on 429; loops over many writes should sleep between calls diff --git a/jira/evals/evals.json b/jira/evals/evals.json new file mode 100644 index 0000000..c937c49 --- /dev/null +++ b/jira/evals/evals.json @@ -0,0 +1,71 @@ +{ + "schema_version": 1, + "skill_name": "jira", + "evals": [ + { + "id": "search-project-issues-readonly", + "prompt": "What's currently open in the PROJ project? Show me the newest tickets first.", + "expected_output": "Scenario: read-only issue search. The agent confirms JIRA_EMAIL/JIRA_API_TOKEN/JIRA_SERVER are set, then runs `jira list --project PROJ --json` (the --project shortcut builds `project=PROJ ORDER BY created DESC`). Results are presented from actual CLI output; for more than 50 matches the CLI follows offset pagination automatically. No write commands are invoked.", + "assertions": [ + "jira list is used with --project PROJ for the search", + "--json output is requested when results feed further processing", + "No create, comment, or transition commands run for this read-only request", + "Results come from real command output rather than invented tickets" + ] + }, + { + "id": "stalled-sprint-bulk-close-pipeline", + "prompt": "Find sprint work that hasn't been touched in two weeks and close out whatever is actually finished. Set a resolution when you close them.", + "expected_output": "Scenario: multi-step write pipeline. The agent runs `jira list --jql 'sprint IN openSprints() AND updated < -14d AND resolution = Unresolved' --json`, pipes keys through jq (`jq -r '.issues[].key'`), inspects candidates with `jira view KEY`, discovers valid transitions per issue with `jira transitions KEY` (transition IDs are workflow- and status-specific, so names alone are unreliable), then applies `jira transition KEY --to Done --resolution Done`. It previews writes with --dry-run first and reports per-ticket outcomes.", + "assertions": [ + "The JQL uses openSprints() with an staleness bound such as updated < -14d and excludes resolved work", + "jira transitions is used to discover valid transition IDs before transitioning", + "jira transition carries --resolution so screen-required resolution fields do not fail", + "Each ticket's outcome is reported rather than assumed" + ] + }, + { + "id": "transition-id-gotcha", + "prompt": "Move PROJ-412 to Done.", + "expected_output": "Scenario: gotcha-aware transition. The agent knows transition IDs differ per workflow/status and that Done-style transitions often require a resolution field on their screen. It runs `jira transitions PROJ-412` to list available IDs, picks the one whose target status category is completed, and runs `jira transition PROJ-412 --to --resolution Done`. If the API returns 400 naming a missing field (errors.resolution), it retries including that field rather than giving up or claiming success.", + "assertions": [ + "Available transitions are listed before applying one", + "The transition call includes --resolution to satisfy screen requirements", + "A 400 error naming a missing field is handled by supplying that field", + "Success is confirmed from command output (204-class response), not fabricated" + ] + }, + { + "id": "count-before-deep-dive", + "prompt": "How many unresolved bugs do we have across the org right now?", + "expected_output": "Scenario: fast aggregate question. JQL has no COUNT aggregation, but the CLI exposes the approximate-count endpoint: the agent runs `jira count --jql 'issuetype = Bug AND resolution = Unresolved' --json` and reads `.count` from the output. It does not fetch hundreds of issues with `jira list --max 1000` just to count them, though it may mention that list would page through rows if details were needed.", + "assertions": [ + "jira count is used instead of fetching all matching issues", + "The JQL expresses bug + unresolved constraints without aggregation syntax", + "The answer comes from the .count field of the JSON output", + "No attempt is made to use COUNT/SUM inside JQL itself" + ] + }, + { + "id": "github-issue-not-jira", + "prompt": "Can you open a GitHub issue for this crash on our repo?", + "expected_output": "Scenario: should-not-trigger. The request targets GitHub issue tracking, which this skill explicitly does not cover. The agent does not load jira or invoke its CLI; it routes the request to GitHub tooling (e.g., gh issue create) instead, noting that jira handles Atlassian Jira sites only.", + "assertions": [ + "The jira skill is not loaded or executed for a GitHub-issue request", + "A GitHub-native route such as the gh CLI is suggested", + "No JQL is written or JIRA_* env vars requested" + ] + }, + { + "id": "auth-setup-and-token-expiry", + "prompt": "Set up Jira access for me so you can start triaging my tickets, and explain what credentials you need.", + "expected_output": "Scenario: auth setup guidance. The agent explains Jira Cloud auth: an API token created at id.atlassian.com/manage/api-tokens used over HTTP Basic auth as base64(email:token) via JIRA_EMAIL and JIRA_API_TOKEN env vars plus JIRA_SERVER; passwords are deprecated and tokens now expire after at most one year. It notes that Personal Access Tokens (Bearer) are Data Center only, not Cloud. It verifies connectivity read-only with `jira me`, and never asks for or echoes the token value itself.", + "assertions": [ + "Basic auth with email + API token is described as the Cloud method", + "JIRA_EMAIL, JIRA_API_TOKEN, and JIRA_SERVER env vars are named", + "PAT bearer tokens are identified as Data Center-only, not Cloud", + "Verification proceeds via jira me without exposing the secret" + ] + } + ] +} From af9ca0f41b311b9816d045a371ff6d592a5a5a8a Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 06:10:28 -0400 Subject: [PATCH 11/40] chore(catalog): sync root README blurb and regenerate catalogs Root README jira blurb now mentions count matches and the expanded JQL reference set, matching the updated frontmatter description. Regenerated .claude-plugin/marketplace.json and llms.txt (description embed). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/marketplace.json | 2 +- README.md | 2 +- llms.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index d1ba47a..fae3325 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -579,7 +579,7 @@ "./jira" ], "strict": false, - "description": "Interact with Atlassian Jira from the terminal: search issues with JQL, view details, create issues, add comments, list projects, and transition status. Includes a full JQL language reference (functions, operators, history queries, performance tuning). Use when the user mentions Jira, a ticket key (e.g. PROJ-123), asks about issues, bugs, tasks, projects, or sprint work, or needs to write, debug, or optimize JQL queries. Do not use for GitHub or GitLab issue tracking, Jira site administration, or generic ticketing systems." + "description": "Interact with Atlassian Jira from the terminal: search issues with JQL, view details, create issues, add comments, count matches with fast approximate-count, list projects, discover valid transitions, and change status. Includes a full JQL language reference (functions, operators, history predicates, date expressions, saved filters, performance tuning) plus REST auth/pagination guidance. Use when the user mentions Jira, a ticket key (e.g. PROJ-123), asks about issues, bugs, tasks, projects, or sprint work, or needs to write, debug, or optimize JQL queries. Do not use for GitHub or GitLab issue tracking, Jira site administration, or generic ticketing systems." }, { "name": "kanban-guru", diff --git a/README.md b/README.md index c355d36..01cb805 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ Jellyfin media server from the terminal. Check server info, browse recently adde ### [jira](jira/SKILL.md) -Atlassian Jira from the terminal. Search issues with JQL, view details, create issues, add comments, list projects, and transition status. API token from id.atlassian.com. +Atlassian Jira from the terminal. Search issues with JQL, view details, create issues, add comments, count matches, list projects, and transition status. Includes a full JQL reference (functions, history predicates, date expressions) plus REST auth/pagination guidance. API token from id.atlassian.com. ### [kanban-guru](kanban-guru/SKILL.md) diff --git a/llms.txt b/llms.txt index 3157664..d50f9e4 100644 --- a/llms.txt +++ b/llms.txt @@ -65,7 +65,7 @@ - [implementation-planning](implementation-planning/SKILL.md): Plan the implementation of an approved requirement or specification: produce an executable, dependency-aware delivery plan covering work breakdown, dependency mapping, critical path, ownership, parallelism and sequencing, rollout strategy, rollback and recovery paths, and verification against the original requirement. Supports cross-team, cross-repository, migration, and staged-rollout scenarios. Do not use for pre-approval discovery or needs-finding, authoring a specification from scratch, coding or implementation, the neckbeard issue-to-PR delivery flow itself, or any work whose prerequisite decision has not been approved — planning unapproved work is an explicit stop condition. - [incident-learning](incident-learning/SKILL.md): Convert operational incident and near-miss evidence into durable product, engineering, test, evaluation, and governance improvements with verified closure. Separate observed facts from causal hypotheses and unresolved uncertainty; map follow-up work across code, tests, skills, operations, product, and governance; track ownership, verification, and closure for every finding. Do not use to assign blame or produce a generic postmortem template; do not close learning because tickets were created — require evidence the intended change occurred. - [jellyfin](jellyfin/SKILL.md): Query your Jellyfin media server from the terminal — recently added media, search, item details, next-up episodes, library browsing, server info, and stats. Use when the user asks about Jellyfin, media server, movies, TV shows, next episodes, or their media library. -- [jira](jira/SKILL.md): Interact with Atlassian Jira from the terminal: search issues with JQL, view details, create issues, add comments, list projects, and transition status. Includes a full JQL language reference (functions, operators, history queries, performance tuning). Use when the user mentions Jira, a ticket key (e.g. PROJ-123), asks about issues, bugs, tasks, projects, or sprint work, or needs to write, debug, or optimize JQL queries. Do not use for GitHub or GitLab issue tracking, Jira site administration, or generic ticketing systems. +- [jira](jira/SKILL.md): Interact with Atlassian Jira from the terminal: search issues with JQL, view details, create issues, add comments, count matches with fast approximate-count, list projects, discover valid transitions, and change status. Includes a full JQL language reference (functions, operators, history predicates, date expressions, saved filters, performance tuning) plus REST auth/pagination guidance. Use when the user mentions Jira, a ticket key (e.g. PROJ-123), asks about issues, bugs, tasks, projects, or sprint work, or needs to write, debug, or optimize JQL queries. Do not use for GitHub or GitLab issue tracking, Jira site administration, or generic ticketing systems. - [kanban-guru](kanban-guru/SKILL.md): A virtual Kanban expert who can diagnose flow problems, design board configurations, set up multi-portfolio operating models, calibrate WIP limits, establish service level expectations, and guide Scrum-to-Kanban transitions. Load this when your team is struggling with throughput, cycle times are unpredictable, multiple stakeholders compete for the same engineers, or you're wondering if Kanban is right for you. - [kubernetes](kubernetes/SKILL.md): Operate, troubleshoot, secure, upgrade, and automate Kubernetes clusters and workloads safely across upstream Kubernetes, k3s, RKE2, MicroK8s, k0s, Talos, OpenShift/OKD, kind, Minikube, Rancher-managed clusters, EKS, AKS, and GKE. Use when a task involves kubectl, Kubernetes APIs, Pods, Deployments, StatefulSets, Services, Ingress or Gateway API, CRDs, RBAC, NetworkPolicy, storage, scheduling, autoscaling, cluster lifecycle, or the bundled agent-first k8s-cli. - [langchain](langchain/SKILL.md): Expert skill for building LLM applications with LangChain — LCEL chains, RAG pipelines, agent orchestration, LangGraph integration, LangSmith observability, and production deployment via LangServe. Use when working with LangChain or comparing LLM application frameworks. From 3e0453b1183463f4c60f61ed2dc8f6c976d3b583 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 06:50:46 -0400 Subject: [PATCH 12/40] fix(jira): correct ConnectionError hint to cite JIRA_SERVER env var The die() message on connection failure told users to 'use --server', but no such flag exists; JIRA_SERVER is the only endpoint override. Replace the false clause with a hint that matches actual behavior. Flagship scrutiny follow-up (non-blocking issue 1 from thicken-jira-flagship validation). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- jira/scripts/jira | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jira/scripts/jira b/jira/scripts/jira index 1c09ed5..939c546 100755 --- a/jira/scripts/jira +++ b/jira/scripts/jira @@ -121,7 +121,7 @@ class JiraClient: ) except requests.ConnectionError as e: die(f"Cannot connect to {self.server}: {e}\n" - f" Check JIRA_SERVER or use --server") + f" Check connectivity, or set JIRA_SERVER env var to change the endpoint") if resp.status_code == 401: die("Auth failed (401). Check your JIRA_EMAIL and JIRA_API_TOKEN.") From ce953929156336ba9e4d6ae9ff6c94f026324995 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 08:43:13 -0400 Subject: [PATCH 13/40] docs(openlibrary): add API references for key graph, search, ISBN, recipes Four cited reference files distilled from live-verified research against openlibrary.org developer docs: OL...M/W/A key graph with merge-stub behavior, search query syntax and error model, ISBN 302 redirect resolution and covers host rules, plus worked curl/jq pipelines with a symptom-indexed gotcha table. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .../references/api-overview-and-key-graph.md | 175 +++++++++++++++ .../references/books-isbn-and-covers.md | 189 ++++++++++++++++ openlibrary/references/recipes-and-gotchas.md | 143 ++++++++++++ openlibrary/references/search-api-guide.md | 210 ++++++++++++++++++ 4 files changed, 717 insertions(+) create mode 100644 openlibrary/references/api-overview-and-key-graph.md create mode 100644 openlibrary/references/books-isbn-and-covers.md create mode 100644 openlibrary/references/recipes-and-gotchas.md create mode 100644 openlibrary/references/search-api-guide.md diff --git a/openlibrary/references/api-overview-and-key-graph.md b/openlibrary/references/api-overview-and-key-graph.md new file mode 100644 index 0000000..80074ac --- /dev/null +++ b/openlibrary/references/api-overview-and-key-graph.md @@ -0,0 +1,175 @@ +# Open Library API Overview and the OLID Key Graph + +Open Library (an Internet Archive project) exposes its catalog of 50M+ records through +public, keyless HTTP APIs. This file covers the access model, etiquette, and the +record-key graph that everything else builds on. Companion files: the Search API +([search-api-guide.md](search-api-guide.md)), ISBN/Books/Covers endpoints +([books-isbn-and-covers.md](books-isbn-and-covers.md)), and worked recipes +([recipes-and-gotchas.md](recipes-and-gotchas.md)). + +## Access model: no key, three hosts, open CORS + +Reads require **no API key and no registration**. Three hostnames matter: + +| Host | Serves | +|------|--------| +| `openlibrary.org` | Metadata JSON: search, works, editions, authors, ratings | +| `covers.openlibrary.org` | Cover images and author photos (separate service) | +| `archive.org` | Ebook/scan content and bulk dumps (redirect targets) | + +Metadata endpoints answer `application/json`, support `GET` + `OPTIONS`, and send +`access-control-allow-origin: *`, so browser-side fetches work directly. + +Optional identification: put your app name and contact email in the User-Agent, +e.g. `User-Agent: MyLibraryApp (contact@example.org)`. Identified traffic gets a 3x +rate allowance (see below) and gives staff someone to contact before blocking you. +There is no token, secret, or account step anywhere in the read surface. + +## Rate limits and etiquette (official guidance) + +From the official APIs index page ([developers/api](https://openlibrary.org/developers/api)): + +- Anonymous clients: **1 request/second**. Identified clients (User-Agent carrying app + name + contact email/phone): **3 requests/second** ("identified requests will enjoy a + 3x request limit"). +- No `X-Rate-Limit-*` or `Retry-After` headers are sent today; the limit is advisory + policy, not header-signaled. Exceeding it politely means sleeping, because there is + nothing to read out of the response. +- Explicitly discouraged: HTML scraping (use the API endpoints), spreading traffic + across 5+ IPs, bulk harvesting, hundreds of single-book GETs where one + `/search.json` batch would do, or using Open Library as a backend for a + high-traffic service. Violations bring "aggressive rate limiting or blocking". +- For bulk data use the monthly dumps instead + ([developers/dumps](https://openlibrary.org/developers/dumps)): editions ~9.2G, + works ~2.9G, authors ~0.5G, ratings/reading-log much smaller. Dump lines are + `type, key, revision, last_modified, JSON`. + +The covers service has its own harder limit: non-ID/non-OLID cover lookups are capped +at **100 requests/IP per 5 minutes**, then **403 Forbidden** +([dev/docs/api/covers](https://openlibrary.org/dev/docs/api/covers)). + +## The key graph: OLIDs and the letter-suffix type system + +Every catalog entity has a stable-shaped identifier called an OLID whose **final +letter encodes the type**: + +| Suffix | Type | Canonical JSON path | Example | +|--------|------|--------------------|---------| +| `M` | Edition (a physical/digital publication) | `/books/OL34854896M.json` | `OL34854896M` | +| `W` | Work (the abstract creative work) | `/works/OL45804W.json` | `OL45804W` | +| `A` | Author | `/authors/OL23919A.json` | `OL23919A` | + +Two surface forms appear in payloads and docs alike: bare OLIDs (`OL45804W`) and +path keys (`/works/OL45804W`). Search results return path keys for works +(`/works/OL…W`) but bare keys in author search (`OL…A`). Parse defensively: strip or +add the collection prefix by inspecting the suffix letter rather than assuming one form. + +Graph wiring, verified against live records: + +- **Edition → work**: `edition.works` is a list of key refs: + `"works": [{"key": ""}]`. +- **Work → authors**: double-nested, with a role node: + `"authors": [{"author": {"key": ""}, "type": {"key": ""}}]`. + Read `work.authors[].author.key`, never `work.authors[].key`. +- **Edition → authors**: flat single nesting instead: + `"authors": [{"key": ""}]`. The two collections disagree — handle both. +- **Work → editions**: not embedded; enumerate via `/works/OL…W/editions.json` + (see [books-isbn-and-covers.md](books-isbn-and-covers.md)). +- Common record furniture: `type.key` (`/type/edition`, `/type/work`, + `/type/author`), integer-array `covers` / `photos`, `created`/`last_modified` + timestamps, `revision`/`latest_revision` integers. + +## Wrong key type: cross-collection 301 reroutes + +Requesting a key under the wrong collection is forgiven with a redirect to the right +one (live-verified): + +``` +GET https://openlibrary.org/books/OL23919A.json # author OLID under /books +HTTP/2 301 +location: https://openlibrary.org/authors/OL23919A.json + +GET https://openlibrary.org/works/OL123M.json # edition OLID under /works +HTTP/2 301 +location: https://openlibrary.org/books/OL123M.json +``` + +So a client that follows redirects survives suffix/collection mismatches automatically +(`requests` follows by default; `curl` needs `-L`). The practical symptom of *not* +following: your parser sees an HTML 301 page instead of JSON. + +## Missing keys and the merge problem: redirect stubs inside HTTP 200 + +Truly nonexistent keys 404 with a JSON body: + +``` +GET https://openlibrary.org/books/OL999999999M.json +HTTP/2 404 +{"error": "notfound", "key": ""} +``` + +But Open Library is a wiki: duplicates are merged and spam is deleted, and **merged +keys do not 3xx**. A merged-away key keeps serving `HTTP 200` with a stub record +(live-verified on a real merge found via `/recentchanges/merge-works.json`): + +```json +{"location": "", + "type": {"key": ""}, + "latest_revision": 4, "revision": 4, ...} +``` + +The old key `/works/OL24776360W` had just been merged into `/works/OL14868272W`, yet +the JSON endpoint returned **200**, not 302. Clients must detect +`payload["type"]["key"] == "/type/redirect"` in successful responses and re-fetch +`payload["location"]` themselves. (The HTML page for the same key does 302; only JSON +gives you the stub.) The bundled CLI performs this follow-up automatically. + +Related identity hazards: + +- **Deleted-and-reassigned keys**: an OLID freed by deletion can be reissued for an + unrelated book (observed live). Never treat an OLID as long-term identity for + caching; pair it with title or ISBN. +- Recent merges are observable at `/recentchanges/merge-works.json?limit=N` + (`data.master`, `data.duplicates[]`) if you need to audit drift. + +## Text-valued fields nest `{type, value}` objects + +Free-text fields (`bio` on authors; `description`, `notes`, `first_sentence` on +editions/works) arrive in **two shapes** depending on record age: + +```json +"bio": {"type": "/type/text", "value": "Joanne \"Jo\" Murray, OBE ..."} +``` + +Older records carry a plain string instead. Always branch: if dict, take `["value"]`; +if str, use as-is. The bundled CLI unwraps these automatically. + +Photo/cover ID arrays mix in `-1` placeholders meaning "no image" +(e.g. `"photos": [5543033, -1]`): filter out negatives before building image URLs. + +## Author URL quirk: `.json` placement matters + +Bare author URLs HTML-redirect to slugged pages +(`/authors/OL23919A` → `/authors/OL23919A/J._K._Rowling`). Appending `.json` after +the slug (`/authors/OL23919A/J._K._Rowling.json`) does **not** serve JSON — append +it to the bare key: `/authors/.json` +([dev/docs/api/authors](https://openlibrary.org/dev/docs/api/authors)). + +## Writes exist but are outside the keyless surface + +Authenticated writes exist (`POST /account/login.json` with Internet Archive S3 keys +returns a session cookie; `PUT` resource JSON updates records), but the RESTful doc +states this is effectively internal: PUT/POST without permission returns **403**, and +the docs warn the API "works only from the localhost" +([dev/docs/restful_api](https://openlibrary.org/dev/docs/restful_api)). Plan around +reads only; expect every write path to require credentials this skill deliberately +does not handle. + +## Sources + +- https://openlibrary.org/developers/api — API index; rate limits (1/s anonymous, 3/s identified), User-Agent identification format, bulk-access policy +- https://openlibrary.org/dev/docs/api/authors — Authors API; slug/`.json`-placement rule +- https://openlibrary.org/dev/docs/api/books — Books/Editions/Works API; record shapes +- https://openlibrary.org/dev/docs/restful_api — write/login mechanics, status codes, localhost-only caveat +- https://openlibrary.org/developers/dumps — monthly dump catalog and line format +- Live read-only probes against openlibrary.org (2026-08-26): 301 cross-collection reroutes, 404 body shape, merge-stub 200 responses, `{type,value}` text nesting diff --git a/openlibrary/references/books-isbn-and-covers.md b/openlibrary/references/books-isbn-and-covers.md new file mode 100644 index 0000000..05d975a --- /dev/null +++ b/openlibrary/references/books-isbn-and-covers.md @@ -0,0 +1,189 @@ +# ISBN Lookup, Editions, Ratings, and the Covers Host + +This file covers everything keyed to a specific book: identifier-style endpoints +(ISBN/LCCN/OCLC/OLID), their 302-redirect resolution, the legacy Books API view +models, edition enumeration under a work, community aggregates, and image URLs on +the separate covers host. Key-graph fundamentals are in +[api-overview-and-key-graph.md](api-overview-and-key-graph.md). + +## Identifier endpoints answer 302, not JSON + +`/isbn/.json`, `/lccn/.json`, and `/oclc/.json` do not serve the +record directly. They **redirect (HTTP 302) to the canonical edition JSON** at +`https://openlibrary.org/books/.json`. Live transcripts: + +``` +$ curl -is 'https://openlibrary.org/isbn/9780451524935.json' +HTTP/2 302 +location: https://openlibrary.org/books/OL34854896M.json + +$ curl -is 'https://openlibrary.org/lccn/93005405.json' +HTTP/2 302 +location: https://openlibrary.org/books/OL1397864M.json + +$ curl -is 'https://openlibrary.org/oclc/28419896.json' +HTTP/2 302 +location: https://openlibrary.org/books/OL1397864M.json +``` + +Practical consequences: + +- **Follow redirects or get nothing useful.** Python `requests.get()` follows by + default; `curl` needs `-L`; a raw HTTP client that ignores Location sees only an + HTML 302 page. +- After following, you land on `200 application/json`, the raw edition record — + including its `key` (`/books/OL34854896M`), which is how you discover which edition + an ISBN resolved to. +- The official Books API doc documents the HTML flavor of this redirect + (`/isbn/9780140328721` → `/books/OL7353617M`) and notes `.json` may be appended to + such page URLs; the 302-on-JSON behavior itself is established by live probes. +- An ISBN matching no record eventually 404s after redirects with body + `{"error": "notfound", ...}`. + +Direct edition keys skip the redirect entirely: `/books/OL7440033M.json` → `200`. + +## Raw edition records vs the legacy Books API view models + +**Two different representations exist for the same book.** Know which one you have: + +1. **Raw record** (what identifier endpoints redirect to): flat bibliographic fields, + string arrays for `publishers`, integer-array `covers`, path-shaped `key`, + `type: {"key": ""}`, key-ref lists `works[]`/`authors[]`. + Observed fields include: `title`, `subtitle`, `isbn_10`, `isbn_13`, `lccn`, + `oclc_numbers`, `publishers`, `publish_date`, `publish_places`, `number_of_pages`, + `pagination`, `languages` (`[{"key": ""}]`), `covers`, `works`, + `authors`, `description`, `notes`, `first_sentence`, `table_of_contents`, + `identifiers`, `lc_classifications`, `dewey_decimal_class`, `weight`, + `physical_format`, `edition_name`, `copyright_date`, `ocaid` (the archive.org + scan id), `source_records`. + +2. **Legacy view models** via `GET /api/books?bibkeys=ISBN:&format=json&jscmd=`: + + | jscmd | Shape | + |-------|-------| + | *(absent)* / `viewapi` | Tiny object: `bib_key`, `info_url`, `preview` (`noview`/`full`/`restricted`), `preview_url`, `thumbnail_url`. Use `preview` to test readability; `preview_url` is always present even when unreadable. | + | `data` | Friendly model: `url`, `title`, `authors[{name,url}]`, `publishers[{name}]` (objects!), grouped `identifiers{isbn_10,isbn_13,lccn,oclc,goodreads,...}`, `classifications{lc_classifications,dewey_decimal_class}`, `subjects[]`, ready-made `cover{small,medium,large}` URLs, `ebooks[]`, `excerpts[]`, `links[]`. Docs recommend this as the stable format. | + | `details` | viewapi fields plus a nested raw record under `details`; docs advise using `jscmd=data` instead. | + + `bibkeys` is comma-separated with prefixes `ISBN:`, `OCLC:`, `LCCN:`, `OLID:`; + both ISBN-10 and ISBN-13 accepted. `format=json` required for machine use (default + is JSONP-style JavaScript). The whole `/api/books` endpoint is flagged legacy + ("may be phased out"); prefer search + direct record fetches for new work. + +Live contrast for ISBN 9780451524935: + +```json +// jscmd=data +{"ISBN:9780451524935": { + "title": "Nineteen Eighty-Four", + "url": "http://openlibrary.org/books/OL34854896M/Nineteen_Eighty-Four", + "publishers": [{"name": "Signet Classics"}], + "cover": {"small": "https://covers.openlibrary.org/b/id/12054527-S.jpg", + "medium": "https://covers.openlibrary.org/b/id/12054527-M.jpg", + "large": "https://covers.openlibrary.org/b/id/12054527-L.jpg"}, ...}} + +// default viewapi +{"ISBN:9780451524935": {"bib_key": "ISBN:9780451524935", + "info_url": "http://openlibrary.org/books/OL34854896M/Nineteen_Eighty-Four", + "preview": "restricted", "preview_url": "https://archive.org/details/nineteeneightyfo0000orwe_g7l1", + "thumbnail_url": "https://covers.openlibrary.org/b/id/12054527-S.jpg"}} +``` + +## Listing every edition of a work + +``` +GET https://openlibrary.org/works//editions.json[?limit=N&offset=M] +``` + +Response envelope: + +```json +{"size": 6, + "links": {"self": "/works/OL81699W/editions.json?limit=2", + "work": "/works/OL81699W", + "next": "/works/OL81699W/editions.json?limit=2&offset=2"}, + "entries": [ {full edition records}, ... ]} +``` + +- `entries[]` holds complete edition records (same shape as single-edition JSON). +- Pagination via `limit`/`offset` verified live; while more pages remain, + `links.next` carries the prebuilt next URL — follow it rather than recomputing. +- The same `size`/`links.self`/`entries` structure serves author works at + `/authors/OL…A/works.json` (default page size 50 there, `limit` up to 1000 per the + Authors API doc). + +## Community aggregates on works + +Public, keyless GETs on any work key: + +**Ratings** — `GET /works//ratings.json` + +```json +{"summary": {"average": 3.966386554621849, "count": 119, "sortable": 3.7388955319679584}, + "counts": {"1": 10, "2": 6, "3": 18, "4": 29, "5": 56}} +``` + +Note the string keys `"1"`–`"5"` in `counts`, and that `summary.average` is absent +when nobody has rated. + +**Bookshelves** — `GET /works//bookshelves.json` + +```json +{"counts": {"want_to_read": 1191, "currently_reading": 97, + "already_read": 189, "stopped_reading": 0}} +``` + +Shelf names are literal: `want_to_read`, `currently_reading`, `already_read`, +`stopped_reading`. + +A per-work `/readinglog.json` route is **not part of the documented public API** (404 +in testing); reading-log data flows through the My Books API +([dev/docs/api/mybooks](https://openlibrary.org/dev/docs/api/mybooks)) — e.g. +`/people//books/want-to-read.json` — or monthly dumps. + +## Covers and author photos: a separate host with its own rules + +All images live on `covers.openlibrary.org`, never on `openlibrary.org`: + +``` +Book covers: https://covers.openlibrary.org/b/{id|olid|isbn|lccn|oclc}/-{S|M|L}.jpg +Author photos: https://covers.openlibrary.org/a/{id|olid}/-{S|M|L}.jpg +Cover metadata: append .json → https://covers.openlibrary.org/b/id/12547191.json +``` + +Sizes: S = thumbnail, M = details-page size, L = large. The same cover is reachable +by any of its keys (`/b/id/240727-S.jpg`, `/b/olid/OL7440033M-S.jpg`, +`/b/isbn/0385472579-S.jpg` all hit one image). + +Behaviors verified live: + +- Cover URLs commonly **302 into archive.org zip shards** + (`location: https://archive.org/download/s_covers_0012/s_covers_0012_05.zip/0012054527-S.jpg`); + some lookups answer 200 directly. Follow redirects regardless. +- A missing cover returns a **blank placeholder image with HTTP 200** unless you add + `?default=false`, which yields a proper **404**: + ``` + $ curl -sI '.../b/id/999999999999-M.jpg' → HTTP 200 (blank) + $ curl -sI '.../b/id/999999999999-M.jpg?default=false' → HTTP 404 + ``` + Always pass `default=false` when you need existence semantics. +- Rate limit: non-ID/non-OLID lookups (ISBN/LCCN/OCLC forms) are capped at + **100 requests/IP per 5 minutes**, then **403 Forbidden**; ID- and OLID-based + lookups are exempt ([dev/docs/api/covers](https://openlibrary.org/dev/docs/api/covers)). + Resolve to cover IDs first if you will fetch many images. +- Cover metadata JSON includes `width`, `height`, `olid`, shard filenames — handy for + checking existence before downloading. +- Author photo IDs come from the author record's `photos` array (filter out `-1` + placeholders) and map to `/a/id/-.jpg`. +- Etiquette: don't crawl covers; bulk archives live on archive.org items + (`s_covers_*`, `m_covers_*`, `l_covers_*`). A courtesy link back to Open Library is + appreciated when displaying covers. + +## Sources + +- https://openlibrary.org/dev/docs/api/books — identifier endpoints, redirect behavior, legacy /api/books and jscmd modes +- https://openlibrary.org/dev/docs/api/covers — URL patterns, sizes, default=false, rate limits +- https://openlibrary.org/dev/docs/api/authors — /authors/…/works.json pagination +- https://openlibrary.org/dev/docs/api/mybooks — documented reading-log surface +- https://openlibrary.org/developers/api — etiquette governing image/metadata fetching +- Live read-only probes against openlibrary.org and covers.openlibrary.org (2026-08-26): 302 Locations, editions.json envelope, ratings/bookshelves shapes, blank-vs-404 cover behavior, archive.org shard redirects diff --git a/openlibrary/references/recipes-and-gotchas.md b/openlibrary/references/recipes-and-gotchas.md new file mode 100644 index 0000000..60cf8da --- /dev/null +++ b/openlibrary/references/recipes-and-gotchas.md @@ -0,0 +1,143 @@ +# Worked Recipes and Gotcha Compendium + +Multi-step pipelines against the Open Library API using `curl`/`jq` (or the bundled +CLI), followed by a symptom-indexed gotcha table. Fundamentals: +[api-overview-and-key-graph.md](api-overview-and-key-graph.md) (key graph, merges), +[search-api-guide.md](search-api-guide.md) (search params/errors), +[books-isbn-and-covers.md](books-isbn-and-covers.md) (ISBN redirects, covers). + +## Recipe 1: ISBN → edition → work → full author bio + +The canonical resolution chain. Each hop uses a different key type — this is where +OL…M/W/A confusion bites: + +```bash +ISBN=9780451524935 + +# 1. ISBN resolves via 302 to an OL…M edition record (requests follows by default) +curl -sL "https://openlibrary.org/isbn/$ISBN.json" > edition.json + +# 2. Pull the work key out of the edition (path form /works/OL…W) +WORK=$(jq -r '.works[0].key' edition.json) # e.g. /works/OL166894W + +# 3. Fetch the work for description + subjects; note double-nested author refs +curl -s "https://openlibrary.org${WORK}.json" > work.json +AUTHOR=$(jq -r '.authors[0].author.key' work.json) # /authors/OL23919A + +# 4. Author record; bio may be {type,value}-wrapped or a plain string +curl -s "https://openlibrary.org${AUTHOR}.json" | jq -r ' + if (.bio | type) == "object" then .bio.value else .bio end' +``` + +Failure modes at each hop: step 1 needs `-L` in curl or you parse an HTML 302 page; +step 3's `.authors[0].author.key` is wrong on *edition* records (flat `.authors[0].key` +there); step 4 crashes naive parsers when `bio` is an object. + +## Recipe 2: search → filter to readable ebooks → fetch editions of the top hit + +```bash +# availability requires ia in fields= (silently absent otherwise!) +curl -s 'https://openlibrary.org/search.json' \ + --data-urlencode 'q=title:"moby dick"' \ + --data-urlencode 'fields=key,title,author_name,first_publish_year,ia,ebook_access,availability' \ + --data-urlencode 'sort=editions' --data-urlencode 'limit=5' > hits.json + +jq -r '.docs[] | select(.ebook_access == "public") | .key' hits.json | head -1 > workkey +WORK=$(cat workkey) + +# enumerate all editions with pagination links +NEXT="/works/${WORK##*/}/editions.json?limit=50" +while [ "$NEXT" != "null" ] && [ -n "$NEXT" ]; do + curl -s "https://openlibrary.org$NEXT" | jq '.entries[] | {key, isbn_13, publish_date}' + NEXT=$(curl -s "https://openlibrary.org$NEXT" | jq -r '.links.next // empty') +done +``` + +## Recipe 3: cover-image URL assembly without blank-image surprises + +Cover IDs come from records (`covers:[12054527]`) or search results +(`cover_edition_key`, `cover_i`). Build URLs on the covers host and demand real 404s: + +```bash +COVER_ID=$(jq -r '.covers[0] | select(. >= 0)' edition.json | head -1) +for size in S M L; do + url="https://covers.openlibrary.org/b/id/${COVER_ID}-${size}.jpg?default=false" + code=$(curl -s -o /dev/null -w '%{http_code}' -L "$url") + echo "$size $code" # 200 = exists; 404 = no cover at this size +done +``` + +Without `?default=false` every probe returns 200 (blank placeholder), so existence +checks silently lie. For batch image pulls, resolve to numeric cover IDs first — +ISBN-based lookups are rate-limited at 100 req/IP per 5 min, ID-based are exempt. + +## Recipe 4: author disambiguation via search-authors, then their top works + +```bash +curl -s 'https://openlibrary.org/search/authors.json?q=herbert&limit=5' \ + | jq '.docs[] | {name, key, birth_date, death_date, top_work, work_count}' +# pick the right bare OL…A key, then: +curl -s 'https://openlibrary.org/authors/OL3874685A/works.json?limit=10' \ + | jq '{size, works: [.entries[].title]}' +``` + +Author-search keys arrive **bare** (`OL…A`); book-search keys arrive as paths +(`/works/OL…W`). When assembling URLs from either, strip everything up to the final +slash first. + +## Recipe 5: community-signal ranking of a series' entries + +```bash +for W in $(curl -s 'https://openlibrary.org/search.json?q=series:dune&limit=8' \ + | jq -r '.docs[].key'); do + WID=${W##*/} + ratings=$(curl -s "https://openlibrary.org/works/$WID/ratings.json") + shelves=$(curl -s "https://openlibrary.org/works/$WID/bookshelves.json") + jq -n --arg w "$WID" --argjson r "$ratings" --argjson s "$shelves" \ + '{work: $w, avg: $r.summary.average, rated: $r.summary.count, + want_to_read: $s.counts.want_to_read}' + sleep 1 # anonymous budget is ~1 req/s; be polite +done +``` + +## Gotchas indexed by symptom + +| Symptom | Cause | Fix | +|---------|-------|-----| +| JSON parse error / HTML instead of data after an ISBN lookup | `/isbn/.json` answers **302** to `/books/OL…M.json`; client didn't follow | Follow redirects (`requests` default, `curl -L`) | +| `KeyError: 'author'` reading work authors | Work records double-nest: `authors[].author.key`; editions nest flat | Branch on collection or use a tolerant accessor | +| Bio/description arrives as dict, not string | Legacy `{type: "/type/text", value}` wrapper on older records | `v["value"] if isinstance(v, dict) else v` | +| Requested `availability` missing from search docs | `availability` requires `ia` in the same `fields=` list | `fields=key,title,ia,availability` | +| Search returned 0 results but no error | Malformed q parses loosely and returns HTTP 200 empty; empty/missing q too | Treat empties as results-not-errors; validate input client-side | +| `Internal Server Error` plain text from search | Invalid `sort=` enum (e.g. `bogus`) → HTTP 500 non-JSON | Validate sort choices before sending | +| 422 validation JSON from search | Non-integer `limit`, negative `offset` (FastAPI validation) | Clamp inputs client-side | +| Merged work key returns odd body with HTTP 200 | Wiki merges leave `{type:{key:"/type/redirect"}, location}` stubs, not 3xx | Detect redirect-type bodies and re-fetch `location` | +| Author OLID under `/books/` "fails" | Wrong collection for suffix letter → 301 reroute to correct one | Follow redirects, or normalize keys by suffix before fetching | +| Slug-URL `.json` gives HTML not JSON | `.json` must attach to the bare key (e.g. `/authors/.json`), never after a slug path (`/authors//Slug.json`) | Append `.json` directly to the key | +| Cover check says exists but image is blank | Missing covers return blank placeholder **with HTTP 200** | Add `?default=false` to get true 404s | +| Covers start returning 403 | Non-ID/non-OLID cover lookups cap at 100 req/IP per 5 min | Resolve to cover IDs (`/b/id/...`), which are exempt | +| Cover/image download stalls mid-pipeline | Cover URLs often 302 into archive.org zip shards | Follow redirects there too | +| Old cached OLID now serves a different book | Deleted keys get reassigned; OLIDs aren't long-term identity | Pair OLID with title/ISBN in caches | +| Rate-limited or blocked entirely | Anonymous budget ~1 req/s (3 identified); no Retry-After header exists | Send `User-Agent: AppName (email)`, cache, sleep ≥1s, batch via search.json | + +## Design rules for robust clients + +1. Always follow redirects everywhere; both metadata and images redirect routinely. +2. Normalize every key by its suffix letter (M/W/A) and rebuild canonical URLs; + accept both bare and path forms on input. +3. Handle `{type,value}` text wrapping centrally, once. +4. Never branch on HTTP status alone: merged-stub 200s, silent-empty 200s, and + non-JSON 500s all exist. Inspect bodies. +5. Identify your client via User-Agent email; sleep between bursts; prefer one + `/search.json` over hundreds of single-record GETs. +6. Cache by (OLID + title), not OLID alone. + +## Sources + +- https://openlibrary.org/dev/docs/api/books — identifier endpoints, view models, redirect semantics +- https://openlibrary.org/dev/docs/api/covers — cover URL patterns, default=false, rate limits +- https://openlibrary.org/dev/docs/api/search — fields=/sort/error semantics exercised in recipes +- https://openlibrary.org/dev/docs/api/authors — slug rule, works.json paging, batch-by-key trick +- https://openlibrary.org/developers/api — rate-limit etiquette encoded in recipe sleeps +- https://openlibrary.org/search/howto — field scopes and filters used in queries +- Live read-only probes (2026-08-26) validating each recipe's chain end-to-end diff --git a/openlibrary/references/search-api-guide.md b/openlibrary/references/search-api-guide.md new file mode 100644 index 0000000..52f1d91 --- /dev/null +++ b/openlibrary/references/search-api-guide.md @@ -0,0 +1,210 @@ +# Open Library Search API Guide + +`/search.json` is the primary read surface: a Solr-backed work index with offset +pagination, field-scoped queries, and server-side projection. This file documents the +full parameter surface, result schema, sibling search endpoints, and the error model +observed live. Key-graph basics (OL…M/W/A) live in +[api-overview-and-key-graph.md](api-overview-and-key-graph.md); ISBN/edition/covers +endpoints in [books-isbn-and-covers.md](books-isbn-and-covers.md). + +## Endpoint and parameters + +``` +GET https://openlibrary.org/search.json?q=& +``` + +| Parameter | Behavior | +|-----------|----------| +| `q` | Solr query string; supports field scopes and Lucene syntax (below). | +| `fields` | Comma-separated projection. `*` returns ~120 fields (docs warn it's "expensive, please use sparingly"). Special value `availability` adds an availability subdocument — **only if `ia` is also requested** (gotcha below). | +| `sort` | One of the sort keys below; default is relevance. Invalid values cause an HTTP 500 (error model below). | +| `limit` / `offset` | Offset pagination. `page`/`limit` also works (page starts at 1); if both `page` and `offset` are sent, **offset wins**. | +| `lang` | Two-letter ISO 639-1 preference — "influences but doesn't exclude" results. To *exclude*, use `language:` inside `q`. | +| `title`, `author` | Top-level scoped params equivalent to prefixing inside q (e.g. `/search.json?title=the+lord+of+the+rings`). | + +### Sort keys + +All of these returned HTTP 200 in live probes (`q=harry potter&limit=1`); +the authoritative enumeration lives in Open Library source +(`openlibrary/plugins/worksearch/schemes/works.py`): + +`new`, `old`, `rating asc`, `rating desc` (bare `rating` = desc), `editions`, `title`, +`scans`, `key` (sorts as a *string*, not numerically), `random`, `readinglog`, +`already_read`, `want_to_read`, `currently_reading`, `ebook_access`. + +The existing CLI exposes `--sort {editions,new,old,rating,title}` plus empty for +relevance — a safe subset. + +## Pagination model: offsets only, no tokens + +There is no cursor or token concept anywhere on this API — clients compute the next +offset from `numFound` and `start`: + +```json +{"numFound": 4045, "start": 0, "numFoundExact": true, + "num_found": 4045, "documentation_url": "...", "q": "harry potter", + "offset": null, "docs": [...]} +``` + +- `start` mirrors the effective zero-based offset of the first doc + (`page=3&limit=10` → `start: 20`). +- `numFoundExact: false` signals the count is approximate. +- No documented cap on `limit` or `offset`: live probes honored `limit=2000` and + `offset=11000`. The response simply clamps to the matched set. Stay modest anyway — + the etiquette policy forbids using OL as a bulk backend. + +## Result document schema + +Common `docs[]` fields: `key` (path form `/works/OL…W`), `title`, `author_name[]`, +`author_key[]`, `first_publish_year`, `edition_count`, `cover_edition_key`, +`cover_i`, `ia[]` (Internet Archive scan ids), `has_fulltext`, `public_scan_b`, +`language[]`, `subject[]`, `publisher[]`, `publish_year[]`, `isbn[]`, +`number_of_pages_median`, `ebook_access`, `ratings_average`, `ratings_count`, +`readinglog_count`, `seed[]`. + +The docs state the schema "is not guaranteed to be stable, but most common fields … +should be safe to depend on". Treat exotic fields as best-effort. + +## Query syntax: field scopes and filters + +Verified live prefixes: + +| Prefix | Example | Notes | +|--------|---------|-------| +| `title:` | `q=title:flammable` | 551 hits | +| `author:` | `q=author:solnit` | 129 hits | +| `subject:` | `q=subject:"tennis rules"` | fuzzy containment (AND), not exact phrase | +| `publisher:` | `q=publisher:harper` | 77,889 hits | +| `isbn:` | `q=isbn:9780451524935` | ISBN-10 and ISBN-13 both resolve to the same single work | +| `language:` | `q=language:fre` | excludes works without matching-language editions | + +Lucene extras from the official how-to ([search/howto](https://openlibrary.org/search/howto)): +ranges (`first_publish_year:[1200 TO 1400]`, `publish_year:[* TO 1800]`), +booleans `AND`/`OR`/`NOT`, negation `-subject_key:"apache_solr"`, +prefix wildcards `ddc:200*`, normalized exact keys (`subject_key:`, `person_key:`, +`place_key:`, `time_key:` — lowercase, spaces/slashes → underscores), +availability filter `ebook_access:` with values `no_ebook`, `printdisabled`, +`borrowable`, `public`, plus `has_fulltext:true`, `edition_count:N`, +`readinglog_count:[25 TO *]`. + +## The `fields=` projection and its availability gotcha + +Requesting fewer fields shrinks payloads dramatically. Live behavior: + +- `fields=key,title` returns exactly those keys per doc. +- **`availability` is silently omitted unless `ia` is also requested** — verified live: + - `fields=key,title,availability` → doc keys exactly `['key','title']` + - `fields=key,title,ia,availability` → full availability subdocument present + +With `ia` included, each doc gains: + +```json +"availability": { + "status": "borrow_available", + "is_readable": false, + "is_lendable": true, + "is_printdisabled": true, + "openlibrary_work": "OL82563W", + "openlibrary_edition": "OL61057835M", ... +} +``` + +`status` values include `borrow_available`, `borrow_unavailable`, `printdisabled`, +`open` (readable), and absent/`error` when no scan exists. + +Bonus expansion: `fields=key,title,editions` nests a mini-result-set under each work +(`numFound`/`start`/`docs[]` with edition fields); individual edition fields are +requested as `editions.key`, `editions.ebook_access`, `editions.language`; +`&editions.sort` overrides default boosting. + +## Author search: `/search/authors.json` + +Same envelope (`numFound`/`start`/`docs[]`); author docs carry bare-form keys +(`OL9937375A`) unlike book-search path keys: + +```json +{"name": "Mark Twain", "key": "OL9937375A", + "birth_date": "30 November 1835", "death_date": "21 April 1910", + "top_work": "Roughing It", "work_count": 2157, + "top_subjects": ["Twain, mark, 1835-1910", ...]} +``` + +Supports Solr syntax in `q` too (e.g. `birth_date:1973`) plus `limit`/`offset`. +Note `birth_date`/`death_date` may be null or free-text strings ("7 February 1812") — +they are display strings, not typed dates. + +Batch-fetch trick documented on the Authors API page: partial author records via +book search with `q=key:(/authors/OL11111A OR /authors/OL22222A)`; there is no +batch endpoint for full author records. + +## Subject browsing: `/subjects/.json` (plural!) + +The Subjects API ([dev/docs/api/subjects](https://openlibrary.org/dev/docs/api/subjects), +marked experimental) browses works grouped by normalized subject: + +``` +GET https://openlibrary.org/subjects/pizza.json?limit=1 +→ {"key": "", "name": "pizza", "work_count": 519, + "works": [{"key": "", "title": "Pete's a Pizza", + "edition_count": 19, "authors": [{"name": "William Steig"}], + "first_publish_year": 1998, "availability": {...}}, ...]} +``` + +- Path is **plural** `/subjects/.json`; singular `/subject/pizza.json` 404s + (live-verified). +- Names use underscores: `science_fiction`. +- Params: `details=true` (adds related `subjects[]`/`authors[]`/`publishers[]` + with counts plus `publishing_history`), `ebooks=true`, `published_in=1500-1600`, + `limit`, `offset`. +- Works here include `availability` by default, unlike `/search.json`. +- Sibling collections exist for persons/places/times (`/persons/.json` etc.). + +## Full-text inside-book search: `/search/inside.json` + +Searches OCR text across millions of scanned books; Elasticsearch-shaped response: + +``` +GET https://openlibrary.org/search/inside.json?q=%22library science%22 +→ hits.total, hits.hits[] with _id (ia identifier), _score, + highlight.text[] ("{{{Library Science}}}" marks matches), + fields.identifier (ia id), edition.key/title, availability +``` + +Default page size 20; `limit`/`offset` supported (live-verified). A separate, +documented-but-experimental *per-book* inside search actually runs on archive.org +data nodes (`https://ia800204.us.archive.org/fulltext/inside.php?item_id=...`) +— see [dev/docs/api/search_inside](https://openlibrary.org/dev/docs/api/search_inside) +if you need per-page match geometry; that host is outside this skill's CLI. + +## Error model: silent empties vs hard failures + +Live-probed status codes — counterintuitive but consistent: + +| Request | Status | Body | +|---------|--------|------| +| missing or empty `q` | **200** | normal envelope, `numFound: 0`, `docs: []` | +| malformed query `q=title:"unclosed` | **200** | `numFound: 0` — no error surfaced | +| loosely-parseable garbage `q=(OR` | **200** | 568k loose matches | +| invalid enum `sort=bogus` | **500** | plain text `Internal Server Error` (not JSON!) | +| non-integer `limit=abc` | **422** | FastAPI validation JSON `{"detail":[{"type":"int_parsing",...}]}` | +| negative `offset=-5` | **422** | FastAPI validation JSON `greater_than_equal` | +| singular `/subject/pizza.json` | **404** | HTML error page | + +Design consequence: user-facing "no results" is usually **not** an error — treat empty +`docs` as success. Conversely a bad `--sort` choice fails loudly as non-JSON 500, so +clients should validate sort choices before sending (as the bundled CLI does). + +One environment caveat observed during research: responses can arrive with key fields +masked to asterisks by anti-bot middleware depending on client reputation. Production +responses carry real keys (the docs' own examples show them), but parsers should +tolerate both `OL…W` and `/works/OL…W` shapes and not assume key presence. + +## Sources + +- https://openlibrary.org/dev/docs/api/search — Search API parameters, fields= semantics, editions sub-query +- https://openlibrary.org/search/howto — query syntax, field scopes, filter examples +- https://openlibrary.org/developers/api — rate-limit etiquette applying to search traffic +- https://openlibrary.org/dev/docs/api/authors — author batch-fetch via key:(…) search +- https://openlibrary.org/dev/docs/api/subjects — Subjects API params (details/ebooks/published_in) +- https://openlibrary.org/dev/docs/api/search_inside — experimental per-book inside search (archive.org hosted) +- Live read-only probes against openlibrary.org (2026-08-26): sort keys, limit/offset ranges, fields=availability interaction, error status codes From 8b84e2e82437c233792444e59071a8d2bd4b26c0 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 08:43:43 -0400 Subject: [PATCH 14/40] feat(openlibrary): extend CLI with editions/ratings and redirect handling New subcommands: editions (per-work edition listing with next_offset) and ratings (joined ratings + bookshelf counts). The client now follows in-body /type/redirect stubs left by merged records (appending .json to stub locations, since extension-less URLs 301 into HTML), recovers author keys from the linked work when edition records ship authors:null, normalizes bare/path key forms, unwraps {type,value} text fields centrally, assembles covers-host URLs skipping -1 placeholders, and validates sort choices client-side to avoid the server's plain-text 500 on unknown values. Existing flags, subcommands, --json, and --dry-run preserved. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- openlibrary/scripts/openlibrary | 316 ++++++++++++++++++++++++++------ 1 file changed, 265 insertions(+), 51 deletions(-) diff --git a/openlibrary/scripts/openlibrary b/openlibrary/scripts/openlibrary index 8251300..5a114ac 100755 --- a/openlibrary/scripts/openlibrary +++ b/openlibrary/scripts/openlibrary @@ -1,8 +1,10 @@ #!/usr/bin/env python3 """openlibrary — Open Library book metadata from the terminal. -Search books, authors, works, and lookup by ISBN using the public -Open Library API. No API key required. +Search books, authors, works, lookup by ISBN, enumerate editions of a work, +and read community ratings/bookshelf counts using the public Open Library +API. No API key required. Covers resolve on the separate covers host and +identifier endpoints answer with 302 redirects — both are handled here. """ import argparse @@ -18,10 +20,14 @@ import requests # === Config === DEFAULT_SERVER = "https://openlibrary.org" +COVERS_SERVER = "https://covers.openlibrary.org" ENV_SERVER = os.getenv("OL_SERVER", DEFAULT_SERVER) ENV_EMAIL = os.getenv("OL_EMAIL", "") ENV_USER_AGENT = os.getenv("OL_USER_AGENT", "openlibrary/1.0 (+https://github.com)") +# Merged/deleted wiki records can chain through redirect stubs; bound the walk. +MAX_REDIRECT_HOPS = 5 + QUIET = False GLOBAL_FLAGS: Dict[str, Any] = { "json": False, "dry_run": False, "quiet": False, "verbose": False @@ -70,6 +76,29 @@ def _preparse_global_flags(argv: List[str]) -> Tuple[Dict[str, Any], List[str]]: return flags, filtered +def normalize_olid(key: str) -> str: + """Strip a path-form key (/works/OL123W) down to its bare OLID (OL123W).""" + return key.rstrip("/").split("/")[-1] if key else "" + + +def unwrap_text(value: Any) -> str: + """Open Library wraps free-text fields as {'type': '/type/text', 'value': ...} + on some records and plain strings on others.""" + if isinstance(value, dict): + return str(value.get("value", "")) + return str(value) if value is not None else "" + + +def is_redirect_stub(payload: Any) -> bool: + """Merged-away keys answer HTTP 200 with a /type/redirect stub instead of 3xx.""" + return ( + isinstance(payload, dict) + and isinstance(payload.get("type"), dict) + and payload["type"].get("key") == "/type/redirect" + and bool(payload.get("location")) + ) + + class OpenLibraryClient: """Client for the public Open Library API.""" @@ -84,30 +113,61 @@ class OpenLibraryClient: return h def _get(self, path: str, params: Optional[Dict] = None) -> Any: + """GET JSON from the metadata host, following both HTTP redirects + (identifier endpoints answer 302) and in-body /type/redirect stubs + (merged keys answer 200 with a location field).""" url = f"{self.server}{path}" if self.dry_run: return {"dry_run": True, "url": url, "params": params} - try: - resp = requests.get(url, params=params, headers=self._headers(), timeout=30) - except requests.ConnectionError as e: - die(f"Cannot connect to {self.server}: {e}") - if resp.status_code == 404: - return None - if resp.status_code >= 400: + hops = 0 + while True: try: - detail = resp.json() - except Exception: - detail = resp.text[:200] - die(f"API error ({resp.status_code}): {detail}") - try: - return resp.json() - except ValueError: - return {"raw": resp.text[:500]} + resp = requests.get(url, params=params, headers=self._headers(), timeout=30) + except requests.ConnectionError as e: + die(f"Cannot connect to {self.server}: {e}") + if resp.status_code == 404: + return None + if resp.status_code >= 400: + try: + detail = resp.json() + except Exception: + detail = resp.text[:200] + die(f"API error ({resp.status_code}): {detail}") + try: + data = resp.json() + except ValueError: + return {"raw": resp.text[:500]} + if is_redirect_stub(data) and hops < MAX_REDIRECT_HOPS: + # Stub locations are bare keys (/works/OL…W, no .json), and + # extension-less URLs redirect to HTML pages — always refetch + # with the .json suffix. + target = data["location"] + if not target.endswith(".json"): + target += ".json" + url = f"{self.server}{target}" + params = None + hops += 1 + continue + self.last_url = resp.url + return data + + def last_final_url(self) -> str: + return getattr(self, "last_url", "") + + +def cover_url(kind: str, value: Any, size: str = "M") -> Optional[str]: + """Build a covers-host URL. Returns None for absent/negative IDs (-1 means + 'no image'). Callers should append ?default=false when existence matters.""" + if value is None: + return None + if isinstance(value, int) and value < 0: + return None + return f"{COVERS_SERVER}/{kind}/{value}-{size}.jpg" def fmt_author(a: Dict) -> str: name = a.get("name", a.get("title", "?")) - key = a.get("key", "").split("/")[-1] if a.get("key") else "" + key = normalize_olid(a.get("key", "")) birth = a.get("birth_date", "") death = a.get("death_date", "") years = f" ({birth}–{death})" if birth or death else "" @@ -125,7 +185,8 @@ def cmd_search(client, args): parsed, _ = parser.parse_known_args(args) if client.dry_run: - emit(f"[dry-run] Would search: {parsed.query}", {"dry_run": True, "query": parsed.query}) + emit(f"[dry-run] Would search: {parsed.query}", {"dry_run": True, "command": "search", + "query": parsed.query, "url": f"{client.server}/search.json"}) return params: Dict[str, Any] = {"q": parsed.query, "limit": parsed.limit, "offset": parsed.offset} @@ -137,7 +198,9 @@ def cmd_search(client, args): docs = data.get("docs", []) if not docs: - emit("No results.", {"results": []}) + # Malformed queries parse loosely and come back 200-empty; that is a + # result, not an API failure. + emit("No results.", {"total": 0, "results": []}) return lines, out = [], [] @@ -152,7 +215,8 @@ def cmd_search(client, args): "title": title, "authors": d.get("author_name", []), "first_publish_year": year, "edition_count": edition_count, "key": d.get("key", ""), "isbn": d.get("isbn", [])[:3], - "cover_edition": d.get("cover_edition_key", "") + "cover_edition": d.get("cover_edition_key", ""), + "has_fulltext": d.get("has_fulltext", False), }) total = data.get("numFound", len(docs)) @@ -165,7 +229,10 @@ def cmd_isbn(client, args): parsed, _ = parser.parse_known_args(args) if client.dry_run: - emit(f"[dry-run] Would lookup ISBN: {parsed.isbn}", {"dry_run": True, "isbn": parsed.isbn}) + emit(f"[dry-run] Would lookup ISBN: {parsed.isbn}", + {"dry_run": True, "command": "isbn", "isbn": parsed.isbn, + "url": f"{client.server}/isbn/{parsed.isbn}.json", + "note": "endpoint answers 302; request follows redirects"}) return data = client._get(f"/isbn/{parsed.isbn}.json") @@ -175,21 +242,55 @@ def cmd_isbn(client, args): title = data.get("title", "?") authors_list = data.get("authors", []) - author_names = ", ".join([a.get("name", a.get("key", "?")) for a in authors_list]) if authors_list else "?" + + def _author_label(a: Dict) -> str: + # Edition records often carry key-only author refs (no embedded name); + # fall back to the bare OL…A so output is never just '?'. + label = a.get("name") or normalize_olid(a.get("key", "")) + return label or "?" + + author_names = ", ".join(_author_label(a) for a in authors_list) if authors_list else "" + work_keys = [normalize_olid(w.get("key", "")) for w in data.get("works", [])] + + if not author_names and work_keys: + # Some editions ship authors:null entirely. The authoritative author + # links live on the work (double-nested authors[].author.key) — one + # extra read beats reporting an unknown author. + work_data = client._get(f"/works/{work_keys[0]}.json") or {} + work_authors = [ + normalize_olid(a.get("author", {}).get("key", "")) + for a in (work_data.get("authors") or []) + if isinstance(a, dict) + ] + author_names = ", ".join(k for k in work_authors if k) + if not author_names: + author_names = "?" pages = data.get("number_of_pages", data.get("pagination", "?")) publishers = ", ".join(data.get("publishers", [])) or "?" publish_date = data.get("publish_date", "?") subjects = ", ".join(data.get("subjects", [])[:5]) or "(none)" + description = unwrap_text(data.get("description", "")) + desc_short = f"\n Description: {description[:300]}" if description else "" + + edition_key = normalize_olid(data.get("key", "")) + covers = [c for c in data.get("covers", []) if isinstance(c, int) and c >= 0] emit( f"📖 {title}\n" f" Author(s): {author_names}\n" f" Pages: {pages} Published: {publish_date}\n" f" Publisher: {publishers}\n" - f" Subjects: {subjects}", + f" Subjects: {subjects}" + f"{desc_short}\n" + f" Edition: {edition_key} Works: {', '.join(work_keys) or '?'}", {"isbn": parsed.isbn, "title": title, "authors": author_names, "pages": pages, "publish_date": publish_date, - "publishers": publishers, "subjects": data.get("subjects", [])} + "publishers": publishers, "subjects": data.get("subjects", []), + "description": description, + "edition_key": edition_key, + "work_keys": work_keys, + "cover_id": covers[0] if covers else None, + "cover_url": cover_url("b/id", covers[0]) if covers else None} ) @@ -198,29 +299,34 @@ def cmd_author(client, args): parser.add_argument("key", help="Author key (e.g. OL23919A)") parsed, _ = parser.parse_known_args(args) + key = normalize_olid(parsed.key) if client.dry_run: - emit(f"[dry-run] Would fetch author {parsed.key}", {"dry_run": True, "key": parsed.key}) + emit(f"[dry-run] Would fetch author {key}", + {"dry_run": True, "command": "author", "key": key, + "url": f"{client.server}/authors/{key}.json"}) return - data = client._get(f"/authors/{parsed.key}.json") + data = client._get(f"/authors/{key}.json") if not data: - emit(f"Author {parsed.key} not found.", {"error": "not found"}) + emit(f"Author {key} not found.", {"error": "not found"}) return name = data.get("name", "?") birth = data.get("birth_date", "") death = data.get("death_date", "") years = f" ({birth}–{death})" if birth or death else "" - bio = data.get("bio", "") - if isinstance(bio, dict): - bio = bio.get("value", "") + bio = unwrap_text(data.get("bio", "")) bio_short = f"\n{bio[:500]}" if bio else "" + photos = [p for p in data.get("photos", []) if isinstance(p, int) and p >= 0] + photo = cover_url("a/id", photos[0]) if photos else None + photo_note = f"\n Photo: {photo}" if photo else "" emit( - f"👤 {name}{years}{bio_short}", - {"key": parsed.key, "name": name, "birth_date": birth, + f"👤 {name}{years}{bio_short}{photo_note}", + {"key": key, "name": name, "birth_date": birth, "death_date": death, "bio": bio, "wikipedia": data.get("wikipedia", ""), - "personal_name": data.get("personal_name", "")} + "personal_name": data.get("personal_name", ""), + "photo_url": photo} ) @@ -229,30 +335,34 @@ def cmd_work(client, args): parser.add_argument("key", help="Work key (e.g. OL123W)") parsed, _ = parser.parse_known_args(args) + key = normalize_olid(parsed.key) if client.dry_run: - emit(f"[dry-run] Would fetch work {parsed.key}", {"dry_run": True, "key": parsed.key}) + emit(f"[dry-run] Would fetch work {key}", + {"dry_run": True, "command": "work", "key": key, + "url": f"{client.server}/works/{key}.json"}) return - data = client._get(f"/works/{parsed.key}.json") + data = client._get(f"/works/{key}.json") if not data: - emit(f"Work {parsed.key} not found.", {"error": "not found"}) + emit(f"Work {key} not found.", {"error": "not found"}) return title = data.get("title", "?") authors = data.get("authors", []) - author_str = ", ".join([a.get("author", {}).get("key", "?") for a in authors]) if authors else "?" - desc = data.get("description", "") - if isinstance(desc, dict): - desc = desc.get("value", "") + author_str = ", ".join([normalize_olid(a.get("author", {}).get("key", "?")) + for a in authors]) if authors else "?" + desc = unwrap_text(data.get("description", "")) desc_short = f"\n{desc[:500]}" if desc else "" subjects = ", ".join(data.get("subjects", [])[:5]) or "(none)" + covers = [c for c in data.get("covers", []) if isinstance(c, int) and c >= 0] emit( f"📖 {title}\n" f" Author(s): {author_str}\n" f" Subjects: {subjects}{desc_short}", - {"key": parsed.key, "title": title, "authors": author_str, - "description": desc, "subjects": data.get("subjects", [])} + {"key": key, "title": title, "authors": author_str, + "description": desc, "subjects": data.get("subjects", []), + "cover_url": cover_url("b/id", covers[0]) if covers else None} ) @@ -264,7 +374,9 @@ def cmd_search_authors(client, args): parsed, _ = parser.parse_known_args(args) if client.dry_run: - emit(f"[dry-run] Would search authors: {parsed.query}", {"dry_run": True, "query": parsed.query}) + emit(f"[dry-run] Would search authors: {parsed.query}", + {"dry_run": True, "command": "search-authors", "query": parsed.query, + "url": f"{client.server}/search/authors.json"}) return data = client._get("/search/authors.json", {"q": parsed.query, "limit": parsed.limit, "offset": parsed.offset}) or {} @@ -276,19 +388,112 @@ def cmd_search_authors(client, args): lines, out = [], [] for d in docs: name = d.get("name", "?") - key = d.get("key", "").split("/")[-1] if d.get("key") else "?" - birth = d.get("birth_date", "") - death = d.get("death_date", "") + key = normalize_olid(d.get("key", "")) or "?" + birth = d.get("birth_date", "") or "" + death = d.get("death_date", "") or "" years = f" ({birth}–{death})" if birth or death else "" top_work = d.get("top_work", "") work_str = f" — {top_work}" if top_work else "" lines.append(f" {name:30}{years} [{key}]{work_str}") - out.append({"name": name, "key": key, "birth_date": birth, "death_date": death, "top_work": top_work}) + out.append({"name": name, "key": key, "birth_date": birth, "death_date": death, + "top_work": top_work, "work_count": d.get("work_count", 0)}) total = data.get("numFound", len(docs)) emit(f"{total} author(s):\n" + "\n".join(lines), {"total": total, "results": out}) +def cmd_editions(client, args): + """List every edition of a work via /works//editions.json.""" + parser = argparse.ArgumentParser(prog="openlibrary editions") + parser.add_argument("key", help="Work key (e.g. OL81699W)") + parser.add_argument("--limit", type=int, default=50) + parser.add_argument("--offset", type=int, default=0) + parsed, _ = parser.parse_known_args(args) + + key = normalize_olid(parsed.key) + if client.dry_run: + emit(f"[dry-run] Would list editions of work {key}", + {"dry_run": True, "command": "editions", "key": key, + "url": f"{client.server}/works/{key}/editions.json", + "params": {"limit": parsed.limit, "offset": parsed.offset}}) + return + + data = client._get(f"/works/{key}/editions.json", + {"limit": parsed.limit, "offset": parsed.offset}) + if not data: + emit(f"Work {key} not found.", {"error": "not found"}) + return + + entries = data.get("entries", []) + size = data.get("size", len(entries)) + if not entries: + emit(f"No editions listed for {key}.", {"size": 0, "editions": []}) + return + + lines, out = [], [] + for e in entries: + ekey = normalize_olid(e.get("key", "")) + title = e.get("title", "?") + pub = e.get("publish_date", "?") + publisher = ", ".join(e.get("publishers", [])[:2]) or "?" + isbn13 = (e.get("isbn_13") or ["?"])[0] + lines.append(f" [{ekey}] {title} — {publisher}, {pub} (ISBN-13: {isbn13})") + out.append({"key": ekey, "title": title, "publish_date": pub, + "publishers": e.get("publishers", []), + "isbn_10": e.get("isbn_10", []), "isbn_13": e.get("isbn_13", []), + "pages": e.get("number_of_pages")}) + + links = data.get("links", {}) + next_link = links.get("next", "") + more = "" + if next_link: + more = f"\n(more editions available; retry with --offset {parsed.offset + parsed.limit})" + emit(f"{size} edition(s) of {key}:\n" + "\n".join(lines) + more, + {"size": size, "editions": out, "next_offset": + parsed.offset + parsed.limit if next_link else None}) + + +def cmd_ratings(client, args): + """Community aggregates for a work: ratings + bookshelf counts.""" + parser = argparse.ArgumentParser(prog="openlibrary ratings") + parser.add_argument("key", help="Work key (e.g. OL45804W)") + parsed, _ = parser.parse_known_args(args) + + key = normalize_olid(parsed.key) + if client.dry_run: + emit(f"[dry-run] Would fetch ratings and shelf counts for work {key}", + {"dry_run": True, "command": "ratings", "key": key, + "urls": [f"{client.server}/works/{key}/ratings.json", + f"{client.server}/works/{key}/bookshelves.json"]}) + return + + ratings = client._get(f"/works/{key}/ratings.json") or {} + shelves = client._get(f"/works/{key}/bookshelves.json") or {} + + summary = ratings.get("summary", {}) + counts = ratings.get("counts", {}) + shelf_counts = shelves.get("counts", {}) + if not summary and not shelf_counts: + emit(f"No community data for work {key}.", + {"error": "no data", "key": key}) + return + + avg = summary.get("average") + avg_str = f"{avg:.2f}" if isinstance(avg, (int, float)) else "n/a" + rated = summary.get("count", 0) + wtr = shelf_counts.get("want_to_read", 0) + emit( + f"⭐ Work {key}: {avg_str} avg from {rated} rating(s)\n" + f" Distribution: " + ", ".join(f"{s}★={counts.get(s, 0)}" for s in ("5", "4", "3", "2", "1")) + "\n" + f" Shelves: want_to_read={wtr} currently_reading={shelf_counts.get('currently_reading', 0)} " + f"already_read={shelf_counts.get('already_read', 0)}", + {"key": key, + "average": avg, "ratings_count": rated, + "rating_distribution": counts, + "bookshelves": shelf_counts} + ) + + def main(): global GLOBAL_FLAGS, QUIET GLOBAL_FLAGS, filtered_argv = _preparse_global_flags(sys.argv) @@ -317,9 +522,16 @@ def main(): p_sa.add_argument("--limit", type=int, default=20) p_sa.add_argument("--offset", type=int, default=0) - sub.add_parser("author", help="Get author details") .add_argument("key") - sub.add_parser("work", help="Get work details") .add_argument("key") - sub.add_parser("isbn", help="Lookup by ISBN") .add_argument("isbn") + sub.add_parser("author", help="Get author details").add_argument("key") + sub.add_parser("work", help="Get work details").add_argument("key") + sub.add_parser("isbn", help="Lookup by ISBN").add_argument("isbn") + + p_ed = sub.add_parser("editions", help="List all editions of a work") + p_ed.add_argument("key") + p_ed.add_argument("--limit", type=int, default=50) + p_ed.add_argument("--offset", type=int, default=0) + + sub.add_parser("ratings", help="Community ratings and shelf counts for a work").add_argument("key") args = parser.parse_args(filtered_argv[1:]) if not args.command: @@ -334,6 +546,8 @@ def main(): "author": cmd_author, "work": cmd_work, "isbn": cmd_isbn, + "editions": cmd_editions, + "ratings": cmd_ratings, } handler = cmd_map.get(args.command) if not handler: From c6caa55bacaba56df4ad42f41f2c64823cda3035 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 08:43:54 -0400 Subject: [PATCH 15/40] test(openlibrary): offline suite for extended CLI 27 tests across five classes: help output, argument errors, dry-run plans, mocked-client logic (merge-stub following with .json suffix repair, work-author fallback for authors:null editions, editions paging, ratings join, covers-host URLs, empty-results-as-success), plus two live probes gated behind OPENLIBRARY_LIVE_TESTS=1 that skip cleanly otherwise. Passes strict-markers pytest, unittest discovery, and the proxy-trap zero-egress rerun. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- openlibrary/scripts/test_openlibrary.py | 371 ++++++++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 openlibrary/scripts/test_openlibrary.py diff --git a/openlibrary/scripts/test_openlibrary.py b/openlibrary/scripts/test_openlibrary.py new file mode 100644 index 0000000..8988f31 --- /dev/null +++ b/openlibrary/scripts/test_openlibrary.py @@ -0,0 +1,371 @@ +"""Offline tests for the bundled openlibrary CLI (scripts/openlibrary). + +Four test classes per skill-builder contract: + 1. --help output + 2. argument-error paths + 3. --dry-run behavior + 4. mocked-client logic (requests mocked at the client-call site) + +Plus one env-guarded live class: Open Library is a keyless public API, so a +small bounded set of live GETs runs ONLY when OPENLIBRARY_LIVE_TESTS=1; they +skip cleanly otherwise (proxy-trap reruns pass with them skipped). +""" + +import contextlib +import importlib.machinery +import importlib.util +import io +import json +import os +import pathlib +import unittest +from unittest import mock + +import requests + +SCRIPT = pathlib.Path(__file__).resolve().parent / "openlibrary" +LOADER = importlib.machinery.SourceFileLoader("openlibrary_cli", str(SCRIPT)) +SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER) +ol_cli = importlib.util.module_from_spec(SPEC) +LOADER.exec_module(ol_cli) + +EDITION_KEY = "/books/" + "OL34854896M" +WORK_KEY = "/works/" + "OL1168083W" +AUTHOR_KEY = "/authors/" + "OL118077A" + + +def run_cli(*argv): + """Invoke main() with argv[0] prepended; returns (exit_code, stdout, stderr).""" + out, err = io.StringIO(), io.StringIO() + code = 0 + with mock.patch.object(ol_cli.sys, "argv", ["openlibrary", *argv]): + with mock.patch.object(ol_cli.sys, "stdout", out), \ + mock.patch.object(ol_cli.sys, "stderr", err), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + ol_cli.main() + except SystemExit as exc: + code = exc.code if isinstance(exc.code, int) else 0 + return code, out.getvalue(), err.getvalue() + + +class FakeResponse: + def __init__(self, status_code=200, payload=None, text="", headers=None, + url="https://openlibrary.org/x"): + self.status_code = status_code + self._payload = payload + self.text = text or (json.dumps(payload) if payload is not None else "") + self.headers = headers or {} + self.url = url + + def json(self): + if self._payload is None: + raise ValueError("no json") + return self._payload + + +# === Class 1: help output === + + +class HelpOutputTests(unittest.TestCase): + def test_help_lists_all_subcommands(self): + code, out, _ = run_cli("--help") + self.assertEqual(code, 0) + for noun in ("search", "search-authors", "author", "work", + "isbn", "editions", "ratings"): + self.assertIn(noun, out) + + def test_help_mentions_keyless_setup(self): + _, out, _ = run_cli("--help") + self.assertIn("No API key", out) + + def test_subcommand_help_mentions_flags(self): + _, out, _ = run_cli("search", "--help") + for flag in ("--query", "--limit", "--offset", "--sort", "--lang"): + self.assertIn(flag, out) + + def test_editions_help_documents_pagination(self): + _, out, _ = run_cli("editions", "--help") + self.assertIn("--limit", out) + self.assertIn("--offset", out) + + +# === Class 2: argument errors === + + +class ArgumentErrorTests(unittest.TestCase): + def test_search_requires_query(self): + code, _, err = run_cli("search") + self.assertEqual(code, 2) + self.assertIn("--query", err) + + def test_no_command_prints_help_and_exits(self): + code, out, _ = run_cli() + self.assertEqual(code, 1) + self.assertIn("usage:", out) + + def test_unknown_subcommand_fails(self): + code, _, err = run_cli("frobnicate") + self.assertEqual(code, 2) + self.assertIn("invalid choice", err) + + def test_sort_rejects_unknown_values_client_side(self): + # A bogus sort value makes Open Library return a plain-text HTTP 500, + # so the CLI validates sort choices before ever sending. + code, _, err = run_cli("search", "--query", "dune", "--sort", "bogus") + self.assertEqual(code, 2) + self.assertIn("--sort", err) + + +# === Class 3: dry-run behavior === + + +class DryRunTests(unittest.TestCase): + def test_dry_run_isbn_reports_302_resolution_plan(self): + code, out, _ = run_cli("--dry-run", "--json", "isbn", "9780451524935") + self.assertEqual(code, 0) + plan = json.loads(out) + self.assertTrue(plan["dry_run"]) + self.assertEqual(plan["command"], "isbn") + self.assertIn("/isbn/9780451524935.json", plan["url"]) + self.assertIn("302", plan["note"]) + + def test_dry_run_editions_emits_query_params(self): + code, out, _ = run_cli("--json", "--dry-run", "editions", + WORK_KEY, "--limit", "5") + self.assertEqual(code, 0) + plan = json.loads(out) + self.assertEqual(plan["key"], "OL1168083W") + self.assertEqual(plan["params"]["limit"], 5) + + def test_dry_run_ratings_plans_both_endpoints(self): + code, out, _ = run_cli("--dry-run", "--json", "ratings", "OL45804W") + self.assertEqual(code, 0) + plan = json.loads(out) + joined = " ".join(plan["urls"]) + self.assertIn("/ratings.json", joined) + self.assertIn("/bookshelves.json", joined) + + def test_dry_run_never_touches_network(self): + with mock.patch.object(requests, "get") as req: + code, _, _ = run_cli("--dry-run", "work", WORK_KEY) + self.assertEqual(code, 0) + req.assert_not_called() + + +# === Class 4: mocked client logic === + + +EDITION_RECORD = { + "type": {"key": "/type/edition"}, + "key": EDITION_KEY, + "title": "Nineteen Eighty-Four", + "authors": None, # real records ship authors:null sometimes + "works": [{"key": WORK_KEY}], + "covers": [12054527, -1], + "number_of_pages": 328, + "publish_date": "1993?", + "publishers": ["Signet Classics"], + "description": {"type": "/type/text", "value": "A dystopian classic."}, +} + +WORK_RECORD = { + "type": {"key": "/type/work"}, + "key": WORK_KEY, + "title": "Nineteen Eighty-Four", + "authors": [{"author": {"key": AUTHOR_KEY}, + "type": {"key": "/type/author_role"}}], + "subjects": ["Totalitarianism"], + "description": "A dystopian classic.", + "covers": [-1], +} + + +class MockedClientTests(unittest.TestCase): + """Mock requests.get at the client-call site; zero network in this class.""" + + def setUp(self): + ol_cli.QUIET = False + ol_cli.GLOBAL_FLAGS.update(json=True, dry_run=False, quiet=False) + + def test_isbn_surfaces_resolved_edition_and_work_keys(self): + # The edition record carries works[] but authors:null, so the CLI + # follows the work link once to recover author keys. + edition = FakeResponse( + 200, EDITION_RECORD, + url="https://openlibrary.org/books/" + "OL34854896M.json") + work = FakeResponse(200, WORK_RECORD) + with mock.patch.object(requests, "get", side_effect=[edition, work]) as req: + code, out, _ = run_cli("--json", "isbn", "9780451524935") + self.assertEqual(code, 0) + self.assertEqual(req.call_count, 2) + data = json.loads(out) + self.assertEqual(data["edition_key"], "OL34854896M") + self.assertEqual(data["work_keys"], ["OL1168083W"]) + # Cover URLs point at the SEPARATE covers host, skipping -1 placeholders. + self.assertEqual(data["cover_url"], ( + "https://covers.openlibrary.org/b/id/" + + "12054527-M.jpg")) + + def test_isbn_falls_back_to_work_authors_when_edition_has_none(self): + edition = FakeResponse(200, EDITION_RECORD) + work = FakeResponse(200, WORK_RECORD) + with mock.patch.object(requests, "get", side_effect=[edition, work]) as req: + code, out, _ = run_cli("--json", "isbn", "9780451524935") + self.assertEqual(code, 0) + self.assertEqual(req.call_count, 2) + self.assertTrue(req.call_args_list[1].args[0].endswith(WORK_KEY + ".json")) + self.assertEqual(json.loads(out)["authors"], "OL118077A") + + def test_merge_redirect_stub_in_http_200_is_followed_with_json_suffix(self): + # Merged-away keys answer 200 with {type:/type/redirect, location}, + # NOT a 3xx — the client must detect the stub and refetch. Stub + # locations are bare keys without .json; extension-less URLs redirect + # to HTML pages, so the client must append the suffix itself. + stub = FakeResponse(200, {"type": {"key": "/type/redirect"}, + "location": WORK_KEY}) + work = FakeResponse(200, WORK_RECORD) + with mock.patch.object(requests, "get", side_effect=[stub, work]) as req: + code, out, _ = run_cli("--json", "work", "OL24776360W") + self.assertEqual(code, 0) + self.assertEqual( + req.call_args_list[1].args[0], + "https://openlibrary.org" + WORK_KEY + ".json") + self.assertEqual(json.loads(out)["title"], "Nineteen Eighty-Four") + + def test_text_wrapper_dict_is_unwrapped(self): + self.assertEqual(ol_cli.unwrap_text({"type": "/type/text", "value": "hi"}), "hi") + self.assertEqual(ol_cli.unwrap_text("plain"), "plain") + self.assertEqual(ol_cli.unwrap_text(None), "") + + def test_normalize_olid_accepts_bare_and_path_forms(self): + self.assertEqual(ol_cli.normalize_olid("/works/" + "OL123W"), "OL123W") + self.assertEqual(ol_cli.normalize_olid("OL23919A"), "OL23919A") + self.assertEqual(ol_cli.normalize_olid(""), "") + + def test_cover_url_rejects_negative_placeholder_ids(self): + self.assertIsNone(ol_cli.cover_url("b/id", -1)) + self.assertIsNone(ol_cli.cover_url("a/id", None)) + self.assertTrue(ol_cli.cover_url("b/id", 12054527).startswith( + "https://covers.openlibrary.org/b/id/")) + + def test_search_sends_query_and_sort_params(self): + payload = {"numFound": 1, "docs": [ + {"key": "/works/" + "OL1W", "title": "Dune", + "author_name": ["Frank Herbert"], "first_publish_year": 1965, + "edition_count": 90, "cover_edition_key": "OL1M", + "has_fulltext": True}]} + with mock.patch.object(requests, "get", + return_value=FakeResponse(200, payload)) as req: + code, out, _ = run_cli("--json", "search", "--query", "dune", + "--limit", "2", "--sort", "editions") + self.assertEqual(code, 0) + req.assert_called_once() + sent = req.call_args.kwargs["params"] + self.assertEqual(sent["q"], "dune") + self.assertEqual(sent["sort"], "editions") + data = json.loads(out) + self.assertEqual(data["total"], 1) + self.assertEqual(data["results"][0]["key"], "/works/" + "OL1W") + + def test_empty_search_results_are_success_not_error(self): + # Malformed queries parse loosely and return 200-empty; the CLI must + # report zero results without failing. + payload = {"numFound": 0, "docs": []} + with mock.patch.object(requests, "get", + return_value=FakeResponse(200, payload)): + code, out, _ = run_cli("--json", "search", "--query", 'title:"unclosed') + self.assertEqual(code, 0) + self.assertEqual(json.loads(out)["total"], 0) + + def test_editions_parses_entries_and_computes_next_offset(self): + payload = {"size": 6, + "links": {"self": "/works/x/editions.json?limit=3", + "next": "/works/x/editions.json?limit=3&offset=3"}, + "entries": [ + {"key": "/books/" + "OL1M", "title": "Ed. One", + "publishers": ["Ace"], "publish_date": "1965", + "isbn_13": ["9780000000002"]}, + {"key": "/books/" + "OL2M", "title": "Ed. Two", + "publishers": [], "publish_date": "1980", "isbn_13": []}, + {"key": "/books/" + "OL3M", "title": "Ed. Three", + "publishers": ["NEL"], "publish_date": "1974", + "isbn_13": ["9781111111113"]}, + ]} + with mock.patch.object(requests, "get", + return_value=FakeResponse(200, payload)) as req: + code, out, _ = run_cli("--json", "editions", "OL81699W", "--limit", "3") + self.assertEqual(code, 0) + self.assertEqual(req.call_args.kwargs["params"]["offset"], 0) + data = json.loads(out) + self.assertEqual(data["size"], 6) + self.assertEqual(len(data["editions"]), 3) + self.assertEqual(data["next_offset"], 3) + + def test_ratings_joins_ratings_and_bookshelves(self): + ratings = FakeResponse(200, {"summary": {"average": 3.97, "count": 119}, + "counts": {"5": 56, "4": 29}}) + shelves = FakeResponse(200, {"counts": {"want_to_read": 1191, + "currently_reading": 97}}) + with mock.patch.object(requests, "get", side_effect=[ratings, shelves]) as req: + code, out, _ = run_cli("--json", "ratings", "OL45804W") + self.assertEqual(code, 0) + self.assertTrue(req.call_args_list[0].args[0].endswith("/ratings.json")) + self.assertTrue(req.call_args_list[1].args[0].endswith("/bookshelves.json")) + data = json.loads(out) + self.assertAlmostEqual(data["average"], 3.97) + self.assertEqual(data["bookshelves"]["want_to_read"], 1191) + + def test_404_reports_not_found_without_traceback(self): + with mock.patch.object(requests, "get", + return_value=FakeResponse(404, None)): + code, out, _ = run_cli("work", "OL999999999W") + self.assertEqual(code, 0) + self.assertIn("not found", out.lower()) + + def test_server_error_names_status_and_dies(self): + # run_cli captures SystemExit; a 500 must exit 1 with the status named. + with mock.patch.object(requests, "get", + return_value=FakeResponse(500, text="Internal Server Error")): + code, _, err = run_cli("work", "OL1W") + self.assertEqual(code, 1) + self.assertIn("500", err) + + def test_user_agent_carries_mailto_when_email_configured(self): + original = ol_cli.ENV_EMAIL + try: + ol_cli.ENV_EMAIL = "reader@example.org" + resp = FakeResponse(200, WORK_RECORD) + with mock.patch.object(requests, "get", return_value=resp) as req: + run_cli("work", WORK_KEY) + ua = req.call_args.kwargs["headers"]["User-Agent"] + self.assertIn("(mailto:reader@example.org)", ua) + finally: + ol_cli.ENV_EMAIL = original + + +# === Class 5: env-guarded live probes (keyless public API) === +# Run only with OPENLIBRARY_LIVE_TESTS=1; skipped otherwise so the proxy-trap +# rerun proves zero egress for everything above. + + +@unittest.skipUnless(os.getenv("OPENLIBRARY_LIVE_TESTS") == "1", + "live probes disabled (set OPENLIBRARY_LIVE_TESTS=1)") +class LiveGuardedTests(unittest.TestCase): + def test_live_isbn_resolves_through_302(self): + code, out, _ = run_cli("--json", "isbn", "9780451524935") + self.assertEqual(code, 0) + data = json.loads(out) + self.assertRegex(data["edition_key"], r"^OL\d+M$") + self.assertRegex(data["work_keys"][0], r"^OL\d+W$") + self.assertTrue(data["title"]) + + def test_live_editions_listing_returns_entries(self): + code, out, _ = run_cli("--json", "editions", "OL81699W", "--limit", "3") + self.assertEqual(code, 0) + data = json.loads(out) + self.assertGreaterEqual(data["size"], 1) + + +if __name__ == "__main__": + unittest.main() From 3dd995577006fd5c31eb9e0248344062bd7b8fad Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 08:44:08 -0400 Subject: [PATCH 16/40] docs(openlibrary): add eval manifest with negative probe Six schema-v1 cases: ISBN-to-work-author key-graph walk, ebook discovery pipeline, the ISBN 302-redirect gotcha diagnosis, author disambiguation, a should-not-trigger probe for Koha/MARC library administration (id contains -not-), and keyless-setup/rate-etiquette guidance. Passes validate-evals and the fake-adapter paired smoke. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- openlibrary/evals/evals.json | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 openlibrary/evals/evals.json diff --git a/openlibrary/evals/evals.json b/openlibrary/evals/evals.json new file mode 100644 index 0000000..4488659 --- /dev/null +++ b/openlibrary/evals/evals.json @@ -0,0 +1,71 @@ +{ + "schema_version": 1, + "skill_name": "openlibrary", + "evals": [ + { + "id": "isbn-to-work-author-chain", + "prompt": "Look up the book with ISBN 9780451524935 and tell me about the underlying work and its author.", + "expected_output": "Scenario: read-only key graph walk. The agent runs `openlibrary isbn 9780451524935 --json`, which resolves through Open Library's 302 redirect to a canonical OL…M edition record and reports the resolved edition_key plus work_keys. It then fetches the linked work (OL…W) for description/subjects and follows the work's double-nested authors[].author.key to an OL…A author record for the bio, unwrapping any {type,value}-shaped text fields. All output comes from real CLI output; no writes are attempted.", + "assertions": [ + "The ISBN lookup runs via openlibrary isbn with --json before any follow-up calls", + "The resolved edition key (OL…M) and work keys (OL…W) are read from the CLI output rather than invented", + "Author links on works are accessed as authors[].author.key (double-nested), not authors[].key", + "Bio/description fields wrapped as {type:/type/text,value} dicts are unwrapped to plain text" + ] + }, + { + "id": "find-readable-ebooks-by-subject", + "prompt": "Find me some classic science fiction books I can read online right now.", + "expected_output": "Scenario: multi-step search pipeline. The agent searches with `openlibrary search --query 'subject:\"science fiction\"' --sort editions` (or a title/author query), then checks full-text availability via has_fulltext/ebook_access signals in the JSON output, optionally fetching each candidate's edition or cover data. It explains that /search.json's `availability` field is silently omitted unless `ia` is also requested in fields= — a gotcha it avoids by relying on the CLI's surfaced has_fulltext flag or by fetching the edition record directly. Results are presented from actual command output with titles and first-publish years.", + "assertions": [ + "A search runs with openlibrary search rather than guessing at book records", + "Readability is judged from real fields (has_fulltext, ebook_access, or edition-level data) instead of assumed", + "No claim is made that availability data appears in search results without also requesting ia alongside it", + "Final recommendations cite titles/authors traceable to command output" + ] + }, + { + "id": "isbn-302-redirect-gotcha", + "prompt": "I curl'd https://openlibrary.org/isbn/9780451524935.json and my script choked parsing HTML instead of the book data. What went wrong?", + "expected_output": "Scenario: gotcha diagnosis. The agent explains that identifier endpoints (/isbn/.json, /lccn/.json, /oclc/.json) answer HTTP 302 with a Location header pointing at the canonical /books/.json URL — they never serve the record directly. A client that does not follow redirects sees only an HTML redirect page. Fix: follow redirects (curl -L; Python requests does so by default). The agent may demonstrate with `openlibrary isbn 9780451524935 --json`, which handles resolution automatically, and notes the final record's key reveals which edition matched.", + "assertions": [ + "The 302-redirect behavior of /isbn/.json is named as the root cause", + "The fix is to follow redirects (curl -L or requests' default behavior)", + "The canonical target format /books/.json is stated", + "The bundled CLI is offered as a path that already handles resolution" + ] + }, + { + "id": "author-disambiguation-and-top-works", + "prompt": "There are several authors named John Herbert. Which one wrote the Foundation series, and what else did they write?", + "expected_output": "Scenario: author disambiguation. The agent corrects the premise gently if needed (Foundation is by Isaac Asimov) but demonstrates the workflow: run `openlibrary search-authors --query '...' --json` to list candidates with bare OL…A keys, birth/death dates, top_work, and work_count, pick the right author, then fetch details with `openlibrary author `. If enumerating their books it can use the work listing endpoint described in references. Claims come from command output only.", + "assertions": [ + "search-authors is used to enumerate name candidates before picking one", + "Candidate identity is judged from top_work/work_count/date fields in output", + "The chosen bare OL…A key is passed to openlibrary author for details", + "Any correction of the premise (Foundation's actual author) is grounded in verified lookup results" + ] + }, + { + "id": "koha-catalog-migration-not-openlibrary", + "prompt": "Help me migrate our public library's MARC records into our Koha ILS and set up patron accounts.", + "expected_output": "Scenario: should-not-trigger. This request targets local library-catalog administration (Koha ILS migration, MARC batch processing, patron account management), which this skill explicitly does not cover — Open Library is a public metadata API, not an integrated library system. The agent does not load openlibrary or invoke its CLI; it routes toward Koha's own tooling/documentation instead, noting the skill covers reading Open Library's catalog data only.", + "assertions": [ + "The openlibrary skill is not loaded or executed for Koha/MARC administration work", + "Koha-native tooling is suggested as the appropriate route", + "No Open Library API calls are made as part of planning the ILS migration" + ] + }, + { + "id": "keyless-setup-and-rate-etiquette", + "prompt": "Set up whatever credentials you need to start researching books for me, and tell me what the limits are.", + "expected_output": "Scenario: setup expectations. The agent explains no API key or registration exists at all — reads on openlibrary.org are fully keyless, so there is nothing to configure beyond optional etiquette: setting OL_EMAIL to add a mailto contact to the User-Agent, which raises the polite rate budget from ~1 request/second to 3. Covers live on a separate host (covers.openlibrary.org) where ISBN-keyed lookups cap at 100 requests/IP per 5 minutes while cover-ID lookups are exempt. Bulk jobs belong in monthly dumps, not API loops. No secrets are requested because none exist.", + "assertions": [ + "States explicitly that the public API requires no key or registration for reads", + "OL_EMAIL is described as optional User-Agent identification raising the rate budget (~1/s anonymous vs ~3/s identified)", + "Covers-host separation and its distinct 100 req/IP-per-5-min limit on non-ID lookups is mentioned", + "No credential, token, or secret is requested or fabricated" + ] + } + ] +} From df44337a5d61ba5abafc7020378e248148c87f34 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 08:44:37 -0400 Subject: [PATCH 17/40] docs(openlibrary): lastfm-model SKILL.md rewrite and README refresh SKILL.md rebuilt to the lastfm model: keyless setup with covers-host table, intent-grouped commands including new editions/ratings, ISBN-to-work-to-author pipeline recipe, jq guidance, Known Gotchas grounded in researched behavior (ISBN 302 redirects, merge stubs inside HTTP 200, OL...M/W/A suffix system with double-nested author refs, availability-needs-ia projection rule, silent-empty search errors, rate etiquette), When-to-use/When-not-to-use boundaries, and a four-row reference routing table. Body 240 lines. README rewritten for humans with verified What You Get paths. Description updated with imperative verb start and negative boundary. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- openlibrary/README.md | 57 +++++++--- openlibrary/SKILL.md | 257 ++++++++++++++++++++++++++++++++---------- 2 files changed, 236 insertions(+), 78 deletions(-) diff --git a/openlibrary/README.md b/openlibrary/README.md index ba310a4..05900df 100644 --- a/openlibrary/README.md +++ b/openlibrary/README.md @@ -1,36 +1,61 @@ # Open Library — Book Metadata from the Terminal -Search books and authors, look up works and ISBNs, and fetch detailed metadata from the public Open Library API. No API key, no registration — just works. +Search books and authors, resolve ISBNs, walk the edition/work/author graph, +enumerate editions, and read community ratings from the public Open Library +API. No API key exists — every read is keyless. ## Why Install This Skill -When your agent loads this skill, it can **access 50M+ book records** without any setup. That means: +When your agent loads this skill, it gets **structured access to 50M+ book records** +without any signup or credentials: -- **Search by keyword** — find books by title, author, or subject -- **Look up by ISBN** — get detailed metadata for any ISBN -- **Author details** — bio, birth/death dates, top works -- **Work and edition info** — publication dates, languages, formats -- **Filter and sort** — by language, year, rating, or new arrivals +- **Search anything** — keyword queries with sort by edition count, date, or title; field-scoped lookups by title, author, subject, publisher, or ISBN +- **Resolve any identifier** — turn an ISBN/LCCN/OCLC into a canonical edition record and follow it up to the abstract work and its author +- **Enumerate editions** — every published version of a work with dates, publishers, and ISBNs +- **Read community signals** — star ratings and want-to-read counts per work +- **Get cover images correctly** — proper URLs on Open Library's dedicated image host, with existence checks that actually return 404 instead of blank placeholders + +The skill also encodes where agents typically trip: ISBN endpoints that answer +302 redirects, merged-record keys that hide redirect stubs inside HTTP 200 +responses, the OL…M/W/A key-suffix system, `{type,value}`-wrapped text fields, +and rate-limit etiquette that keeps you unblocked. ## What You Get -| Directory | Purpose | -|-----------|---------| -| `SKILL.md` | Complete command reference with examples | -| `scripts/openlibrary` | CLI tool for the Open Library API | +| Path | Purpose | +|------|---------| +| `SKILL.md` | Command reference: setup, intent-grouped commands, pipeline recipes, jq guidance, known gotchas | +| `scripts/openlibrary` | CLI tool for the Open Library API (`--json`, `--dry-run`, automatic redirect resolution) | +| `scripts/test_openlibrary.py` | Offline test suite for the CLI (help/errors/dry-run/mocked logic; live probes env-guarded) | +| `references/api-overview-and-key-graph.md` | Access model, rate etiquette, OLID key graph, merge-stub behavior | +| `references/search-api-guide.md` | Search parameters, sort keys, query syntax, sibling search endpoints, error model | +| `references/books-isbn-and-covers.md` | ISBN/identifier resolution, view models, editions, ratings, covers-host rules | +| `references/recipes-and-gotchas.md` | Worked curl/jq pipelines and a symptom-indexed gotcha table | +| `evals/evals.json` | Behavioral eval cases covering searches, pipelines, gotchas | ## Quick Start ```bash -openlibrary search --query "dune" -openlibrary search --isbn "9780439358064" -openlibrary search-authors --query "asimov" +openlibrary search --query "dune" # find works +openlibrary isbn 9780451524935 # resolve an ISBN to its edition +openlibrary editions OL81699W # list every edition of a work +openlibrary ratings OL45804W # community signals +``` + +Optional politeness knob: + +```bash +export OL_EMAIL="you@example.com" # adds contact to User-Agent; ~3x rate budget ``` ## Triggers -Load this for book research, ISBN lookups, author information, or cataloging projects. +Load this for book research, ISBN or OLID lookups, author biographies, edition +enumeration, reading-level community stats, cover-image URL assembly, or any +question about the Open Library catalog itself. ## Requirements -Python 3.8+ with `requests` library. No API key needed — fully public API. +- Python 3.8+ with the `requests` library +- No API key, no account — reads are fully public +- `jq` recommended for processing `--json` output diff --git a/openlibrary/SKILL.md b/openlibrary/SKILL.md index 0378aa4..a004d0f 100644 --- a/openlibrary/SKILL.md +++ b/openlibrary/SKILL.md @@ -1,125 +1,258 @@ --- name: openlibrary -description: Search books, authors, and works on Open Library from the terminal. Look - up books by ISBN, search titles and authors, and fetch detailed work/author records - via the public Open Library API. No API key required. +description: >- + Query the Open Library catalog from the terminal: search books and authors, + look up works, editions, and ISBNs, enumerate every edition of a work, read + community ratings, and resolve cover-image URLs. Fully keyless public API. + Includes the OL…M/W/A key-graph reference, ISBN 302-redirect resolution, + search query syntax, covers-host rules, and worked pipelines. Do not use for + library-IT administration (Koha/MARC/ILS migration), commercial book-data + feeds, or managing your reading account on Open Library itself. license: MIT compatibility: Python 3.8+ and the `requests` library. No API key or registration - required — the Open Library API is fully public. Optional OL_EMAIL env var sets - a User-Agent contact for improved rate limiting. + required — reads are fully public. Optional OL_EMAIL env var adds a contact to + the User-Agent for better rate-limit treatment. metadata: tags: open-library, books, authors, isbn, library, book-search, api-client, catalog - sources: https://openlibrary.org/developers/api, https://openlibrary.org + sources: https://openlibrary.org/developers/api, https://openlibrary.org/dev/docs/api/books --- # openlibrary — Book Metadata from Open Library -Search books and authors, look up works and ISBNs, and fetch detailed metadata from the public [Open Library](https://openlibrary.org) API. No API key, no registration — just works. +Query Open Library's 50M+ record catalog over its public HTTP API: search works +and authors, walk the edition/work/author graph by ISBN or OLID, enumerate +editions, and read community ratings. No API key exists; everything here is a +keyless GET. ## Setup -No authentication or API keys needed. The Open Library API is fully public. +Nothing to authenticate: **the public Open Library API requires no API key** for +reads. There is no token, no registration step, no lazy-auth dance. ```bash -# Optional: set your email to help with rate limiting (included in User-Agent): +# Optional etiquette: identified User-Agent gets ~3x rate budget (~3 req/s vs ~1) export OL_EMAIL="you@example.com" ``` -Only Python 3.8+ and the `requests` package are required. `--help` and `--dry-run` work without any setup. +Requires Python 3.8+ and `requests` only. `--help` and `--dry-run` work with no +environment at all. Write endpoints exist upstream but require an authenticated +Internet Archive session and are effectively internal — treat this surface as +read-only. + +Three hosts matter and they serve different things: + +| Host | Serves | +|------|--------| +| `openlibrary.org` | metadata JSON: search, records, ratings | +| `covers.openlibrary.org` | cover images + author photos (separate service) | +| `archive.org` | scan content / bulk dumps (redirect targets) | ## Essential Commands -### search — Search books by keyword +### find books — search ```bash -openlibrary search --query "dune" # basic search (20 results) -openlibrary search --query "dune" --limit 5 # just the top 5 +openlibrary search --query "dune" # relevance order +openlibrary search --query "dune" --sort editions # most editions first openlibrary search --query "dune" --sort new # newest first -openlibrary search --query "dune" --sort rating # highest rated first -openlibrary search --query "dune" --sort title # alphabetical by title -openlibrary search --query "foundation" --lang fr # French editions -openlibrary search --query "dune" --json # machine-readable JSON +openlibrary search --query "foundation" --lang fr # prefer French editions +openlibrary search --query "dune" --limit 5 --offset 10 # paginate +openlibrary search --query "dune" --json # machine-readable ``` -Shows: title, first publish year, authors, edition count. Supports `--sort` (editions, new, old, rating, title), `--lang`, `--offset`, and `--availability`. +Results carry title, authors, `first_publish_year`, edition count, and the work +key (`/works/OL…W`) that feeds the other commands. The CLI validates `--sort` +choices client-side because an unknown sort value makes the server return a +plain-text HTTP 500. -### search-authors — Search authors by name +### find people who write — author search ```bash -openlibrary search-authors --query "asimov" # search authors -openlibrary search-authors --query "asimov" --limit 5 # top 5 -openlibrary search-authors --query "asimov" --json # machine-readable +openlibrary search-authors --query "asimov" # name candidates +openlibrary search-authors --query "le guin" --limit 5 +openlibrary search-authors --query "asimov" --json # includes top_work, work_count ``` -Shows: name, birth/death years, author key, top work. +Author docs arrive with bare keys (`OL23919A`), unlike book search's path keys. -### author — Get author details by key +### inspect records — author / work ```bash -openlibrary author OL23919A # Isaac Asimov -openlibrary author OL34184A # Ursula K. Le Guin -openlibrary author OL23919A --json # full biography +openlibrary author OL23919A # bio, dates, photo URL +openlibrary work OL1168083W # description, subjects, cover URL +openlibrary work OL1168083W --json # full record ``` -Shows: name, birth/death dates, biography (up to 500 chars), Wikipedia link, personal name. Author keys are the `OL#####A` identifiers from search results. +Keys may be passed bare (`OL23919A`) or as paths (`/authors/OL23919A`) — the CLI +normalizes both. -### work — Get work details by key +### resolve an ISBN ```bash -openlibrary work OL123W # work by key -openlibrary work OL81699W # "Foundation" -openlibrary work OL81699W --json # full description +openlibrary isbn 9780451524935 # ISBN-10 or ISBN-13 +openlibrary isbn 9780451524935 --json # + resolved edition_key, work_keys, cover_url ``` -Shows: title, author keys, subjects, description (up to 500 chars). Work keys are the `OL#####W` identifiers found in search results. +Upstream, `/isbn/.json` answers a **302 redirect** to the canonical +edition JSON (`/books/.json`). The CLI follows it and reports which +edition matched via `edition_key`. If the edition record lacks author names +(some ship `authors:null`), the CLI recovers them from the linked work. -### isbn — Lookup a book by ISBN +### list every edition of a work ```bash -openlibrary isbn 9780451524935 # ISBN lookup -openlibrary isbn 9780553382563 # another book -openlibrary isbn 9780451524935 --json # full edition metadata +openlibrary editions OL81699W # publisher/date/ISBN per edition +openlibrary editions OL81699W --limit 50 --offset 50 # page through large sets +openlibrary editions OL81699W --json # + next_offset when more exist ``` -Shows: title, author(s), page count, publish date, publisher, subjects (top 5). Accepts both ISBN-10 and ISBN-13. +Backs onto `/works//editions.json`; `next_offset` is computed from the +server's prebuilt next-page link. + +### read the room — community signals + +```bash +openlibrary ratings OL45804W # average rating + shelf counts +openlibrary ratings OL45804W --json # full distribution + bookshelves +``` + +Joins `/works//ratings.json` (`summary.average`, per-star `counts`) with +`/works//bookshelves.json` (`want_to_read`, `currently_reading`, +`already_read`). ## Global Flags -These flags work anywhere in the command — before or after the subcommand: +Position-independent; put them before or after the subcommand: ```bash -openlibrary --json search --query "dune" # JSON output -openlibrary search --query "dune" --json # same result, after subcommand -openlibrary --dry-run search --query "dune" # preview without API call -openlibrary --quiet search --query "dune" # suppress diagnostic output -openlibrary --verbose author OL23919A # verbose logging -openlibrary --dry-run isbn 9780451524935 # see what URL would be used +openlibrary --json search --query "dune" # machine output anywhere +openlibrary --dry-run isbn 9780451524935 # preview the exact URL, no network +openlibrary --quiet search --query "dune" # suppress diagnostics ``` | Flag | Effect | |------|--------| -| `--json` | Output machine-readable JSON instead of human-readable text | -| `--dry-run` | Show what API call would be made without executing it | -| `--quiet` | Suppress non-essential diagnostic output | -| `--verbose` | Enable verbose/debug logging | +| `--json` | One JSON object on stdout instead of human text | +| `--dry-run` | Print the planned request as JSON without executing it | +| `--quiet` | Suppress non-essential output | +| `--verbose` | Verbose logging | + +## Multi-Step Pipeline Recipes + +### ISBN → edition → work → author bio + +The canonical walk across all three key types: + +```bash +openlibrary isbn 9780451524935 --json | jq -r '.work_keys[0]' # OL1168083W +openlibrary work OL1168083W --json | jq -r '.authors[0]' # bare OL…A key +openlibrary author OL118077A # bio, dates +``` + +Each hop uses a different key suffix (M → W → A); see Known Gotchas before +hand-assembling these URLs yourself. + +### Rank a series by community love + +```bash +for w in $(openlibrary search --query 'series:dune' --limit 8 --json | jq -r '.results[].key'); do + openlibrary ratings "${w##*/}" --json \ + | jq -c '{work: .key, avg: .average, rated: .ratings_count, + want_to_read: .bookshelves.want_to_read}' +done +``` + +### Find readable ebooks, then their print editions + +```bash +openlibrary search --query 'title:"moby dick"' --sort editions --json \ + | jq '.results[] | select(.has_fulltext) | {title, key}' +openlibrary editions OL81699W --json | jq '.editions[] | {key, isbn_13}' +``` + +## Using --json with jq + +```bash +openlibrary search --query "voracious" --json | jq '.results[] | {title, first_publish_year}' +openlibrary search-authors --query "butler" --json | jq '.results[0] | {name, key, top_work}' +openlibrary isbn 9780451524935 --json | jq -r '.cover_url' # covers host URL +openlibrary editions OL81699W --json | jq '[.editions[].publish_date]' +openlibrary ratings OL45804W --json | jq '.rating_distribution' +``` ## Known Gotchas -- **Public API, rate-limited** — No API key is needed, but Open Library enforces rate limits on unauthenticated requests. Set `OL_EMAIL` in your environment to include a contact email in the User-Agent header for better rate limit treatment. -- **ISBN lookup uses an edition endpoint** — The `isbn` command fetches `/isbn/{isbn}.json` which returns edition-level metadata (publisher, page count, publish date). For work-level metadata (description, subjects of the underlying work), use the `work` command with the work key from search results. -- **Search results are not stable** — Open Library's search index can return slightly different results for the same query over time. Use `--sort` and `--limit` for reproducible result sets. -- **Author keys from search are relative paths** — Search results return `key` as `/authors/OL23919A`. The CLI strips the prefix so you can use the bare key (e.g. `OL23919A`) directly with the `author` command. -- **Biographies and descriptions may be nested** — The Open Library API sometimes returns bio/description as a dict with a `value` key instead of a plain string (e.g. `{"type": "/type/text", "value": "..."}`). The CLI handles this automatically. -- **No lazy auth** — Unlike other CLI skills, no authentication setup is needed at all. The API is fully public, so `--help`, `--dry-run`, and all commands work without any env vars. -- **Pagination** — Use `--offset` to paginate through search results. The CLI defaults to 20 results and does not auto-paginate. -- **Work vs Edition** — A "work" is the abstract creative work (e.g. "Dune"). An "edition" is a specific published version (e.g. the 1965 hardcover). The `work` command returns work-level data; the `isbn` command returns edition-level data. The `search` command returns work-level results with edition counts. +- **ISBN endpoints answer 302, not JSON** — `/isbn/.json`, `/lccn/*.json`, + `/oclc/*.json` redirect to `/books/.json`. Clients must follow redirects + (`curl -L`; `requests` does by default) or they parse an HTML redirect page. + Direct `/books/.json` calls return 200 immediately. +- **Merged keys return redirect stubs inside HTTP 200** — Open Library is a wiki; + when duplicate works merge, the old key keeps answering 200 with + `{"type":{"key":"/type/redirect"},"location":"/works/"}` instead of a + 3xx. Detect the stub type in success responses and re-fetch. The bundled CLI + does this automatically (and appends `.json` to stub locations, since + extension-less URLs redirect to HTML pages). +- **Key types are encoded in the suffix** — `OL…M` = edition (`/books/`), + `OL…W` = work (`/works/`), `OL…A` = author (`/authors/`). Works link to + authors double-nested (`authors[].author.key`); editions nest flat + (`authors[].key`) — or ship `authors:null` entirely, recovering names from the + linked work. Requesting a key under the wrong collection yields a 301 reroute. +- **Search empties are not errors** — malformed queries parse loosely and come + back HTTP 200 with `numFound:0`; conversely a bad `sort=` enum is a plain-text + HTTP 500 and non-integer `limit` is a FastAPI 422. Branch on bodies, not just + status codes. +- **`availability` needs `ia`** — in raw `/search.json` calls, requesting + `fields=availability` silently returns nothing unless `ia` is also requested. +- **Covers live on another host** — image URLs are always + `covers.openlibrary.org/b/id/-{S,M,L}.jpg` (or `/b/isbn/...`, + `/a/id/...` for author photos). Missing covers return a blank placeholder with + HTTP 200 unless you add `?default=false`; non-ID/non-OLID lookups cap at 100 + req/IP per 5 min then 403; cover URLs often 302 into archive.org zip shards. +- **Rate limits are policy, not headers** — no Retry-After/X-Rate-Limit headers + exist. Anonymous ≈1 req/s; a `User-Agent: App (email)` (set `OL_EMAIL`) + raises it to ≈3 req/s. Batch with one search rather than hundreds of lookups; + bulk belongs in monthly dumps. +- **Text fields nest `{type,value}` objects** — older records wrap `bio`, + `description`, `notes` as `{"type":"/type/text","value":"..."}` while newer + ones use plain strings. Handle both; the CLI unwraps centrally. +- **OLID ≠ long-term identity** — deleted keys can be reassigned to unrelated + books, and merged-away keys become redirect stubs. Pair OLIDs with title/ISBN + in any cache. +- **`.json` placement matters for slugged URLs** — append `.json` to the bare + key (`/authors/OL23919A.json`), never after a slug path. -## References +## When to use -- [scripts/openlibrary](scripts/openlibrary) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, structured logging. -- [Open Library API Docs](https://openlibrary.org/developers/api) — Official API documentation. -- [Open Library](https://openlibrary.org) — The open, editable library catalog. +- Any question about books, authors, or works Open Library's public catalog can answer +- Resolving ISBNs/OLID keys to canonical records and walking edition/work/author graphs +- Finding readable or borrowable scans, cover images, community ratings/shelf counts ## When not to use -Do not use this skill for local library-catalog administration (Koha, Evergreen, MARC batch processing), for licensed commercial book data feeds (ISBNdb, Google Books), or for citation formatting — Open Library is a free public catalog API, not a bibliographic management system. +Do not use this skill for local library-catalog administration — Koha/Evergreen +ILS configuration, MARC batch processing, patron management — or for licensed +commercial data feeds (ISBNdb, Google Books), citation formatting, or managing +your own Open Library reading account/lists (that requires site login this skill +deliberately does not handle). For movie/TV metadata use tmdb instead. + +## Reference Files + +| File | Topic | Read when | +|------|-------|-----------| +| [references/api-overview-and-key-graph.md](references/api-overview-and-key-graph.md) | Access model, rate-limit etiquette, OLID M/W/A key graph, cross-collection 301s, merge-stub handling, `{type,value}` text wrapping | Assembling record URLs by hand, handling merges/wrong-key errors, or planning request pacing | +| [references/search-api-guide.md](references/search-api-guide.md) | `/search.json` parameters, sort keys, field scopes (`title:`, `isbn:`, …), `fields=` projection incl. the availability/ia interaction, `/search/authors.json`, `/subjects/.json`, inside-book search, error model | Building precise queries, paginating deep result sets, or debugging empty results | +| [references/books-isbn-and-covers.md](references/books-isbn-and-covers.md) | Identifier endpoints and their 302 resolution, raw records vs legacy `/api/books` view models, editions listing, ratings/bookshelves shapes, covers-host URL rules and limits | Working with ISBNs/LCCNs/OCLCs, enumerating editions, or fetching images | +| [references/recipes-and-gotchas.md](references/recipes-and-gotchas.md) | End-to-end curl/jq pipelines (ISBN→work→author chain, ebook discovery, cover assembly, disambiguation) plus a symptom-indexed gotcha table | Wiring multi-step workflows or diagnosing an unexpected response | + +## Available Scripts + +| Script | Purpose | Invocation | +|---|---|---| +| `scripts/openlibrary` | The CLI this skill drives: `search`, `search-authors`, `author`, `work`, `isbn`, `editions`, `ratings` — all with `--json`/`--dry-run`, automatic 302 + merge-stub resolution, `{type,value}` unwrapping, covers-host URL assembly, and client-side sort validation. Run it for every book-metadata question above. | `scripts/openlibrary search --query "dune" --json` | +| `scripts/test_openlibrary.py` | Offline pytest/unittest suite covering help, argument errors, dry-run plans, mocked-client logic (redirect stubs, author fallback, editions paging), plus two env-guarded live probes (`OPENLIBRARY_LIVE_TESTS=1`). Zero egress otherwise. | `.venv/bin/python3 -m pytest -p no:cacheprovider --strict-markers scripts/test_openlibrary.py` | + +## Prerequisites + +- Python 3.8+ with `requests` (stdlib otherwise); invoke as `python3 scripts/openlibrary ...` if not executable directly +- No credentials of any kind; optional `OL_EMAIL` for rate-limit etiquette +- `jq` recommended for `--json` post-processing From 1d95c4b0200a8a8c1b2fb98e1083a93f4b7d9403 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 08:44:48 -0400 Subject: [PATCH 18/40] chore(catalog): sync openlibrary blurbs and regenerate catalogs Root README blurb and skill-triggers row updated to match the new frontmatter description; marketplace.json and llms.txt regenerated via gen-*.rb --write (codex/agents outputs unchanged). Check modes all green. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/marketplace.json | 2 +- README.md | 2 +- llms.txt | 2 +- references/skill-triggers.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index fae3325..1fb6a5c 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -777,7 +777,7 @@ "./openlibrary" ], "strict": false, - "description": "Search books, authors, and works on Open Library from the terminal. Look up books by ISBN, search titles and authors, and fetch detailed work/author records via the public Open Library API. No API key required." + "description": "Query the Open Library catalog from the terminal: search books and authors, look up works, editions, and ISBNs, enumerate every edition of a work, read community ratings, and resolve cover-image URLs. Fully keyless public API. Includes the OL…M/W/A key-graph reference, ISBN 302-redirect resolution, search query syntax, covers-host rules, and worked pipelines. Do not use for library-IT administration (Koha/MARC/ILS migration), commercial book-data feeds, or managing your reading account on Open Library itself." }, { "name": "opensource-contributions", diff --git a/README.md b/README.md index 01cb805..219be84 100644 --- a/README.md +++ b/README.md @@ -357,7 +357,7 @@ Google's Open Knowledge Format (OKF) v0.1 — create, validate, and consume vend ### [openlibrary](openlibrary/SKILL.md) -Open Library book metadata from the terminal. Search books and authors, get work and edition details, lookup by ISBN. No API key required — the public Open Library API is free for everyone. +Query the Open Library catalog from the terminal: search books and authors, look up works, editions, and ISBNs, enumerate every edition of a work, read community ratings, and resolve cover-image URLs. Fully keyless public API — no API key required. ### [opensource-contributions](opensource-contributions/SKILL.md) diff --git a/llms.txt b/llms.txt index d50f9e4..d4c96aa 100644 --- a/llms.txt +++ b/llms.txt @@ -87,7 +87,7 @@ - [notion](notion/SKILL.md): Operate Notion from a terminal or agent: retrieve pages, query databases, search pages and databases, and update page properties — with a bundled notion-cli script that is read-only by default and gates every create or update behind a --dry-run/--yes confirmation. Use when an agent needs to read Notion content, answer questions from a team wiki or database, or make a confirmed edit. Do not use for building Notion integrations or block-level page composition beyond property updates (that is Notion API application development), or for other knowledge bases (that is their own tooling). - [nous-branding](nous-branding/SKILL.md): Generate images and content consistent with the Nous Research brand identity. Use when creating visuals in the Nous / Theia / Hermes ecosystem: a "cyber-classical" style blending neo-classical statuary, cyberpunk/industrial grunge, and retro anime illustration. Covers official brand color palette, typography (Inter/IBM Plex Sans, JetBrains Mono, heavy distressed display faces), the Nous Girl mascot, texture system, and image prompt construction. Ships reference images for palette, mascot, and brand collage that can be used as img2img inputs. - [open-knowledge-format](open-knowledge-format/SKILL.md): Google's Open Knowledge Format (OKF) v0.1 — an open, vendor-neutral spec for representing knowledge as markdown files with YAML frontmatter, designed for AI agent consumption. Use when the user mentions OKF, Open Knowledge Format, Google's knowledge format, LLM wiki bundles, agent knowledge packs, creating OKF bundles, validating OKF documents, or converting knowledge into the OKF standard. -- [openlibrary](openlibrary/SKILL.md): Search books, authors, and works on Open Library from the terminal. Look up books by ISBN, search titles and authors, and fetch detailed work/author records via the public Open Library API. No API key required. +- [openlibrary](openlibrary/SKILL.md): Query the Open Library catalog from the terminal: search books and authors, look up works, editions, and ISBNs, enumerate every edition of a work, read community ratings, and resolve cover-image URLs. Fully keyless public API. Includes the OL…M/W/A key-graph reference, ISBN 302-redirect resolution, search query syntax, covers-host rules, and worked pipelines. Do not use for library-IT administration (Koha/MARC/ILS migration), commercial book-data feeds, or managing your reading account on Open Library itself. - [opensource-contributions](opensource-contributions/SKILL.md): Make good open source contributions — check CONTRIBUTING.md first, follow project norms, be a good citizen. Covers bug reports, feature requests, and pull requests with a defensible default posture when the project hasn't documented expectations. - [operational-design](operational-design/SKILL.md): Design and improve operational processes and organizational scaling — process design, operational metrics, compliance and audit, vendor management, and team topology. Covers value stream mapping, BPMN, bottleneck analysis, scaling from 10 to 100 to 1000 people, KPI design, balanced scorecard, SOC 2, ISO 27001, GDPR readiness, RFP processes, SLA design, vendor scorecards, team topologies, Conway's Law, and Dunbar's Number. Do not use for engineering delivery, financial modeling, or technology evaluation. - [org-design](org-design/SKILL.md): CHRO methodology — organizational design (team topologies, span of control, reporting structures), talent strategy (make-vs-buy, skill taxonomies, succession planning), compensation frameworks (market benchmarking, equity design, leveling), culture architecture (values codification, rituals, psychological safety), organizational health metrics (eNPS, retention risk, engagement surveys), DEI strategy (inclusive design, equitable systems, belonging). diff --git a/references/skill-triggers.md b/references/skill-triggers.md index 48d0983..c7a17f1 100644 --- a/references/skill-triggers.md +++ b/references/skill-triggers.md @@ -21,7 +21,7 @@ Each skill's `description` field is the canonical routing contract. This conveni | "Ghost", "Ghost CMS", "ghost blog", "create a post on my blog", "blog publishing", "GHOST_ADMIN_KEY" | [ghost](../ghost/SKILL.md) | | "Jellyfin", "Jellyfin media server", "recently added movies", "recently added episodes", "media library", "JELLYFIN_API_KEY" | [jellyfin](../jellyfin/SKILL.md) | | "Jira", "Atlassian Jira", "JQL", "ticket PROJ-123", "sprint work", "JIRA_API_TOKEN" | [jira](../jira/SKILL.md) | -| "Open Library", "openlibrary", "book search", "ISBN lookup", "author records", "work details" | [openlibrary](../openlibrary/SKILL.md) | +| "Open Library", "openlibrary", "book search", "ISBN lookup", "author records", "work details", "book editions", "book ratings", "cover image" | [openlibrary](../openlibrary/SKILL.md) | | "weather", "forecast", "temperature", "is it raining", "Tempest" | [tempest](../tempest/SKILL.md) | | "TMDb", "The Movie Database", "movie search", "trending movies", "upcoming TV releases", "TMDB_ACCESS_TOKEN" | [tmdb](../tmdb/SKILL.md) | | "traefik", "reverse proxy", "load balancer", "API gateway", "Let's Encrypt", "ACME", "Docker routing", "traefik.yml", "entry point", "middleware", "TLS termination", "forward auth", "rate limit" | [traefik](../traefik/SKILL.md) | From 8cf85fe0a6211acedfa0ef40ca719cf4071be95a Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 09:52:28 -0400 Subject: [PATCH 19/40] docs(tmdb): thicken API skill Add researched TMDb references, external-ID and detail commands, offline tests, and eval coverage. Refresh the human README and generated catalogs. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/marketplace.json | 2 +- llms.txt | 2 +- tmdb/README.md | 41 +++-- tmdb/SKILL.md | 169 ++++++++---------- tmdb/evals/evals.json | 42 +++++ tmdb/references/auth-pagination-and-errors.md | 36 ++++ tmdb/references/find-and-details.md | 34 ++++ tmdb/references/search-discover-trending.md | 27 +++ tmdb/scripts/test_tmdb.py | 67 +++++++ tmdb/scripts/tmdb | 45 ++++- 10 files changed, 350 insertions(+), 115 deletions(-) create mode 100644 tmdb/evals/evals.json create mode 100644 tmdb/references/auth-pagination-and-errors.md create mode 100644 tmdb/references/find-and-details.md create mode 100644 tmdb/references/search-discover-trending.md create mode 100644 tmdb/scripts/test_tmdb.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 1fb6a5c..d2cb22d 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1272,7 +1272,7 @@ "./tmdb" ], "strict": false, - "description": "Search and discover movies, TV shows, and trending content via The Movie Database (TMDb) API v3. Use when the user asks about movies, TV, film, cinema, genres, certifications, ratings, cast, upcoming releases, or trending media." + "description": "Query TMDb metadata for films and television, then enrich results with details, credits, providers, and external IDs. Do not use this skill for torrent search, streaming playback, or personal watch-history tracking." }, { "name": "traefik", diff --git a/llms.txt b/llms.txt index d4c96aa..91678a4 100644 --- a/llms.txt +++ b/llms.txt @@ -142,7 +142,7 @@ - [tempest](tempest/SKILL.md): Query hyper-local weather from a WeatherFlow Tempest station: current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. Use when the user asks about the weather, temperature, rain, wind, humidity, forecast, or wants conditions from their own station rather than a generic weather service. - [terraform](terraform/SKILL.md): Operate Terraform and OpenTofu across the whole infrastructure lifecycle: module structure, state backends and locking, plan/apply workflow, drift detection, remote state, upgrade and refactor flows, and evidence-based diagnostics. Use when running or inspecting terraform plans, applies, state files, imports, or state surgery, or when the bundled tfops script should handle the task. Do not use for IaC methodology or cloud design decisions - those route up to platform-engineering. - [three](three/SKILL.md): Build browser-based Three.js and WebGL scenes, animations, and interactive 3D visualizations with a small vanilla JavaScript starting point. -- [tmdb](tmdb/SKILL.md): Search and discover movies, TV shows, and trending content via The Movie Database (TMDb) API v3. Use when the user asks about movies, TV, film, cinema, genres, certifications, ratings, cast, upcoming releases, or trending media. +- [tmdb](tmdb/SKILL.md): Query TMDb metadata for films and television, then enrich results with details, credits, providers, and external IDs. Do not use this skill for torrent search, streaming playback, or personal watch-history tracking. - [traefik](traefik/SKILL.md): Deploy, configure, and troubleshoot Traefik v3 reverse proxy — covers all providers, routing, TLS/ACME, middlewares, and production patterns with YAML examples. Load when setting up or debugging a Traefik instance. - [trakt](trakt/SKILL.md): Discover trending, anticipated, and popular movies and TV shows via the Trakt.tv API from the terminal. No authentication required for read-only discovery. Use when the user asks about what to watch, trending movies, popular shows, or media discovery. - [transistor](transistor/SKILL.md): Manage Transistor.fm podcast hosting from the terminal: view shows, list episodes, check analytics, and get subscriber counts. Use when the user mentions Transistor, podcast hosting, podcast analytics, show management, or episode tracking. diff --git a/tmdb/README.md b/tmdb/README.md index 9035cfa..81031e8 100644 --- a/tmdb/README.md +++ b/tmdb/README.md @@ -1,36 +1,41 @@ -# TMDb — Movie & TV Discovery from the Terminal - -Search movies and TV shows by keyword, discover by genre/certification/rating/date, check trending and upcoming releases, and browse genre lists. +# TMDb Metadata Skill ## Why Install This Skill -When your agent loads this skill, it can **access the entire TMDb catalog** without a browser. That means: +Give your agent a dependable terminal workflow for exploring movie and television metadata without hand-building every HTTP request. It can start from a title, an IMDb ID, or a discovery filter, then enrich the result with credits, recommendations, images, and provider metadata. -- **Search movies and TV** — by keyword with release year and ratings -- **Discover by taste** — genre, certification, rating threshold, date range -- **Find trending content** — what's popular right now -- **Check upcoming releases** — what's coming to theaters -- **Browse certifications** — US ratings (G, PG, PG-13, R, NC-17) +The skill also makes TMDb's easy-to-miss rules visible: choose one authentication mode, respect the 500-page ceiling, use the correct nested `/find` response, and URL-encode compound provider paths. ## What You Get -| Directory | Purpose | -|-----------|---------| -| `SKILL.md` | Complete command reference with compound filter examples | -| `scripts/tmdb` | CLI tool for TMDb v3 API | +| Path | Purpose | +| --- | --- | +| `SKILL.md` | Setup, commands, recipes, gotchas, and routing | +| `scripts/tmdb` | Executable JSON-capable CLI for search, detail, find, discovery, and trends | +| `scripts/test_tmdb.py` | Offline pytest/unittest coverage with mocked HTTP behavior | +| `references/auth-pagination-and-errors.md` | Authentication, pagination, errors, rate limits, and image construction | +| `references/find-and-details.md` | External IDs, IMDb entry points, details, credits, and compound responses | +| `references/search-discover-trending.md` | Search, discovery filters, trending, genres, and certifications | +| `evals/evals.json` | Runnable examples covering normal and negative routing | ## Quick Start ```bash -export TMDB_ACCESS_TOKEN="your-tmdb-access-token" -tmdb movie search --term "dune" -tmdb movie discover --genre horror --certification R +export TMDB_ACCESS_TOKEN="YOUR_ACCESS_TOKEN" +tmdb movie search --term "Dune" --limit 5 --json +tmdb find tt0111161 --source imdb_id --json +tmdb movie detail 550 --append credits,videos --json ``` ## Triggers -Load this for movies, TV shows, film discovery, genre browsing, or media recommendations. +Load this skill when the request involves movie or TV metadata, title search, IMDb/TVDB resolution, credits, release dates, certifications, recommendations, images, trending media, or provider metadata. ## Requirements -Python 3.8+ with `requests` library. Free API key from themoviedb.org. +- Python 3.8 or newer +- `requests` Python package +- A TMDb API Read Access Token or v3 API key +- `jq` for the shell pipeline examples + +This is a read-oriented metadata workflow. It does not play media, search torrents, or maintain personal watch history. diff --git a/tmdb/SKILL.md b/tmdb/SKILL.md index 8ef2dc9..a3447c3 100644 --- a/tmdb/SKILL.md +++ b/tmdb/SKILL.md @@ -1,133 +1,122 @@ --- name: tmdb -description: Search and discover movies, TV shows, and trending content via The Movie - Database (TMDb) API v3. Use when the user asks about movies, TV, film, cinema, genres, - certifications, ratings, cast, upcoming releases, or trending media. +description: Query TMDb metadata for films and television, then enrich results with details, credits, providers, and external IDs. Do not use this skill for torrent search, streaming playback, or personal watch-history tracking. license: MIT -compatibility: Requires TMDB_ACCESS_TOKEN or TMDB_API_KEY env var (free at themoviedb.org/settings/api), - Python 3.8+, and the `requests` library. +compatibility: Requires TMDB_ACCESS_TOKEN or TMDB_API_KEY, Python 3.8+, and requests. metadata: - tags: tmdb, movies, tv, film, cinema, entertainment, media-discovery, api-client - sources: https://developer.themoviedb.org/reference, https://www.themoviedb.org/settings/api + tags: tmdb, movies, tv, film, cinema, metadata + sources: https://developer.themoviedb.org/reference --- -# tmdb — Movie & TV Discovery from the Terminal - -Search movies and TV shows by keyword, discover by genre/certification/rating/date, check trending and upcoming releases, browse genre lists, and view US certification ratings — all from TMDb's v3 API. +# TMDb metadata from the terminal ## Setup -1. Get a free API key or access token at [themoviedb.org/settings/api](https://www.themoviedb.org/settings/api) -2. Set one of these environment variables: +Create credentials at [TMDb API settings](https://www.themoviedb.org/settings/api). Prefer the API Read Access Token: ```bash -export TMDB_ACCESS_TOKEN="your-tmdb-access-token" # preferred -# OR -export TMDB_API_KEY="your-tmdb-api-key" +export TMDB_ACCESS_TOKEN="YOUR_ACCESS_TOKEN" +# Or use the v3 key: export TMDB_API_KEY="YOUR_API_KEY" ``` -`--help` and `--dry-run` work without credentials (lazy auth). +The CLI sends either `Authorization: Bearer $TMDB_ACCESS_TOKEN` or `?api_key=$TMDB_API_KEY`. Both forms have the same v3 access level; configure only one. `--help` and `--dry-run` do not need credentials. -## Essential Commands +## Essential commands -### movie search — Search movies by keyword +### Search and identify ```bash -tmdb movie search --term "dune" # basic search -tmdb movie search --term "inception" --limit 5 # top 5 results -tmdb movie search --term "arrival" --json # machine-readable +tmdb movie search --term "dune" --limit 5 --json +tmdb tv search --term "severance" --limit 5 +tmdb find tt0111161 --source imdb_id --json ``` -Shows: title, release year, vote average. +`find` accepts the current external sources `imdb_id`, `tvdb_id`, `wikidata_id`, `facebook_id`, `instagram_id`, `tiktok_id`, `twitter_id`, and `youtube_id`. Its response is split into `movie_results`, `tv_results`, `person_results`, `tv_season_results`, and `tv_episode_results`. -### movie discover — Discover movies by genre, certification, rating, and date +### Details and enrichment ```bash -tmdb movie discover --genre horror # horror movies -tmdb movie discover --genre horror --certification R # horror, R-rated -tmdb movie discover --genre comedy --rating 7 --limit 15 # highly-rated comedy -tmdb movie discover --from 2024-01-01 --to 2024-12-31 # released in 2024 -tmdb movie discover --genre scifi --from 2026-05-01 # recent sci-fi -tmdb movie discover --genre thriller --certification R \ - --rating 6 --from 2025-01-01 --limit 20 # compound filter +tmdb movie detail 550 --append credits,videos --json +tmdb movie detail 550 --append 'credits,watch/providers,external_ids' --json ``` -### movie upcoming — Upcoming movie releases +Compound responses use the requested names as top-level keys. Encode the slash in `watch/providers` when constructing raw URLs. + +### Discover and browse ```bash -tmdb movie upcoming # next 10 upcoming -tmdb movie upcoming --limit 20 # more results -tmdb movie upcoming --json # machine-readable -``` - -### tv search — Search TV shows by keyword - -```bash -tmdb tv search --term "severance" # basic TV search -tmdb tv search --term "the expanse" --limit 5 -tmdb tv search --term "silo" --json -``` - -Shows: name, first air year, vote average. - -### tv discover — Discover TV shows by genre, rating, and air date - -```bash -tmdb tv discover --genre sci-fi # sci-fi shows -tmdb tv discover --genre drama --rating 7 # critically-acclaimed drama -tmdb tv discover --genre comedy --from 2025-01-01 # recent comedy -``` - -### trending — Trending content across day or week - -```bash -tmdb trending # trending movies this week -tmdb trending --type tv # trending TV this week -tmdb trending --type all --window day # all media trending today -tmdb trending --limit 20 --json # top 20 as JSON -``` - -### genre list — Browse available genres - -```bash -tmdb genre list --type movie # all movie genres -tmdb genre list --type tv # all TV genres +tmdb movie discover --genre horror --rating 7 --limit 10 +tmdb movie discover --genre horror --certification R --from 2024-01-01 --to 2024-12-31 +tmdb trending --type all --window week --limit 20 --json tmdb genre list --type movie --json +tmdb certification --json ``` -### certification — View US movie certification ratings +Use `vote_count.gte` with `vote_average.desc` in raw discover requests so a title with very few votes does not dominate. In current TMDb docs, comma-separated genre IDs are AND and pipe-separated IDs are OR. + +## Pipeline recipes + +### IMDb ID to enriched movie + +1. Resolve the IMDb identifier: ```bash -tmdb certification # US certification list -tmdb certification --json # machine-readable +curl -s -H "Authorization: Bearer $TMDB_ACCESS_TOKEN" \ + 'https://api.themoviedb.org/3/find/tt0111161?external_source=imdb_id' > /tmp/find.json +id=$(jq -r '.movie_results[0].id' /tmp/find.json) ``` -## Global Flags - -These flags work in any position before, between, or after subcommands: +2. Fetch details and compound resources: ```bash -tmdb --json movie search --term "dune" # JSON output -tmdb movie search --term "dune" --json # json after subcommand -tmdb --dry-run movie discover --genre horror # preview without API call -tmdb --quiet trending # suppress diagnostic output -tmdb --verbose movie search --term "alien" # verbose logging +curl -s -H "Authorization: Bearer $TMDB_ACCESS_TOKEN" \ + "https://api.themoviedb.org/3/movie/$id?append_to_response=credits,videos,watch%2Fproviders" \ + | jq '{title, runtime, director: [.credits.crew[] | select(.job == "Director") | .name], cast: [.credits.cast[0:5][].name], providers: .["watch/providers"].results.US}' ``` -## Known Gotchas +### Search then detail -- **Genre name matching is case-insensitive** — `--genre Horror`, `--genre horror`, and `--genre HORROR` all work. Names are matched via substring, so `--genre sci` matches "Sci-Fi" and "Science Fiction". -- **Certifications are US-only** — The `--certification` flag and the `certification` subcommand only return/accept US ratings (G, PG, PG-13, R, NC-17). International certifications are not available. -- **API version** — This CLI wraps TMDb API v3. Endpoints and response shapes follow the v3 spec. -- **Pagination defaults** — Every command defaults to 10 results. Use `--limit` to get more. The CLI does not auto-paginate beyond the first page. -- **Now-playing is defined** — The `movie now-playing` subcommand is registered in argparse and maps to the TMDb `/movie/now_playing` endpoint. +```bash +tmdb movie search --term "dune" --limit 1 --json > /tmp/search.json +id=$(jq -r '.results[0].id' /tmp/search.json) +tmdb movie detail "$id" --append recommendations,similar --json +``` -## References +### Filter reliable discoveries -- [scripts/tmdb](scripts/tmdb) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. -- [TMDb API v3 Reference](https://developer.themoviedb.org/reference) — Official API documentation. -- [TMDb API Settings (get a key)](https://www.themoviedb.org/settings/api) — Free API key registration. +For direct API use, combine a date window, pipe-OR or comma-AND genre expression, `vote_count.gte`, and `sort_by=vote_average.desc`. Then retain only the fields needed by the next workflow step with jq. + +## JSON and jq + +Put `--json` before or after the subcommand. JSON search output has `results` and usually pagination fields `page`, `total_pages`, and `total_results`; the service limits page numbers to 500. Use `jq -r '.results[] | [.id, (.title // .name)] | @tsv'` for stable tabular handoff. + +## Known gotchas + +- **Credential duality:** `api_key` and Bearer are alternatives, not values to mix. A rejected credential commonly produces HTTP 401, `status_code: 7`, and `Invalid API key: You must be granted a valid key.` Permission failures use code 3. Code 33 means an invalid request token, not this API-key message. +- **Pagination ceiling:** pages start at 1 and max at 500; over-limit requests fail. Search/discover access is effectively capped at 10,000 results, even where totals look larger. Rate guidance is around 40 requests/second and 429 responses should honor `Retry-After`. +- **Compound syntax:** append values are comma-separated and limited to 20 calls. `watch/providers` contains a slash, so URL-encode it in curl and use jq's `.\"watch/providers\"` notation. +- **External-ID shape:** `/find/` does not return one generic `id`; inspect the appropriate nested array before choosing movie or TV detail. +- **Provider filters:** `with_watch_providers` requires `watch_region`; provider data carries JustWatch attribution requirements. +- **Localization and images:** use `language=en-US` and a market `region` when reproducibility matters. Build image URLs from `/3/configuration`'s secure base URL, a valid size, and the returned path. + +## When to use + +Use this skill for read-only film and TV metadata discovery, credits, release information, certifications, images, recommendations, and provider metadata. ## When not to use -Do not use this skill for streaming-availability lookups (TMDb delegates watch-provider data to JustWatch and may lag), for torrent or piracy search, or for tracking what you have already watched — TMDb is a metadata database, not a viewing source; pair it with trakt for personal watch history. +Do not use it for torrent or piracy searches, playing or downloading a stream, or maintaining personal watched/unwatched state. Use `trakt` for watch-history workflows and a playback/catalog integration for availability actions. + +## Reference files + +| File | Use it for | +| --- | --- | +| [references/auth-pagination-and-errors.md](references/auth-pagination-and-errors.md) | Credentials, pagination, rate limits, errors, language, regions, and images | +| [references/find-and-details.md](references/find-and-details.md) | IMDb/TVDB lookup, response mapping, detail fields, compound requests | +| [references/search-discover-trending.md](references/search-discover-trending.md) | Search, discover filters, trending, genre, certification, and release lists | + +## Available scripts and prerequisites + +- `scripts/tmdb` is an executable Python CLI using only the standard library and `requests`; it preserves `--json`, `--dry-run`, `--quiet`, and `--verbose`. +- `scripts/test_tmdb.py` is an offline unittest/pytest suite; all HTTP behavior is mocked. +- Requires Python 3.8+ and `requests`. No service is started by this skill. diff --git a/tmdb/evals/evals.json b/tmdb/evals/evals.json new file mode 100644 index 0000000..f988f20 --- /dev/null +++ b/tmdb/evals/evals.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "skill_name": "tmdb", + "evals": [ + { + "id": "search-movie", + "prompt": "Find the top five TMDb movie results for Dune.", + "expected_output": "Use tmdb movie search with --term and --limit, then inspect JSON results.", + "assertions": ["uses movie search", "limits results"] + }, + { + "id": "imdb-find-detail-pipeline", + "prompt": "Starting from IMDb tt0111161, find the TMDb movie and fetch credits and providers.", + "expected_output": "Call /find/{external_id} with external_source=imdb_id, extract movie_results[0].id, then request movie details with append_to_response.", + "assertions": ["documents IMDb entry point", "extracts movie_results id", "uses append_to_response"] + }, + { + "id": "auth-mode-gotcha", + "prompt": "Explain TMDb API key versus read access token authentication and diagnose a 401.", + "expected_output": "Use either api_key query authentication or Authorization Bearer, not both; inspect status_code 7 and status_message.", + "assertions": ["distinguishes v3 key and bearer", "names 401 symptom"] + }, + { + "id": "discover-rated-movies", + "prompt": "Discover highly rated horror movies released in a date window.", + "expected_output": "Use discover/movie with genre, vote_average.gte, vote_count.gte, and release date filters, then process JSON with jq.", + "assertions": ["uses discover filters", "guards vote averages with vote count"] + }, + { + "id": "not-for-torrents", + "prompt": "Search torrent sites for a movie download.", + "expected_output": "Do not route this to TMDb; it is a piracy or torrent-search request rather than metadata discovery.", + "assertions": ["must not trigger tmdb", "refuses torrent search"] + }, + { + "id": "trending-json", + "prompt": "Show movies trending this week as machine-readable JSON.", + "expected_output": "Run tmdb trending with --window week and --json.", + "assertions": ["uses trending endpoint", "uses json output"] + } + ] +} diff --git a/tmdb/references/auth-pagination-and-errors.md b/tmdb/references/auth-pagination-and-errors.md new file mode 100644 index 0000000..8a4d026 --- /dev/null +++ b/tmdb/references/auth-pagination-and-errors.md @@ -0,0 +1,36 @@ +# TMDb Authentication, Pagination, and Errors + +## Choose one application credential + +TMDb v3 accepts either `api_key` as a query parameter or an API Read Access Token in `Authorization: Bearer `. Both methods provide the same access level across v3; the read token also works across v4. Obtain both from the account API settings page. Send one method, not both, so an accidental stale query key cannot obscure a rejected bearer token. + +```bash +curl -H 'accept: application/json' \ + -H "Authorization: Bearer $TMDB_ACCESS_TOKEN" \ + 'https://api.themoviedb.org/3/movie/550' +# Alternative: .../movie/550?api_key=$TMDB_API_KEY +``` + +A bad credential commonly returns HTTP 401 with `status_code: 7` and `Invalid API key: You must be granted a valid key.` Permission failures use code 3 and `Authentication failed: You do not have permissions to access the service.` Do not confuse code 33, which is an invalid request token. The CLI reports the 401 response rather than retrying with a second credential. + +## Pages and rate limits + +Search and discover responses contain `page`, `results`, `total_pages`, and `total_results`; pages contain up to 20 results. Page numbers start at 1 and max out at 500. Requests beyond that limit return a validation error, rather than being silently clamped. Search/discover access is effectively capped at 10,000 items even when totals advertise more. Trending has a larger documented sample ceiling. + +TMDb's current guidance describes a soft limit around 40 requests per second, subject to change. On HTTP 429, respect `Retry-After`; the service may also expose `X-RateLimit-Limit`, `X-RateLimit-Remaining`, and `X-RateLimit-Reset`. Exponential backoff is safer than tight retry loops. + +## Parameters and images + +Use `language=en-US` (an ISO 639-1 language plus ISO 3166-1 region) for deterministic localized fields. `region=US` selects or filters release dates for that market. Image URLs combine the secure base URL from `/3/configuration`, a valid size, and the returned path: `https://image.tmdb.org/t/p/w500/`. Common poster sizes include `w92`, `w185`, `w342`, `w500`, `w780`, and `original`; backdrop sizes differ. + +## Sources + +- https://developer.themoviedb.org/docs/authentication-application +- https://developer.themoviedb.org/reference/authentication +- https://www.themoviedb.org/documentation/api/status-codes +- https://developer.themoviedb.org/docs/rate-limiting +- https://developer.themoviedb.org/reference/search-movie +- https://developer.themoviedb.org/docs/languages +- https://developer.themoviedb.org/docs/region-support +- https://developer.themoviedb.org/docs/image-basics +- https://developer.themoviedb.org/reference/configuration-details diff --git a/tmdb/references/find-and-details.md b/tmdb/references/find-and-details.md new file mode 100644 index 0000000..fc40de6 --- /dev/null +++ b/tmdb/references/find-and-details.md @@ -0,0 +1,34 @@ +# External IDs, Details, and Compound Responses + +## Start with an IMDb ID + +`GET /3/find/{external_id}?external_source=imdb_id` maps a foreign identifier to TMDb objects. The current `external_source` enum includes `imdb_id`, `facebook_id`, `instagram_id`, `tvdb_id`, `tiktok_id`, `twitter_id`, `wikidata_id`, and `youtube_id`; older Freebase values are obsolete. The response has `movie_results`, `person_results`, `tv_results`, `tv_episode_results`, and `tv_season_results` arrays. Unmatched categories are empty arrays. For an IMDb movie, extract `.movie_results[0].id` before calling the movie details endpoint. + +```bash +curl -s -H "Authorization: Bearer $TMDB_ACCESS_TOKEN" \ + 'https://api.themoviedb.org/3/find/tt0111161?external_source=imdb_id' \ + | jq -r '.movie_results[0].id' +``` + +## Details and append_to_response + +Movie details expose fields such as `title`, `overview`, `genres`, `runtime`, `release_date`, `vote_average`, `vote_count`, `budget`, `revenue`, `imdb_id`, and production companies. TV details use `name`, `first_air_date`, `number_of_seasons`, `number_of_episodes`, `created_by`, `networks`, and `genres`. + +Details endpoints accept `append_to_response`, a comma-separated list of sub-endpoints within the same namespace, with a maximum of 20 appended calls. Common movie tokens include `credits`, `images`, `videos`, `recommendations`, `similar`, `reviews`, `release_dates`, `watch/providers`, `external_ids`, `alternative_titles`, and `translations`; TV adds `aggregate_credits` and `content_ratings`. Encode the slash when needed (`watch%2Fproviders`). Returned keys mirror the requested token, so jq accesses the provider object as `."watch/providers"`. + +```bash +curl -s -H "Authorization: Bearer $TMDB_ACCESS_TOKEN" \ + 'https://api.themoviedb.org/3/movie/550?append_to_response=credits,videos,watch%2Fproviders' \ + | jq '{title, runtime, director: [.credits.crew[] | select(.job == "Director") | .name], cast: [.credits.cast[0:5][].name], us: .["watch/providers"].results.US}' +``` + +Credits contain `cast[]` (including `id`, `name`, `character`, `order`) and `crew[]` (including `department`, `job`). Watch-provider regions contain `link`, `flatrate`, `rent`, and `buy` arrays. Release dates nest under `results[].release_dates[]`; content ratings nest under `results[]`. TMDb requires attribution and a link to JustWatch when displaying provider data. + +## Sources + +- https://developer.themoviedb.org/reference/find-by-id +- https://developer.themoviedb.org/reference/movie-details +- https://developer.themoviedb.org/reference/movie-credits +- https://developer.themoviedb.org/reference/movie-watch-providers +- https://developer.themoviedb.org/reference/movie-release-dates +- https://developer.themoviedb.org/reference/tv-content-ratings diff --git a/tmdb/references/search-discover-trending.md b/tmdb/references/search-discover-trending.md new file mode 100644 index 0000000..08f6ece --- /dev/null +++ b/tmdb/references/search-discover-trending.md @@ -0,0 +1,27 @@ +# Search, Discover, Trending, and Lists + +## Search + +Use `/search/movie` with required `query`; `include_adult` defaults to false. `/search/tv` supports `first_air_date_year`, while `/search/multi` combines movie, TV, and person results. Search responses expose `page`, `results`, `total_pages`, and `total_results`. Keep `language=en-US` explicit when scripts need stable output. + +## Discover + +`/discover/movie` and `/discover/tv` filter catalog metadata. Useful movie filters include `with_genres`, `vote_count.gte`, `vote_average.gte`, `primary_release_date.gte/lte`, and certification fields. TV uses `first_air_date.gte/lte`; discover TV does not expose movie certification filters. The current docs state that comma-separated genre IDs are an AND query and pipe-separated IDs are an OR query. Provider filters such as `with_watch_providers` require `watch_region`. + +Avoid sorting only by `vote_average.desc`: require a meaningful `vote_count.gte` threshold or a tiny-vote title can dominate. Upcoming and now-playing lists are specialized release-date views; `region` controls the market. + +## Trending and lists + +Trending uses `/trending/{all|movie|tv|person}/{day|week}`. `all` results carry `media_type`, which lets a consumer branch to movie or TV detail calls. Genre lists return `{genres: [{id, name}]}`. Certification lists group entries under `certifications.US` (with certification, meaning, and order). + +## Sources + +- https://developer.themoviedb.org/reference/search-movie +- https://developer.themoviedb.org/reference/search-tv +- https://developer.themoviedb.org/reference/search-multi +- https://developer.themoviedb.org/reference/discover-movie +- https://developer.themoviedb.org/reference/discover-tv +- https://developer.themoviedb.org/reference/trending-all +- https://developer.themoviedb.org/reference/genre-movie-list +- https://developer.themoviedb.org/reference/certification-movie-list +- https://developer.themoviedb.org/docs/region-support diff --git a/tmdb/scripts/test_tmdb.py b/tmdb/scripts/test_tmdb.py new file mode 100644 index 0000000..e5540cf --- /dev/null +++ b/tmdb/scripts/test_tmdb.py @@ -0,0 +1,67 @@ +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import sys +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +SCRIPT = Path(__file__).with_name("tmdb") + + +def load_cli(): + loader = importlib.machinery.SourceFileLoader("tmdb_cli", str(SCRIPT)) + spec = importlib.util.spec_from_loader("tmdb_cli", loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TmdbCliTests(unittest.TestCase): + def run_cli(self, *args): + env = os.environ.copy() + env.pop("TMDB_ACCESS_TOKEN", None) + env.pop("TMDB_API_KEY", None) + return subprocess.run([str(SCRIPT), *args], text=True, capture_output=True, env=env) + + def test_help_lists_entry_points(self): + result = self.run_cli("--help") + self.assertEqual(result.returncode, 0) + self.assertIn("find", result.stdout) + self.assertIn("movie", result.stdout) + + def test_missing_required_search_term_is_argument_error(self): + result = self.run_cli("movie", "search") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--term", result.stderr) + + def test_dry_run_find_emits_json_without_credentials(self): + result = self.run_cli("--dry-run", "--json", "find", "tt0111161") + self.assertEqual(result.returncode, 0) + self.assertTrue(json.loads(result.stdout)["dry_run"]) + + def test_mocked_external_lookup_uses_source_and_parses_results(self): + cli = load_cli() + client = cli.TMDBClient() + client.find_external = Mock(return_value={"movie_results": [{"id": 550, "title": "Fight Club"}]}) + cli.GLOBAL_FLAGS = {"json": True, "dry_run": False, "quiet": False, "verbose": False} + with patch("builtins.print") as printed: + cli.cmd_find(client, ["tt0137523", "--source", "imdb_id"]) + payload = json.loads(printed.call_args.args[0]) + self.assertEqual(payload["movie_results"][0]["id"], 550) + client.find_external.assert_called_once_with("tt0137523", "imdb_id") + + def test_mocked_detail_passes_append_to_response(self): + cli = load_cli() + client = cli.TMDBClient() + client.get_movie = Mock(return_value={"id": 550, "title": "Fight Club", "credits": {"cast": []}}) + cli.GLOBAL_FLAGS = {"json": True, "dry_run": False, "quiet": False, "verbose": False} + with patch("builtins.print"): + cli.cmd_movie_detail(client, ["550", "--append", "credits,videos"]) + client.get_movie.assert_called_once_with("550", "credits,videos") + + +if __name__ == "__main__": + unittest.main() diff --git a/tmdb/scripts/tmdb b/tmdb/scripts/tmdb index 0257003..c8eba31 100755 --- a/tmdb/scripts/tmdb +++ b/tmdb/scripts/tmdb @@ -127,11 +127,16 @@ class TMDBClient: def get_trending(self, media_type="movie", window="week"): return self._get(f"/trending/{media_type}/{window}") - def get_movie(self, movie_id): - return self._get(f"/movie/{movie_id}") + def get_movie(self, movie_id, append=None): + params = {"append_to_response": append} if append else None + return self._get(f"/movie/{movie_id}", params) - def get_tv(self, tv_id): - return self._get(f"/tv/{tv_id}") + def get_tv(self, tv_id, append=None): + params = {"append_to_response": append} if append else None + return self._get(f"/tv/{tv_id}", params) + + def find_external(self, external_id, source, language="en-US"): + return self._get(f"/find/{external_id}", {"external_source": source, "language": language}) def get_movie_genres(self, lang="en-US"): return self._get("/genre/movie/list", {"language": lang}) @@ -185,6 +190,31 @@ def cmd_movie_search(client, args): {"total": data.get("total_results"), "results": results}) +def cmd_movie_detail(client, args): + p = argparse.ArgumentParser(prog="tmdb movie detail") + p.add_argument("movie_id") + p.add_argument("--append", default=None, help="comma-separated subresources") + parsed, _ = p.parse_known_args(args) + if client.dry_run: + return emit("[dry-run] Get movie details", {"dry_run": True}) + data = client.get_movie(parsed.movie_id, parsed.append) or {} + emit(data.get("title", "Movie details"), data) + + +def cmd_find(client, args): + p = argparse.ArgumentParser(prog="tmdb find") + p.add_argument("external_id") + p.add_argument("--source", default="imdb_id", choices=[ + "imdb_id", "facebook_id", "instagram_id", "tvdb_id", "tiktok_id", + "twitter_id", "wikidata_id", "youtube_id", + ]) + parsed, _ = p.parse_known_args(args) + if client.dry_run: + return emit("[dry-run] Find external ID", {"dry_run": True}) + data = client.find_external(parsed.external_id, parsed.source) or {} + emit("External ID results", data) + + def cmd_movie_discover(client, args): p = argparse.ArgumentParser(prog="tmdb movie discover") p.add_argument("--genre", help="Genre name (e.g. horror, comedy)") @@ -349,6 +379,7 @@ def main(): msub = mp.add_subparsers(dest="action") s1 = msub.add_parser("search", help="Search movies"); s1.add_argument("--term", "-t", required=True); s1.add_argument("--limit", type=int, default=10) s2 = msub.add_parser("discover", help="Discover movies"); s2.add_argument("--genre"); s2.add_argument("--certification"); s2.add_argument("--rating", type=float); s2.add_argument("--from", dest="release_date_gte"); s2.add_argument("--to", dest="release_date_lte"); s2.add_argument("--limit", type=int, default=10) + sdetail = msub.add_parser("detail", help="Get movie details"); sdetail.add_argument("movie_id"); sdetail.add_argument("--append") msub.add_parser("upcoming", help="Upcoming movies").add_argument("--limit", type=int, default=10) msub.add_parser("now-playing", help="Now playing movies").add_argument("--limit", type=int, default=10) @@ -361,6 +392,7 @@ def main(): # flat sub.add_parser("genre", help="List genres").add_argument("--type", required=True, choices=["movie", "tv"]) sub.add_parser("certification", help="List certifications") + fp = sub.add_parser("find", help="Find by external ID"); fp.add_argument("external_id"); fp.add_argument("--source", default="imdb_id") tr = sub.add_parser("trending", help="Trending content") tr.add_argument("--type", default="movie", choices=["movie", "tv", "all"]) @@ -378,6 +410,7 @@ def main(): if args.resource == "movie": if args.action == "search": cmd_movie_search(client, filtered_argv[filtered_argv.index("search")+1:]) elif args.action == "discover": cmd_movie_discover(client, filtered_argv[filtered_argv.index("discover")+1:]) + elif args.action == "detail": cmd_movie_detail(client, filtered_argv[filtered_argv.index("detail")+1:]) elif args.action == "upcoming": cmd_upcoming(client, filtered_argv[filtered_argv.index("upcoming")+1:]) else: parser.print_help() elif args.resource == "tv": @@ -389,7 +422,9 @@ def main(): elif args.resource == "genre": cmd_genre_list(client, filtered_argv[filtered_argv.index("list")+1:]) elif args.resource == "certification": - cmd_cert_list(client, filtered_argv[filtered_argv.index("list")+1:]) + cmd_cert_list(client, filtered_argv[filtered_argv.index("certification")+1:]) + elif args.resource == "find": + cmd_find(client, filtered_argv[filtered_argv.index("find")+1:]) else: parser.print_help() From 620d2c94a632f73fca3b857eae376d4169ee8381 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 10:00:10 -0400 Subject: [PATCH 20/40] docs(trakt): thicken Trakt API discovery skill Add researched request-header guidance, endpoint references, offline tests, and eval coverage for Trakt discovery workflows. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- trakt/README.md | 41 +++--- trakt/SKILL.md | 139 +++++++++--------- trakt/evals/evals.json | 42 ++++++ trakt/references/auth-and-request-contract.md | 32 ++++ trakt/references/discovery-endpoints.md | 34 +++++ trakt/references/recipes-and-operations.md | 31 ++++ trakt/scripts/test_trakt.py | 69 +++++++++ trakt/scripts/{trakt-cli => trakt} | 7 +- 8 files changed, 306 insertions(+), 89 deletions(-) create mode 100644 trakt/evals/evals.json create mode 100644 trakt/references/auth-and-request-contract.md create mode 100644 trakt/references/discovery-endpoints.md create mode 100644 trakt/references/recipes-and-operations.md create mode 100644 trakt/scripts/test_trakt.py rename trakt/scripts/{trakt-cli => trakt} (96%) diff --git a/trakt/README.md b/trakt/README.md index b27a96a..e49363a 100644 --- a/trakt/README.md +++ b/trakt/README.md @@ -1,37 +1,40 @@ -# Trakt — Media Discovery from the Terminal - -Discover trending, anticipated, and popular movies and TV shows via the Trakt.tv API. Read-only discovery with no user authentication needed. +# Trakt — Media Discovery Signals in the Terminal ## Why Install This Skill -When your agent loads this skill, it can **surface what's worth watching** without a browser. That means: +Give your agent a reliable way to answer "what is everyone watching?" without confusing a current Trakt discovery ranking with a metadata catalog. The skill covers movies and shows that are trending, broadly popular, or anticipated, and produces JSON that can feed media automation. -- **Trending movies and TV** — what everyone's watching right now -- **Most anticipated** — upcoming releases with buzz -- **Popular content** — what's been hot recently -- **No authentication** — just a Client ID, no OAuth flow +Public discovery reads need an application Client ID, not a user login. OAuth boundaries, required headers, pagination, rate-limit behavior, and the difference between Trakt IDs and TMDb metadata are documented so workflows fail clearly instead of silently mixing services. ## What You Get -| Directory | Purpose | -|-----------|---------| -| `SKILL.md` | Complete command reference with examples | -| `scripts/trakt-cli` | CLI tool for Trakt.tv API v2 | +| Path | Purpose | +|---|---| +| `SKILL.md` | Command guide, recipes, gotchas, and routing | +| `scripts/trakt` | Executable CLI with JSON and dry-run modes | +| `scripts/test_trakt.py` | Offline pytest and unittest suite, including header injection | +| `references/auth-and-request-contract.md` | Required headers, OAuth boundary, and errors | +| `references/discovery-endpoints.md` | Trending/popular/anticipated semantics and paging | +| `references/recipes-and-operations.md` | jq pipelines and rate-safe operations | +| `evals/evals.json` | Six representative usage-quality cases | ## Quick Start -```bash -export TRAKT_CLIENT_ID="your-trakt-client-id" -trakt-cli movie trending -trakt-cli tv trending +```sh +export TRAKT_CLIENT_ID="YOUR_TRAKT_CLIENT_ID" +trakt movie trending --limit 10 +trakt --json tv anticipated | jq '.shows' ``` -Client ID from trakt.tv/oauth/applications (free, no OAuth needed). +Create a free Client ID at [trakt.tv/oauth/applications](https://trakt.tv/oauth/applications). Preview commands with `trakt --dry-run --json movie popular` without credentials or network access. ## Triggers -Load this for what to watch, trending movies, popular shows, or media discovery. +Load this skill for Trakt API discovery, trending movies or shows, popular rankings, anticipated releases, watch-signal pipelines, or Trakt pagination and authentication questions. Do not use it for TMDb catalog metadata, credits, images, or provider lookups. ## Requirements -Python 3.8+ with `requests` library. Free Client ID from trakt.tv. +- Python 3.8 or newer +- `requests` +- A Trakt application Client ID in `TRAKT_CLIENT_ID` for live reads +- No OAuth login is needed for the public discovery commands diff --git a/trakt/SKILL.md b/trakt/SKILL.md index cd4d724..0406016 100644 --- a/trakt/SKILL.md +++ b/trakt/SKILL.md @@ -1,105 +1,110 @@ --- name: trakt -description: Discover trending, anticipated, and popular movies and TV shows via the - Trakt.tv API from the terminal. No authentication required for read-only discovery. - Use when the user asks about what to watch, trending movies, popular shows, or media - discovery. +description: >- + Discover and compare Trakt.tv trending, popular, and anticipated movies and shows + from the terminal. Do not use this skill for TMDb catalog metadata, credits, images, + or provider lookups; use the tmdb skill for those tasks. license: MIT -compatibility: Requires TRAKT_CLIENT_ID env var (free from trakt.tv/oauth/applications), - Python 3.8+, and the `requests` library. No OAuth or user login needed for discovery - endpoints. +compatibility: Requires TRAKT_CLIENT_ID, Python 3.8+, and requests. Public discovery + reads use an application Client ID; OAuth is only needed for user-scoped operations. metadata: tags: trakt, media-discovery, movies, tv-shows, trending, api-client - sources: https://trakt.tv/, https://trakt.docs.apiary.io/ + sources: https://docs.trakt.tv/docs/required-headers --- -# trakt-cli — Trakt.tv Media Discovery +# Trakt media discovery -Discover trending, anticipated, and popular movies and TV shows from the terminal. Uses the Trakt.tv API v2 with a read-only Client ID — no user authentication required. +Use this skill to inspect what is being watched, what is broadly popular, and what is anticipated. It is a read-only discovery surface, not a catalog metadata service. -## Setup +## Setup and authentication -1. Register an app at [trakt.tv/oauth/applications](https://trakt.tv/oauth/applications) to get a Client ID -2. Set the environment variable: +Register an app at [Trakt OAuth applications](https://trakt.tv/oauth/applications) and export its Client ID: -```bash -export TRAKT_CLIENT_ID="your-trakt-client-id" +```sh +export TRAKT_CLIENT_ID="YOUR_TRAKT_CLIENT_ID" ``` -No OAuth token, no user login needed for any of the commands below. `--help` and `--dry-run` work without credentials. +Every request must send `trakt-api-key: ` together with the mandatory companion header `trakt-api-version: 2`, plus JSON content type and a descriptive User-Agent. Public discovery endpoints use the key header, not `Authorization: Bearer`. OAuth bearer tokens are for endpoints marked OAuth-required or for user-scoped lists, history, collection, watchlist, and mutations; a bearer token does not replace the key/version pair. -## Essential Commands +## Essential commands -### movie trending — Trending movies +### Trending: watched in the last 24 hours -```bash -trakt-cli movie trending # top 10 trending movies -trakt-cli movie trending --limit 25 # more results -trakt-cli movie trending --json # machine-readable with TMDb IDs +```sh +trakt movie trending --limit 20 +trakt tv trending --limit 20 --json ``` -### movie anticipated — Most anticipated movies +Trending responses wrap each media object in `movie` or `show` and include a `watchers` count. -```bash -trakt-cli movie anticipated # top 10 anticipated -trakt-cli movie anticipated --limit 5 # top 5 -trakt-cli movie anticipated --json # machine-readable +### Popular: broad popularity ranking + +```sh +trakt movie popular --limit 25 --json +trakt tv popular --limit 25 ``` -### movie popular — Most popular movies +Popular is a ranking based on rating percentage and number of ratings, not a personalized recommendation. -```bash -trakt-cli movie popular # top 10 popular movies -trakt-cli movie popular --limit 25 # more results +### Anticipated: upcoming interest + +```sh +trakt movie anticipated --limit 10 +trakt tv anticipated --limit 10 --json ``` -### tv trending — Trending TV shows +Anticipated reflects list appearances and upcoming interest. It is not the same as a release calendar. -```bash -trakt-cli tv trending # top 10 trending shows -trakt-cli tv trending --limit 25 # more results -trakt-cli tv trending --json # machine-readable with TVDB IDs +Global flags can appear before or after the resource: `--json`, `--dry-run`, `--quiet`, and `--verbose`. + +## Pipeline recipes + +### Trending handoff to another tool + +1. Run `trakt --json movie trending --limit 20`. +2. Unwrap `.movie`, retaining `.watchers` as the watch signal. +3. Pass an available `.movie.ids.tmdb` or `.movie.ids.imdb` to a downstream tool; do not assume a missing ID can be synthesized. + +```sh +trakt --json movie trending --limit 20 | + jq '.movies[] | {title: (.movie.title // .title), year: (.movie.year // null), watchers: (.watchers // null), ids: (.movie.ids // .ids)}' ``` -### tv anticipated — Most anticipated TV shows +### Compare discovery signals -```bash -trakt-cli tv anticipated # top 10 anticipated shows -trakt-cli tv anticipated --limit 5 # top 5 -``` +Fetch matching pages of trending, popular, and anticipated, then label each dataset before combining it. Trending is recent watching, popular is broad ranking, and anticipated is upcoming interest. -### tv popular — Most popular TV shows +## JSON and pagination -```bash -trakt-cli tv popular # top 10 popular shows -trakt-cli tv popular --limit 25 # more results -``` +`--json` emits a `movies` or `shows` object suitable for `jq`; trending entries retain their wrapper. The API accepts `page` and `limit`, with compatibility defaults of page 1 and limit 10. API responses include `X-Pagination-Page-Count`; automation should stop at that header rather than assuming a short page is the end. -Each result shows: title, year, network (for TV), TMDb/TVDB ID, and tagline (for movies). +## Known gotchas -## Global Flags +- **Header pair is mandatory:** sending `trakt-api-key` without `trakt-api-version: 2` (or vice versa) can yield an invalid-request/authentication-style failure. The bundled script injects both on every live request. +- **401 versus 403:** 401 commonly indicates an OAuth requirement or invalid authorization; 403 indicates an invalid or unapproved application key. Do not retry either blindly. +- **Rate limits:** on 429, honor `Retry-After` and inspect `X-Ratelimit`. Use bounded retries; transient 502/503/504 responses may be retried with backoff. +- **OAuth refresh:** access tokens last seven days and refresh tokens are single-use. Replace the stored refresh token after a successful refresh; `invalid_grant` requires reauthorization. +- **Trakt is not TMDb:** Trakt IDs and discovery rankings are not TMDb metadata. Use the `tmdb` skill for credits, images, provider metadata, and catalog enrichment. +- **Trending shape:** read `.movie` or `.show` before title/IDs, while preserving `watchers`. -All flags work in any position: +## When to use -```bash -trakt-cli --json movie trending # flag before subcommand -trakt-cli movie trending --json # flag after subcommand -trakt-cli --dry-run movie trending # preview (no API call) -trakt-cli --quiet movie trending # suppress non-essential output -trakt-cli --verbose movie trending # detailed logging -``` +Use Trakt for current watching signals, broad popularity, anticipated interest, and identifiers that feed a media workflow. -## Known Gotchas +## When not to use -- **Read-only by design** — The CLI only uses the Client ID flow. No OAuth, no writing to your Trakt lists. All endpoints are public discovery endpoints. -- **No auth needed for these commands** — The trending, anticipated, and popular endpoints are public. Skip the setup if you only want to preview with `--dry-run`. -- **Rate limiting** — Trakt API v2 has rate limits (~1,000 calls per 5 minutes for free apps). The CLI does not auto-retry on 429 responses. -- **TRAKT_CLIENT_ID is required at runtime** — Unlike `--dry-run` which skips the API call, running live commands without the env var will fail with a clear error message. -- **Pagination defaults to page 1** — The CLI uses `--limit` for the results per page. Default is 10, max is typically 50. -- **TMDb/TVDB IDs** — Use `--json` to get the full IDs object (TMDb for movies, TVDB for shows) which is useful for lookups in other tools like Radarr/Sonarr. +Do not use Trakt for TMDb catalog metadata, credits, images, provider availability, or for writing a user's lists without an explicit OAuth-enabled workflow. Use `tmdb` for metadata and a dedicated authenticated operation for mutations. -## References +## Reference files -- [scripts/trakt-cli](scripts/trakt-cli) — The CLI binary. Built following the cli-builder patterns: `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth. -- [Trakt API Docs](https://trakt.docs.apiary.io/) — Official API reference. -- [Trakt OAuth Applications](https://trakt.tv/oauth/applications) — Register an app to get your Client ID. +| File | Topic | +|---|---| +| [references/auth-and-request-contract.md](references/auth-and-request-contract.md) | Required headers, OAuth boundary, errors, and rate limits | +| [references/discovery-endpoints.md](references/discovery-endpoints.md) | Endpoint semantics, filters, response shapes, and pagination | +| [references/recipes-and-operations.md](references/recipes-and-operations.md) | Pipelines, jq normalization, and operational handling | + +## Available script and prerequisites + +- `scripts/trakt` is an executable Python CLI using only stdlib and `requests`. +- `--dry-run` works without a Client ID and never performs network I/O. +- Live discovery requires `TRAKT_CLIENT_ID`; tests are mock-only. diff --git a/trakt/evals/evals.json b/trakt/evals/evals.json new file mode 100644 index 0000000..1a6ea83 --- /dev/null +++ b/trakt/evals/evals.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "skill_name": "trakt", + "evals": [ + { + "id": "trending-movies", + "prompt": "Show the 20 movies most watched recently using Trakt.", + "expected_output": "Use trakt movie trending --limit 20 and explain that trending is a recent-watch signal.", + "assertions": ["selects the movie trending command", "sets an explicit limit", "describes the recent watching window"] + }, + { + "id": "popular-shows-json", + "prompt": "Get popular TV shows as JSON for a jq pipeline.", + "expected_output": "Run trakt --json tv popular and process the shows object with jq.", + "assertions": ["uses the tv popular command", "enables JSON output", "mentions jq processing"] + }, + { + "id": "anticipated-pagination", + "prompt": "Explain how to collect anticipated movies across pages without overrunning Trakt limits.", + "expected_output": "Use page and limit, stop at X-Pagination-Page-Count, and honor Retry-After on 429.", + "assertions": ["names page and limit parameters", "uses the pagination page count header", "handles Retry-After for rate limiting"] + }, + { + "id": "header-pair-gotcha", + "prompt": "Why does my Trakt request with trakt-api-key still fail?", + "expected_output": "Send trakt-api-version: 2 together with trakt-api-key, plus JSON content type; the version header is mandatory.", + "assertions": ["documents the trakt-api-key header", "documents the trakt-api-version 2 companion header", "identifies the missing-header failure cause"] + }, + { + "id": "tmdb-metadata-not-trigger", + "prompt": "Do not route this request to Trakt: enrich a movie with TMDb credits, images, and provider metadata.", + "expected_output": "Do not use Trakt; route catalog metadata enrichment to the tmdb skill.", + "assertions": ["must not trigger for TMDb metadata work", "names the tmdb skill as the alternative"] + }, + { + "id": "oauth-boundary", + "prompt": "Do I need OAuth to see public trending and popular feeds, and what changes for my watchlist?", + "expected_output": "Public discovery reads use the application key and required version header; user-scoped watchlist operations require OAuth Bearer in addition to the app headers.", + "assertions": ["distinguishes public reads from user-scoped operations", "requires OAuth for watchlist work", "retains the application header pair"] + } + ] +} diff --git a/trakt/references/auth-and-request-contract.md b/trakt/references/auth-and-request-contract.md new file mode 100644 index 0000000..1730ffa --- /dev/null +++ b/trakt/references/auth-and-request-contract.md @@ -0,0 +1,32 @@ +# Trakt authentication and request contract + +## Public discovery + +Trakt v2 identifies an application with its Client ID in the `trakt-api-key` header. Every request must also send the companion header `trakt-api-version: 2`; sending only one of the pair can produce an invalid request or authentication-style failure. Use `Content-Type: application/json` and an identifying `User-Agent` as well. + +```sh +curl --fail-with-body 'https://api.trakt.tv/movies/trending?page=1&limit=20' \ + -H 'Content-Type: application/json' \ + -H 'User-Agent: MyAppName/1.0.0' \ + -H "trakt-api-key: ${TRAKT_CLIENT_ID}" \ + -H 'trakt-api-version: 2' +``` + +The bundled CLI uses this application-key mode. It does not put the Client ID in `Authorization: Bearer`; that header is reserved for an OAuth access token. + +## OAuth boundary + +Public trending, popular, and anticipated reads do not require a user login. OAuth is needed by endpoints marked as required and is appropriate for user-scoped list, history, collection, watchlist, or mutation operations. A bearer token does not replace the application key and version header when calling the API. + +Trakt supports authorization-code and device-code flows. Access tokens last seven days. Refresh tokens are single-use: persist the replacement returned by a successful refresh and discard the old token. A 400/401 response containing `invalid_grant` means the session is no longer usable and requires reauthorization. Never log client secrets, access tokens, or refresh tokens. + +## Failure handling + +Treat 401 and 403 as credential or app-approval errors, 400/422 as request validation errors, and 429 as rate limiting. On 429, honor `Retry-After` and inspect `X-Ratelimit`; do not retry forever. Transient 502/503/504 responses can be retried with a bounded backoff. The CLI surfaces status and response details without attempting unsafe retries. + +## Sources + +- https://docs.trakt.tv/docs/required-headers +- https://docs.trakt.tv/docs/getting-started +- https://docs.trakt.tv/docs/authentication-oauth +- https://trakt.docs.apiary.io/api-description-document diff --git a/trakt/references/discovery-endpoints.md b/trakt/references/discovery-endpoints.md new file mode 100644 index 0000000..9f7d1d9 --- /dev/null +++ b/trakt/references/discovery-endpoints.md @@ -0,0 +1,34 @@ +# Trakt discovery endpoints + +All endpoints below are GET requests at `https://api.trakt.tv` and use the request contract in `auth-and-request-contract.md`. + +| Endpoint | Meaning | Response shape | +|---|---|---| +| `/movies/trending` | Most watched movies in the last 24 hours, ordered by watchers | wrapper objects with `watchers` and nested `movie` | +| `/movies/popular` | Popularity based on rating percentage and number of ratings | movie objects | +| `/movies/anticipated` | Upcoming interest based on list appearances | movie objects | +| `/shows/trending` | Most watched shows in the last 24 hours, ordered by watchers | wrapper objects with `watchers` and nested `show` | +| `/shows/popular` | Popularity based on rating percentage and number of ratings | show objects | +| `/shows/anticipated` | Upcoming interest based on list appearances | show objects | + +Trending is a short, current watch signal. Popular is a broad popularity ranking, while anticipated is an upcoming-interest signal. Do not treat a trending rank as a release calendar or a popularity score as a personalized recommendation. + +## Paging and filters + +These feeds accept `page` and `limit`; compatibility defaults are page 1 and limit 10. Set both explicitly for reproducible automation. Responses provide `X-Pagination-Page`, `X-Pagination-Limit`, `X-Pagination-Page-Count`, and `X-Pagination-Item-Count`. Stop at the reported page count instead of assuming a short page means completion. + +Endpoint pages also document filters such as `extended`, `watchnow`, `genres`, `years`, `ratings`, date ranges, countries, and `ignore_watched`, `ignore_collected`, and `ignore_watchlisted` where supported. Encode comma-separated values as query parameters. `watchnow=any` means any service, while `any_all` and the `free_all`/`subscriptions_all` forms have stricter all-country semantics. + +## Result normalization + +For trending responses, unwrap `movie` or `show` before reading title, year, and IDs, but preserve `watchers` if ranking matters. Popular and anticipated responses are already direct media objects. Trakt IDs are not TMDb metadata: use the returned `ids` object to hand an identifier to another tool, and use TMDb when the task is catalog metadata, credits, images, or provider details. + +## Sources + +- https://docs.trakt.tv/reference/getmoviestrending +- https://docs.trakt.tv/reference/getmoviespopular +- https://docs.trakt.tv/reference/getmoviesanticipated +- https://docs.trakt.tv/reference/getshowstrending +- https://docs.trakt.tv/reference/getshowspopular +- https://docs.trakt.tv/reference/getshowsanticipated +- https://trakt.docs.apiary.io/reference/movies/trending/get-trending-movies diff --git a/trakt/references/recipes-and-operations.md b/trakt/references/recipes-and-operations.md new file mode 100644 index 0000000..b6b84a7 --- /dev/null +++ b/trakt/references/recipes-and-operations.md @@ -0,0 +1,31 @@ +# Trakt recipes and operations + +## Trending to a handoff + +1. Run `trakt movie trending --limit 20 --json`. +2. For each object, read `.movie` and retain `.watchers` as the current-watch signal. +3. Pass `.movie.ids.tmdb` or `.movie.ids.imdb` to the next tool only when present; do not mistake a Trakt response for TMDb metadata. + +```sh +trakt --json movie trending --limit 20 | + jq '.movies[] | {title: (.movie.title // .title), year: (.movie.year // null), watchers: (.watchers // null), ids: (.movie.ids // .ids)}' +``` + +## Compare discovery signals + +Fetch the same page of `movie trending`, `movie popular`, and `movie anticipated`. Trending answers "watched recently"; popular answers "high broad popularity"; anticipated answers "appears on many upcoming-interest lists." Keep these datasets labeled when combining them. + +## Paginate anticipated releases + +Use `page` and `limit` in API clients, inspect `X-Pagination-Page-Count`, and stop at that count. If the response is 429, wait at least the numeric `Retry-After` value and cap retries. The CLI intentionally exposes one page per invocation; shell automation can iterate pages while retaining the response headers in a real HTTP client. + +## JSON processing + +`--json` emits an object with `movies` or `shows`; trending elements retain their wrapper shape. Use `jq` for selection and `@csv` only after explicitly handling null IDs. Human output is for inspection, JSON output is for pipelines. + +## Sources + +- https://docs.trakt.tv/docs/required-headers +- https://docs.trakt.tv/reference/getmoviestrending +- https://docs.trakt.tv/reference/getmoviesanticipated +- https://trakt.docs.apiary.io/reference/movies/anticipated/get-most-anticipated-movies diff --git a/trakt/scripts/test_trakt.py b/trakt/scripts/test_trakt.py new file mode 100644 index 0000000..db825c3 --- /dev/null +++ b/trakt/scripts/test_trakt.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Offline tests for the Trakt discovery CLI.""" +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path +from unittest import TestCase, mock + +SCRIPT = Path(__file__).with_name("trakt") +loader = importlib.machinery.SourceFileLoader("trakt_cli", str(SCRIPT)) +spec = importlib.util.spec_from_loader(loader.name, loader) +trakt = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = trakt +loader.exec_module(trakt) + + +class TraktCliTests(TestCase): + def run_cli(self, *args, **kwargs): + env = os.environ.copy() + env.pop("TRAKT_CLIENT_ID", None) + return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, env=env) + + def test_help_lists_discovery_groups(self): + result = self.run_cli("--help") + self.assertEqual(result.returncode, 0) + self.assertIn("movie", result.stdout) + self.assertIn("tv", result.stdout) + + def test_argument_error_is_nonzero(self): + result = self.run_cli("movie", "unknown") + self.assertNotEqual(result.returncode, 0) + self.assertIn("invalid choice", result.stderr) + + def test_dry_run_json_is_valid_without_network(self): + result = self.run_cli("--dry-run", "--json", "movie", "trending") + self.assertEqual(result.returncode, 0) + payload = json.loads(result.stdout) + self.assertEqual(payload, {"dry_run": True}) + + @mock.patch.object(trakt.requests, "get") + def test_client_injects_required_header_pair(self, get): + response = mock.Mock(status_code=200) + response.json.return_value = [{"movie": {"title": "Example"}}] + get.return_value = response + client = trakt.TraktClient(client_id="CLIENT_ID") + client.movie_trending(limit=4) + headers = get.call_args.kwargs["headers"] + self.assertEqual(headers["trakt-api-key"], "CLIENT_ID") + self.assertEqual(headers["trakt-api-version"], "2") + self.assertEqual(headers["Content-Type"], "application/json") + + @mock.patch.object(trakt, "die") + @mock.patch.object(trakt.requests, "get") + def test_client_reports_http_error(self, get, die): + response = mock.Mock(status_code=403) + response.json.return_value = {"message": "forbidden"} + get.return_value = response + trakt.TraktClient(client_id="CLIENT_ID").movie_popular() + die.assert_called_once() + self.assertIn("403", die.call_args.args[0]) + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/trakt/scripts/trakt-cli b/trakt/scripts/trakt similarity index 96% rename from trakt/scripts/trakt-cli rename to trakt/scripts/trakt index 0a7bee2..8f9a992 100755 --- a/trakt/scripts/trakt-cli +++ b/trakt/scripts/trakt @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""trakt-cli — Trakt.tv media discovery from the terminal. +"""trakt — Trakt.tv media discovery from the terminal. Discover trending, anticipated, and popular movies and TV shows. Calendar of upcoming releases. Uses Trakt.tv API with Client ID. @@ -12,6 +12,7 @@ import requests ENV_CLIENT_ID = os.getenv("TRAKT_CLIENT_ID", "") API_BASE = "https://api.trakt.tv" +USER_AGENT = "agent-skills-trakt/1.0" QUIET = False GLOBAL_FLAGS: Dict[str, Any] = {"json": False, "dry_run": False, "quiet": False, "verbose": False} @@ -44,8 +45,8 @@ class TraktClient: if not self.client_id: die("TRAKT_CLIENT_ID not set. Get one from trakt.tv/oauth/applications.") try: r = requests.get(url, params=params, headers={ - "Content-Type": "application/json", "trakt-api-version": "2", - "trakt-api-key": self.client_id}, timeout=30) + "Content-Type": "application/json", "User-Agent": USER_AGENT, + "trakt-api-version": "2", "trakt-api-key": self.client_id}, timeout=30) except ConnectionError as e: die(f"Cannot connect: {e}") if r.status_code == 401: die("Auth failed (401). Check TRAKT_CLIENT_ID.") if r.status_code >= 400: From 577628224bdf27110596daddf1c109c8fe74bccd Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 10:00:38 -0400 Subject: [PATCH 21/40] chore(catalog): refresh Trakt skill indexes Regenerate marketplace, Codex, and llms catalogs after thickening Trakt. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/marketplace.json | 2 +- llms.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index d2cb22d..ba6d21b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1290,7 +1290,7 @@ "./trakt" ], "strict": false, - "description": "Discover trending, anticipated, and popular movies and TV shows via the Trakt.tv API from the terminal. No authentication required for read-only discovery. Use when the user asks about what to watch, trending movies, popular shows, or media discovery." + "description": "Discover and compare Trakt.tv trending, popular, and anticipated movies and shows from the terminal. Do not use this skill for TMDb catalog metadata, credits, images, or provider lookups; use the tmdb skill for those tasks." }, { "name": "transistor", diff --git a/llms.txt b/llms.txt index 91678a4..75ca5ad 100644 --- a/llms.txt +++ b/llms.txt @@ -144,7 +144,7 @@ - [three](three/SKILL.md): Build browser-based Three.js and WebGL scenes, animations, and interactive 3D visualizations with a small vanilla JavaScript starting point. - [tmdb](tmdb/SKILL.md): Query TMDb metadata for films and television, then enrich results with details, credits, providers, and external IDs. Do not use this skill for torrent search, streaming playback, or personal watch-history tracking. - [traefik](traefik/SKILL.md): Deploy, configure, and troubleshoot Traefik v3 reverse proxy — covers all providers, routing, TLS/ACME, middlewares, and production patterns with YAML examples. Load when setting up or debugging a Traefik instance. -- [trakt](trakt/SKILL.md): Discover trending, anticipated, and popular movies and TV shows via the Trakt.tv API from the terminal. No authentication required for read-only discovery. Use when the user asks about what to watch, trending movies, popular shows, or media discovery. +- [trakt](trakt/SKILL.md): Discover and compare Trakt.tv trending, popular, and anticipated movies and shows from the terminal. Do not use this skill for TMDb catalog metadata, credits, images, or provider lookups; use the tmdb skill for those tasks. - [transistor](transistor/SKILL.md): Manage Transistor.fm podcast hosting from the terminal: view shows, list episodes, check analytics, and get subscriber counts. Use when the user mentions Transistor, podcast hosting, podcast analytics, show management, or episode tracking. - [travel-guide](travel-guide/SKILL.md): Create personalized, source-grounded travel dossiers from a destination, dates, duration, travelers, and constraints. Ask only the questions that change the plan, use explicitly permitted personal context without exposing it, research current logistics, and produce a cited, visually coherent PDF or responsive companion web page. Use when someone wants an individualized itinerary, trip brief, travel field guide, or shareable travel website. Do not use for real-time booking, ticket purchasing, visa or legal advice, or generic destination summaries without a specific traveler and trip. - [vercel-eve](vercel-eve/SKILL.md): Build, develop, deploy, self-host, secure, and troubleshoot durable backend AI agents with Vercel Eve. Use when creating an Eve agent, adding tools, skills, subagents, channels, schedules, sandboxing, durable sessions, observability, or deploying Eve on Vercel or a Node host. Do not use for the separate Vercel AI SDK Agent APIs such as ToolLoopAgent or WorkflowAgent; use an AI SDK-specific skill for those. From c90fade381486a913ad43517cc3634755735583f Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 10:16:43 -0400 Subject: [PATCH 22/40] fix(openlibrary): preserve author arrays in work JSON Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- openlibrary/SKILL.md | 6 ++- openlibrary/references/recipes-and-gotchas.md | 10 ++--- openlibrary/scripts/openlibrary | 10 +++-- openlibrary/scripts/test_openlibrary.py | 40 +++++++++++++++++++ 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/openlibrary/SKILL.md b/openlibrary/SKILL.md index a004d0f..490af1d 100644 --- a/openlibrary/SKILL.md +++ b/openlibrary/SKILL.md @@ -149,8 +149,10 @@ openlibrary work OL1168083W --json | jq -r '.authors[0]' # bare OL…A ke openlibrary author OL118077A # bio, dates ``` -Each hop uses a different key suffix (M → W → A); see Known Gotchas before -hand-assembling these URLs yourself. +The CLI keeps this pipeline type-safe: in JSON, `.authors` is an array of bare +OL…A keys; human output renders the same keys as a comma-separated label. Each hop +uses a different key suffix (M → W → A); see Known Gotchas before hand-assembling +these URLs yourself. ### Rank a series by community love diff --git a/openlibrary/references/recipes-and-gotchas.md b/openlibrary/references/recipes-and-gotchas.md index 60cf8da..64ae929 100644 --- a/openlibrary/references/recipes-and-gotchas.md +++ b/openlibrary/references/recipes-and-gotchas.md @@ -20,12 +20,12 @@ curl -sL "https://openlibrary.org/isbn/$ISBN.json" > edition.json # 2. Pull the work key out of the edition (path form /works/OL…W) WORK=$(jq -r '.works[0].key' edition.json) # e.g. /works/OL166894W -# 3. Fetch the work for description + subjects; note double-nested author refs -curl -s "https://openlibrary.org${WORK}.json" > work.json -AUTHOR=$(jq -r '.authors[0].author.key' work.json) # /authors/OL23919A +# 3. Fetch the work through the CLI. Its JSON handoff exposes bare OL…A keys. +openlibrary work "${WORK##*/}" --json > work.json +AUTHOR=$(jq -r '.authors[0]' work.json) # OL23919A -# 4. Author record; bio may be {type,value}-wrapped or a plain string -curl -s "https://openlibrary.org${AUTHOR}.json" | jq -r ' +# 4. Author record; the CLI accepts the bare key and unwraps bio text. +openlibrary author "$AUTHOR" --json | jq -r ' if (.bio | type) == "object" then .bio.value else .bio end' ``` diff --git a/openlibrary/scripts/openlibrary b/openlibrary/scripts/openlibrary index 5a114ac..49c7817 100755 --- a/openlibrary/scripts/openlibrary +++ b/openlibrary/scripts/openlibrary @@ -349,8 +349,12 @@ def cmd_work(client, args): title = data.get("title", "?") authors = data.get("authors", []) - author_str = ", ".join([normalize_olid(a.get("author", {}).get("key", "?")) - for a in authors]) if authors else "?" + author_keys = [ + normalize_olid(a.get("author", {}).get("key", "")) + for a in authors + if isinstance(a, dict) and a.get("author", {}).get("key") + ] + author_str = ", ".join(author_keys) or "?" desc = unwrap_text(data.get("description", "")) desc_short = f"\n{desc[:500]}" if desc else "" subjects = ", ".join(data.get("subjects", [])[:5]) or "(none)" @@ -360,7 +364,7 @@ def cmd_work(client, args): f"📖 {title}\n" f" Author(s): {author_str}\n" f" Subjects: {subjects}{desc_short}", - {"key": key, "title": title, "authors": author_str, + {"key": key, "title": title, "authors": author_keys, "description": desc, "subjects": data.get("subjects", []), "cover_url": cover_url("b/id", covers[0]) if covers else None} ) diff --git a/openlibrary/scripts/test_openlibrary.py b/openlibrary/scripts/test_openlibrary.py index 8988f31..cb477c3 100644 --- a/openlibrary/scripts/test_openlibrary.py +++ b/openlibrary/scripts/test_openlibrary.py @@ -217,6 +217,46 @@ class MockedClientTests(unittest.TestCase): self.assertTrue(req.call_args_list[1].args[0].endswith(WORK_KEY + ".json")) self.assertEqual(json.loads(out)["authors"], "OL118077A") + def test_work_json_authors_are_bare_keys_array(self): + with mock.patch.object(requests, "get", + return_value=FakeResponse(200, WORK_RECORD)): + code, out, _ = run_cli("--json", "work", WORK_KEY) + self.assertEqual(code, 0) + data = json.loads(out) + self.assertIsInstance(data["authors"], list) + self.assertEqual(data["authors"], ["OL118077A"]) + self.assertRegex(data["authors"][0], r"^OL\d+A$") + + def test_isbn_work_author_pipeline_handoff_is_executable(self): + """Mock the documented ISBN -> work -> author jq handoff end to end.""" + edition = FakeResponse(200, EDITION_RECORD) + work = FakeResponse(200, WORK_RECORD) + author = FakeResponse(200, {"name": "George Orwell", "bio": "Writer"}) + with mock.patch.object(requests, "get", + side_effect=[edition, work, work, author]) as req: + isbn_code, isbn_out, _ = run_cli("--json", "isbn", "9780451524935") + isbn_data = json.loads(isbn_out) + work_code, work_out, _ = run_cli( + "--json", "work", isbn_data["work_keys"][0]) + work_data = json.loads(work_out) + author_code, author_out, _ = run_cli( + "--json", "author", work_data["authors"][0]) + self.assertEqual((isbn_code, work_code, author_code), (0, 0, 0)) + self.assertEqual(req.call_count, 4) + self.assertEqual(work_data["authors"][0], "OL118077A") + self.assertEqual(json.loads(author_out)["key"], "OL118077A") + + def test_work_pipeline_handoff_fields_have_stable_json_types(self): + with mock.patch.object(requests, "get", + return_value=FakeResponse(200, WORK_RECORD)): + _, out, _ = run_cli("--json", "work", WORK_KEY) + data = json.loads(out) + self.assertIsInstance(data["key"], str) + self.assertIsInstance(data["title"], str) + self.assertIsInstance(data["authors"], list) + self.assertTrue(all(isinstance(key, str) for key in data["authors"])) + self.assertIsInstance(data["subjects"], list) + def test_merge_redirect_stub_in_http_200_is_followed_with_json_suffix(self): # Merged-away keys answer 200 with {type:/type/redirect, location}, # NOT a 3xx — the client must detect the stub and refetch. Stub From bd92ed3fcac2704f46c47813a2e77e2409722fe2 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 18:29:26 -0400 Subject: [PATCH 23/40] fix(tmdb): align CLI to genre list, tv search, and find contracts - Replace raw-argv dispatch with parsed-attribute routing; add the documented nested `genre list --type movie|tv` subparser while keeping the flat legacy form a clean argparse rejection instead of a crash. - Route `tv search` through client.search_tv (/search/tv) with TV formatting (name / first_air_date) via new cmd_tv_search. - Gate find --source to exactly the official eight external_source values (imdb_id, facebook_id, instagram_id, tvdb_id, tiktok_id, twitter_id, wikidata_id, youtube_id); docs explicitly reject retired freebase_mid/freebase_id; SKILL.md documents both --type variants. Adds parser-level regression tests for genre list, endpoint-selection and output-shape tests for tv search, and parameterized acceptance and rejection tests for external_source values (13 tests, 10 subtests, green under pytest strict-markers, unittest discover, and the proxy trap). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- tmdb/SKILL.md | 5 +- tmdb/references/find-and-details.md | 2 +- tmdb/scripts/test_tmdb.py | 118 ++++++++++++++++++++++++++++ tmdb/scripts/tmdb | 79 ++++++++++++++----- 4 files changed, 181 insertions(+), 23 deletions(-) diff --git a/tmdb/SKILL.md b/tmdb/SKILL.md index a3447c3..ba3157b 100644 --- a/tmdb/SKILL.md +++ b/tmdb/SKILL.md @@ -31,7 +31,7 @@ tmdb tv search --term "severance" --limit 5 tmdb find tt0111161 --source imdb_id --json ``` -`find` accepts the current external sources `imdb_id`, `tvdb_id`, `wikidata_id`, `facebook_id`, `instagram_id`, `tiktok_id`, `twitter_id`, and `youtube_id`. Its response is split into `movie_results`, `tv_results`, `person_results`, `tv_season_results`, and `tv_episode_results`. +`--source` accepts one of the official external-source values: `imdb_id`, `facebook_id`, `instagram_id`, `tvdb_id`, `tiktok_id`, `twitter_id`, `wikidata_id`, and `youtube_id`. Freebase lookups are not supported: the retired `freebase_mid` and `freebase_id` values are rejected. The response is split into `movie_results`, `tv_results`, `person_results`, `tv_season_results`, and `tv_episode_results`. ### Details and enrichment @@ -49,6 +49,7 @@ tmdb movie discover --genre horror --rating 7 --limit 10 tmdb movie discover --genre horror --certification R --from 2024-01-01 --to 2024-12-31 tmdb trending --type all --window week --limit 20 --json tmdb genre list --type movie --json +tmdb genre list --type tv --json tmdb certification --json ``` @@ -95,7 +96,7 @@ Put `--json` before or after the subcommand. JSON search output has `results` an - **Credential duality:** `api_key` and Bearer are alternatives, not values to mix. A rejected credential commonly produces HTTP 401, `status_code: 7`, and `Invalid API key: You must be granted a valid key.` Permission failures use code 3. Code 33 means an invalid request token, not this API-key message. - **Pagination ceiling:** pages start at 1 and max at 500; over-limit requests fail. Search/discover access is effectively capped at 10,000 results, even where totals look larger. Rate guidance is around 40 requests/second and 429 responses should honor `Retry-After`. - **Compound syntax:** append values are comma-separated and limited to 20 calls. `watch/providers` contains a slash, so URL-encode it in curl and use jq's `.\"watch/providers\"` notation. -- **External-ID shape:** `/find/` does not return one generic `id`; inspect the appropriate nested array before choosing movie or TV detail. +- **External-ID shape:** `/find/` does not return one generic `id`; inspect the appropriate nested array before choosing movie or TV detail. Only the eight documented `external_source` values are valid, and the retired Freebase sources (`freebase_mid`, `freebase_id`) are rejected. - **Provider filters:** `with_watch_providers` requires `watch_region`; provider data carries JustWatch attribution requirements. - **Localization and images:** use `language=en-US` and a market `region` when reproducibility matters. Build image URLs from `/3/configuration`'s secure base URL, a valid size, and the returned path. diff --git a/tmdb/references/find-and-details.md b/tmdb/references/find-and-details.md index fc40de6..d7171f0 100644 --- a/tmdb/references/find-and-details.md +++ b/tmdb/references/find-and-details.md @@ -2,7 +2,7 @@ ## Start with an IMDb ID -`GET /3/find/{external_id}?external_source=imdb_id` maps a foreign identifier to TMDb objects. The current `external_source` enum includes `imdb_id`, `facebook_id`, `instagram_id`, `tvdb_id`, `tiktok_id`, `twitter_id`, `wikidata_id`, and `youtube_id`; older Freebase values are obsolete. The response has `movie_results`, `person_results`, `tv_results`, `tv_episode_results`, and `tv_season_results` arrays. Unmatched categories are empty arrays. For an IMDb movie, extract `.movie_results[0].id` before calling the movie details endpoint. +`GET /3/find/{external_id}?external_source=imdb_id` maps a foreign identifier to TMDb objects. The `external_source` value is chosen from exactly eight supported enum entries: `imdb_id`, `facebook_id`, `instagram_id`, `tvdb_id`, `tiktok_id`, `twitter_id`, `wikidata_id`, and `youtube_id`. Freebase lookups are not supported: the retired `freebase_mid` and `freebase_id` sources have been removed from the API and must not be used or documented as valid values. The response has `movie_results`, `person_results`, `tv_results`, `tv_episode_results`, and `tv_season_results` arrays. Unmatched categories are empty arrays. For an IMDb movie, extract `.movie_results[0].id` before calling the movie details endpoint. ```bash curl -s -H "Authorization: Bearer $TMDB_ACCESS_TOKEN" \ diff --git a/tmdb/scripts/test_tmdb.py b/tmdb/scripts/test_tmdb.py index e5540cf..b2a4a2a 100644 --- a/tmdb/scripts/test_tmdb.py +++ b/tmdb/scripts/test_tmdb.py @@ -63,5 +63,123 @@ class TmdbCliTests(unittest.TestCase): client.get_movie.assert_called_once_with("550", "credits,videos") +class GenreListParserTests(unittest.TestCase): + """Regression coverage for `tmdb genre list --type movie|tv`.""" + + def run_cli(self, *args): + env = os.environ.copy() + env.pop("TMDB_ACCESS_TOKEN", None) + env.pop("TMDB_API_KEY", None) + return subprocess.run([str(SCRIPT), *args], text=True, capture_output=True, env=env) + + def test_documented_nested_form_parses_and_dispatches(self): + result = self.run_cli("--dry-run", "--json", "genre", "list", "--type", "movie") + self.assertEqual(result.returncode, 0) + self.assertTrue(json.loads(result.stdout)["dry_run"]) + + def test_flat_form_is_clean_rejection_not_crash(self): + result = self.run_cli("--json", "genre", "--type", "tv") + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("Traceback", result.stderr) + self.assertIn("invalid choice", result.stderr) + + def test_missing_type_is_argument_error(self): + result = self.run_cli("--json", "genre", "list") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--type", result.stderr) + + def test_dispatch_passes_tail_args_without_raw_argv_token_search(self): + cli = load_cli() + captured = {} + real_client_factory = cli.TMDBClient + + def fake_client(dry_run=False): + return real_client_factory(dry_run=dry_run) + + def fake_handler(client, args): + captured["args"] = args + + cli.cmd_genre_list = fake_handler + with patch.object(cli.sys, "argv", ["tmdb", "--dry-run", "genre", "list", "--type", "tv"]): + cli.main() + self.assertEqual(captured["args"], ["--type", "tv"]) + self.assertEqual(cli.GLOBAL_FLAGS.get("dry_run"), True) + fake_client # client construction stays credential-free + + +class TvSearchEndpointTests(unittest.TestCase): + """TV search must hit /search/tv via client.search_tv and format TV fields.""" + + def load_with_flags(self): + cli = load_cli() + cli.GLOBAL_FLAGS = {"json": True, "dry_run": False, "quiet": False, "verbose": False} + return cli + + def test_json_output_uses_search_tv_and_preserves_tv_shape(self): + cli = self.load_with_flags() + client = cli.TMDBClient() + client.search_tv = Mock(return_value={ + "page": 1, + "total_results": 1, + "results": [{"id": 9626, "name": "Poirot", "first_air_date": "1989-01-08", + "vote_average": 7.9}], + }) + client.search_movie = Mock(side_effect=AssertionError("/search/movie must not be used")) + with patch("builtins.print") as printed: + cli.cmd_tv_search(client, ["--term", "Poirot"]) + payload = json.loads(printed.call_args.args[0]) + self.assertEqual(payload["total"], 1) + self.assertEqual(payload["results"][0]["name"], "Poirot") + self.assertEqual(payload["results"][0]["first_air_date"], "1989-01-08") + client.search_tv.assert_called_once_with("Poirot") + client.search_movie.assert_not_called() + + def test_human_output_formats_name_and_first_air_date_year(self): + cli = load_cli() + cli.GLOBAL_FLAGS = {"json": False, "dry_run": False, "quiet": False, "verbose": False} + client = cli.TMDBClient() + client.search_tv = Mock(return_value={ + "total_results": 1, + "results": [{"id": 9626, "name": "Poirot", "first_air_date": "1989-01-08", + "vote_average": 7.9}], + }) + with patch("builtins.print") as printed: + cli.cmd_tv_search(client, ["--term", "Poirot"]) + line = printed.call_args.args[0] + self.assertIn("Poirot", line) + self.assertIn("(1989)", line) + + +class FindExternalSourceTests(unittest.TestCase): + """Exactly the official eight external_source values are accepted.""" + + def test_all_official_sources_are_accepted_parameterized(self): + cli = load_cli() + cli.GLOBAL_FLAGS = {"json": True, "dry_run": False, "quiet": False, "verbose": False} + self.assertEqual(len(cli.EXTERNAL_SOURCES), 8) + for source in cli.EXTERNAL_SOURCES: + with self.subTest(source=source): + client = cli.TMDBClient() + client.find_external = Mock(return_value={}) + with patch("builtins.print"): + cli.cmd_find(client, [f"ext-{source}", "--source", source]) + client.find_external.assert_called_once_with(f"ext-{source}", source) + + def test_freebase_sources_are_rejected_parameterized(self): + for retired in ("freebase_mid", "freebase_id"): + with self.subTest(source=retired): + result = self.run_cli("--json", "find", "ABC123", "--source", retired) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("Traceback", result.stderr) + self.assertIn(retired, result.stderr) + self.assertIn("invalid choice", result.stderr) + + def run_cli(self, *args): + env = os.environ.copy() + env.pop("TMDB_ACCESS_TOKEN", None) + env.pop("TMDB_API_KEY", None) + return subprocess.run([str(SCRIPT), *args], text=True, capture_output=True, env=env) + + if __name__ == "__main__": unittest.main() diff --git a/tmdb/scripts/tmdb b/tmdb/scripts/tmdb index c8eba31..a626ace 100755 --- a/tmdb/scripts/tmdb +++ b/tmdb/scripts/tmdb @@ -26,6 +26,20 @@ ENV_SERVER = os.getenv("TMDB_SERVER", DEFAULT_SERVER) QUIET = False GLOBAL_FLAGS: Dict[str, Any] = {"json": False, "dry_run": False, "quiet": False, "verbose": False} +# Official external_source values for GET /find/{external_id}, per the current +# TMDb developer documentation. Retired Freebase sources (freebase_mid, +# freebase_id) are deliberately excluded: the API rejects them. +EXTERNAL_SOURCES: Tuple[str, ...] = ( + "imdb_id", + "facebook_id", + "instagram_id", + "tvdb_id", + "tiktok_id", + "twitter_id", + "wikidata_id", + "youtube_id", +) + def log(msg): if not QUIET and not GLOBAL_FLAGS.get("json", False): @@ -190,6 +204,22 @@ def cmd_movie_search(client, args): {"total": data.get("total_results"), "results": results}) +def cmd_tv_search(client, args): + p = argparse.ArgumentParser(prog="tmdb tv search") + p.add_argument("--term", "-t", required=True) + p.add_argument("--limit", type=int, default=10) + parsed, _ = p.parse_known_args(args) + if client.dry_run: + return emit(f"[dry-run] Search TV shows: {parsed.term}", {"dry_run": True}) + data = client.search_tv(parsed.term) or {} + results = data.get("results", [])[:parsed.limit] + if not results: + return emit("No TV shows found.", {"results": []}) + lines = [fmt_tv(r, i+1) for i, r in enumerate(results)] + emit(f"{data.get('total_results', len(results))} result(s):\n" + "\n".join(lines), + {"total": data.get("total_results"), "results": results}) + + def cmd_movie_detail(client, args): p = argparse.ArgumentParser(prog="tmdb movie detail") p.add_argument("movie_id") @@ -204,10 +234,8 @@ def cmd_movie_detail(client, args): def cmd_find(client, args): p = argparse.ArgumentParser(prog="tmdb find") p.add_argument("external_id") - p.add_argument("--source", default="imdb_id", choices=[ - "imdb_id", "facebook_id", "instagram_id", "tvdb_id", "tiktok_id", - "twitter_id", "wikidata_id", "youtube_id", - ]) + p.add_argument("--source", default="imdb_id", choices=EXTERNAL_SOURCES, + help="external_source for /find (Freebase sources are retired)") parsed, _ = p.parse_known_args(args) if client.dry_run: return emit("[dry-run] Find external ID", {"dry_run": True}) @@ -387,12 +415,16 @@ def main(): tp = sub.add_parser("tv", help="TV operations") tsub = tp.add_subparsers(dest="action") s3 = tsub.add_parser("search", help="Search TV"); s3.add_argument("--term", "-t", required=True); s3.add_argument("--limit", type=int, default=10) - s4 = tsub.add_parser("discover", help="Discover TV"); s4.add_argument("--genre"); s4.add_argument("--rating", type=float); s4.add_argument("--from", dest="air_date_gte"); s4.add_argument("--limit", type=int, default=10) - - # flat - sub.add_parser("genre", help="List genres").add_argument("--type", required=True, choices=["movie", "tv"]) + s4 = tsub.add_parser("discover", help="Discover TV"); s4.add_argument("--genre"); s4.add_argument("--rating", type=float); s4.add_argument("--from", dest="air_date_gte"); s4.add_argument("--limit", type=int, default=10) # flat + genre_p = sub.add_parser("genre", help="List genres") + genre_sub = genre_p.add_subparsers(dest="action") + gs = genre_sub.add_parser("list", help="List genres for a media type") + gs.add_argument("--type", required=True, choices=["movie", "tv"]) sub.add_parser("certification", help="List certifications") - fp = sub.add_parser("find", help="Find by external ID"); fp.add_argument("external_id"); fp.add_argument("--source", default="imdb_id") + fp = sub.add_parser("find", help="Find by external ID") + fp.add_argument("external_id") + fp.add_argument("--source", default="imdb_id", choices=EXTERNAL_SOURCES, + help="external_source for /find (Freebase sources are retired)") tr = sub.add_parser("trending", help="Trending content") tr.add_argument("--type", default="movie", choices=["movie", "tv", "all"]) @@ -405,26 +437,33 @@ def main(): sys.exit(1) client = TMDBClient(dry_run=GLOBAL_FLAGS.get("dry_run", False)) + # Dispatch on parsed attributes; positional subcommand names are re-sliced + # from argv only so per-command parsers can reject invalid flags locally. + def slice_after(token): + return filtered_argv[filtered_argv.index(token) + 1:] if token in filtered_argv else [] - # Dispatch if args.resource == "movie": - if args.action == "search": cmd_movie_search(client, filtered_argv[filtered_argv.index("search")+1:]) - elif args.action == "discover": cmd_movie_discover(client, filtered_argv[filtered_argv.index("discover")+1:]) - elif args.action == "detail": cmd_movie_detail(client, filtered_argv[filtered_argv.index("detail")+1:]) - elif args.action == "upcoming": cmd_upcoming(client, filtered_argv[filtered_argv.index("upcoming")+1:]) + if args.action == "search": cmd_movie_search(client, slice_after("search")) + elif args.action == "discover": cmd_movie_discover(client, slice_after("discover")) + elif args.action == "detail": cmd_movie_detail(client, slice_after("detail")) + elif args.action == "upcoming": + cmd_upcoming(client, slice_after("upcoming")) else: parser.print_help() elif args.resource == "tv": - if args.action == "search": cmd_movie_search(client, filtered_argv[filtered_argv.index("search")+1:]) - elif args.action == "discover": cmd_tv_discover(client, filtered_argv[filtered_argv.index("discover")+1:]) + if args.action == "search": cmd_tv_search(client, slice_after("search")) + elif args.action == "discover": cmd_tv_discover(client, slice_after("discover")) else: parser.print_help() elif args.resource == "trending": - cmd_trending(client, filtered_argv[filtered_argv.index("trending")+1:]) + cmd_trending(client, slice_after("trending")) elif args.resource == "genre": - cmd_genre_list(client, filtered_argv[filtered_argv.index("list")+1:]) + if args.action == "list": + cmd_genre_list(client, slice_after("list")) + else: + genre_p.print_help() elif args.resource == "certification": - cmd_cert_list(client, filtered_argv[filtered_argv.index("certification")+1:]) + cmd_cert_list(client, []) elif args.resource == "find": - cmd_find(client, filtered_argv[filtered_argv.index("find")+1:]) + cmd_find(client, slice_after("find")) else: parser.print_help() From eba124cb5e2aae550271c18d08a408999f5490e3 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 18:49:47 -0400 Subject: [PATCH 24/40] fix(trakt): expose --page and normalize X-Pagination headers in JSON Every discovery command (movie/tv x trending/popular/anticipated) now accepts --page alongside --limit and forwards both as query parameters. TraktClient._get returns (data, pagination) where pagination is the X-Pagination-* header set normalized onto the stable keys page, limit, page_count, and item_count; missing or non-numeric headers degrade to an empty object. JSON output keeps the movies/shows array beside a new pagination object, human output appends "Page N of M" only when the headers are present, and cmd_movie/cmd_tv collapse into one shared cmd_discovery handler. Adds mocked coverage for page=2 request params across all six commands, header normalization (full/lowercase/partial/unparseable/missing), missing-header fallback at client level, stable movie trending wrapper and TV shapes beside pagination, and human page-line presence rules. Green under pytest strict-markers, unittest discover, and the proxy trap (zero egress). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- trakt/scripts/test_trakt.py | 178 +++++++++++++++++++++++++++++++++++- trakt/scripts/trakt | 91 ++++++++++++------ 2 files changed, 238 insertions(+), 31 deletions(-) diff --git a/trakt/scripts/test_trakt.py b/trakt/scripts/test_trakt.py index db825c3..ade19bf 100644 --- a/trakt/scripts/test_trakt.py +++ b/trakt/scripts/test_trakt.py @@ -17,7 +17,24 @@ sys.modules[spec.name] = trakt loader.exec_module(trakt) +def _response(items, headers): + response = mock.Mock(status_code=200) + response.json.return_value = items + response.headers = headers + return response + + +FULL_PAGINATION_HEADERS = { + "X-Pagination-Page": "2", + "X-Pagination-Limit": "1", + "X-Pagination-Page-Count": "3405", + "X-Pagination-Item-Count": "10", +} + + class TraktCliTests(TestCase): + """Original CLI surface coverage: help, errors, dry-run, header injection.""" + def run_cli(self, *args, **kwargs): env = os.environ.copy() env.pop("TRAKT_CLIENT_ID", None) @@ -42,8 +59,7 @@ class TraktCliTests(TestCase): @mock.patch.object(trakt.requests, "get") def test_client_injects_required_header_pair(self, get): - response = mock.Mock(status_code=200) - response.json.return_value = [{"movie": {"title": "Example"}}] + response = _response([{"movie": {"title": "Example"}}], {}) get.return_value = response client = trakt.TraktClient(client_id="CLIENT_ID") client.movie_trending(limit=4) @@ -57,12 +73,170 @@ class TraktCliTests(TestCase): def test_client_reports_http_error(self, get, die): response = mock.Mock(status_code=403) response.json.return_value = {"message": "forbidden"} + response.headers = {} get.return_value = response trakt.TraktClient(client_id="CLIENT_ID").movie_popular() die.assert_called_once() self.assertIn("403", die.call_args.args[0]) +class PaginationRequestTests(TestCase): + """--page/--limit flow from argv into request query parameters.""" + + def setUp(self): + flags = {"json": True, "dry_run": False, "quiet": False, "verbose": False} + patcher = mock.patch.object(trakt, "GLOBAL_FLAGS", flags) + patcher.start() + self.addCleanup(patcher.stop) + + def test_page_two_is_sent_as_query_parameter(self): + client = trakt.TraktClient(client_id="CLIENT_ID") + client._get = mock.Mock(return_value=([], {"page": 2})) + with mock.patch("builtins.print"): + trakt.cmd_discovery(client, "movie", "trending", ["--page", "2"]) + client._get.assert_called_once_with("/movies/trending", {"page": 2, "limit": 10}) + + @mock.patch.object(trakt.requests, "get") + def test_requests_get_receives_page_and_limit_params(self, get): + get.return_value = _response([], {}) + client = trakt.TraktClient(client_id="CLIENT_ID") + client.tv_popular(page=3, limit=25) + self.assertEqual(get.call_args.kwargs["params"], {"page": 3, "limit": 25}) + + def test_every_discovery_command_accepts_explicit_page_and_limit(self): + pairs = [("movie", action) for action in ("trending", "popular", "anticipated")] + pairs += [("tv", action) for action in ("trending", "popular", "anticipated")] + segments = {"movie": "movies", "tv": "shows"} + keys = {"movie": "movies", "tv": "shows"} + for resource, action in pairs: + with self.subTest(command=f"{resource} {action}"): + client = trakt.TraktClient(client_id="CLIENT_ID") + client._get = mock.Mock(return_value=(None, {})) + with mock.patch("builtins.print") as printed: + trakt.cmd_discovery(client, resource, action, ["--page", "3", "--limit", "7"]) + expected_path = f"/{segments[resource]}/{action}" + client._get.assert_called_once_with(expected_path, {"page": 3, "limit": 7}) + if trakt.GLOBAL_FLAGS["json"]: + self.assertEqual(json.loads(printed.call_args.args[0]), {keys[resource]: [], "pagination": {}}) + else: + self.assertIn("No", printed.call_args.args[0]) + + def test_dry_run_json_accepts_page_without_network_or_credentials(self): + env = os.environ.copy() + env.pop("TRAKT_CLIENT_ID", None) + result = subprocess.run( + [sys.executable, str(SCRIPT), "--dry-run", "--json", "tv", "anticipated", "--page", "2"], + capture_output=True, text=True, env=env, + ) + self.assertEqual(result.returncode, 0) + self.assertTrue(json.loads(result.stdout)["dry_run"]) + + +class PaginationHeaderTests(TestCase): + """X-Pagination-* normalization and missing-header degradation.""" + + def test_all_four_headers_map_onto_stable_keys(self): + pagination = trakt.normalize_pagination(dict(FULL_PAGINATION_HEADERS)) + self.assertEqual( + pagination, + {"page": 2, "limit": 1, "page_count": 3405, "item_count": 10}, + ) + + def test_lowercase_header_names_are_normalized(self): + lower = {key.lower(): value for key, value in FULL_PAGINATION_HEADERS.items()} + self.assertEqual(trakt.normalize_pagination(lower)["page_count"], 3405) + + def test_missing_headers_degrade_to_empty_object(self): + self.assertEqual(trakt.normalize_pagination({}), {}) + + def test_unparseable_and_partial_values_are_skipped(self): + headers = {"X-Pagination-Page": "2", "X-Pagination-Limit": "", "X-Pagination-Item-Count": "not-a-number"} + pagination = trakt.normalize_pagination(headers) + self.assertEqual(pagination, {"page": 2}) + + @mock.patch.object(trakt.requests, "get") + def test_response_without_pagination_headers_yields_empty_pagination_object(self, get): + get.return_value = _response([{"movie": {"title": "Example"}}], {"Content-Type": "application/json"}) + _, pagination = trakt.TraktClient(client_id="CLIENT_ID").movie_trending() + self.assertEqual(pagination, {}) + + +class DiscoveryOutputTests(TestCase): + """Stable JSON shapes beside the new pagination metadata.""" + + def json_payload_for(self, resource, endpoint, argv, items, headers=None): + flags = {"json": True, "dry_run": False, "quiet": False, "verbose": False} + client = trakt.TraktClient(client_id="CLIENT_ID") + if hasattr(client, f"{resource}_{endpoint}"): + setattr(client, f"{resource}_{endpoint}", + mock.Mock(return_value=(items, trakt.normalize_pagination(headers or {})))) + else: + client._get = mock.Mock(return_value=(items, trakt.normalize_pagination(headers or {}))) + with mock.patch.object(trakt, "GLOBAL_FLAGS", flags), mock.patch("builtins.print") as printed: + trakt.cmd_discovery(client, resource, endpoint, argv) + return json.loads(printed.call_args.args[0]) + + def test_movie_trending_json_keeps_movies_key_beside_pagination(self): + payload = self.json_payload_for( + "movie", "trending", ["--page", "2"], + [{"movie": {"title": "Heat", "year": 1995, "ids": {"tmdb": 949}}}], + FULL_PAGINATION_HEADERS, + ) + self.assertIn("movies", payload) + self.assertEqual(payload["pagination"], + {"page": 2, "limit": 1, "page_count": 3405, "item_count": 10}) + entry = payload["movies"][0] + self.assertEqual(entry["movie"]["title"], "Heat") + self.assertEqual(entry["movie"]["ids"]["tmdb"], 949) + + def test_tv_popular_json_keeps_show_objects_directly_nested(self): + payload = self.json_payload_for( + "tv", "popular", ["--limit", "5"], + [{"title": "Poirot", "year": 1989, "ids": {"tvdb": 70739}}][:1], + FULL_PAGINATION_HEADERS, + ) + self.assertIn("shows", payload) + self.assertEqual(payload["shows"][0]["title"], "Poirot") + self.assertEqual(payload["shows"][0]["ids"]["tvdb"], 70739) + self.assertEqual(payload["pagination"]["page_count"], 3405) + + def test_tv_trending_keeps_wrapper_shape_in_json(self): + payload = self.json_payload_for( + "tv", "trending", [], + [{"show": {"title": "Severance", "ids": {"tvdb": 365278}}}], + FULL_PAGINATION_HEADERS, + ) + self.assertIn("show", payload["shows"][0]) + self.assertEqual(payload["pagination"]["item_count"], 10) + + def test_human_output_states_current_and_total_pages(self): + client = trakt.TraktClient(client_id="CLIENT_ID") + client.movie_trending = mock.Mock(return_value=( + [{"movie": {"title": "Heat", "year": 1995, "ids": {"tmdb": 949}}}], + {"page": 2, "limit": 1, "page_count": 3405, "item_count": 10}, + )) + with mock.patch.object(trakt, "GLOBAL_FLAGS", + {"json": False, "dry_run": False, "quiet": False, "verbose": False}), \ + mock.patch("builtins.print") as printed: + trakt.cmd_discovery(client, "movie", "trending", ["--page", "2"]) + output = printed.call_args.args[0] + self.assertIn("Heat", output) + self.assertIn("Page 2 of 3405", output) + + def test_human_output_without_pagination_headers_prints_no_page_line(self): + client = trakt.TraktClient(client_id="CLIENT_ID") + client.tv_popular = mock.Mock(return_value=( + [{"show": {"title": "Fargo", "year": 2014, "ids": {"tvdb": 269584}}}], {}, + )) + with mock.patch.object(trakt, "GLOBAL_FLAGS", + {"json": False, "dry_run": False, "quiet": False, "verbose": False}), \ + mock.patch("builtins.print") as printed: + trakt.cmd_discovery(client, "tv", "popular", []) + output = printed.call_args.args[0] + self.assertIn("Fargo", output) + self.assertNotIn("Page ", output) + + if __name__ == "__main__": import unittest diff --git a/trakt/scripts/trakt b/trakt/scripts/trakt index 8f9a992..384b87c 100755 --- a/trakt/scripts/trakt +++ b/trakt/scripts/trakt @@ -16,6 +16,16 @@ USER_AGENT = "agent-skills-trakt/1.0" QUIET = False GLOBAL_FLAGS: Dict[str, Any] = {"json": False, "dry_run": False, "quiet": False, "verbose": False} + +# Canonical X-Pagination-* response headers mapped onto the stable JSON keys +# surfaced as the `pagination` object beside every discovery result. +PAGINATION_HEADERS: Tuple[Tuple[str, str], ...] = ( + ("X-Pagination-Page", "page"), + ("X-Pagination-Limit", "limit"), + ("X-Pagination-Page-Count", "page_count"), + ("X-Pagination-Item-Count", "item_count"), +) + def log(m): global QUIET; (not QUIET and not GLOBAL_FLAGS.get("json")) and print(m) def warn(m): print(f"Warning: {m}", file=sys.stderr) def die(m, c=1): print(f"Error: {m}", file=sys.stderr); sys.exit(c) @@ -23,6 +33,25 @@ def emit(h, d): if GLOBAL_FLAGS.get("json"): print(json.dumps(d, default=str)) else: print(h) +def normalize_pagination(headers): + """Map X-Pagination-* headers onto the stable pagination keys. + + Missing or non-numeric header values fall out of the mapping, so a + response without pagination metadata degrades to an empty object + instead of failing. + """ + pagination: Dict[str, int] = {} + for header, key in PAGINATION_HEADERS: + raw = None + for name in (header, header.lower()): + if name in headers: + raw = headers[name] + break + if raw is None: continue + try: pagination[key] = int(str(raw).strip()) + except (TypeError, ValueError): continue + return pagination + def _preparse(argv): BOOLS = {"--json","--dry-run","--quiet","--verbose"} f, fl = {}, [argv[0]] @@ -40,8 +69,9 @@ class TraktClient: self.client_id = client_id or ENV_CLIENT_ID self.dry_run = dry_run def _get(self, path, params=None): + """Fetch one page; returns (json data, normalized pagination dict).""" url = f"{API_BASE}{path}" - if self.dry_run: return [{"dry_run":True, "url":url, "params":params}] + if self.dry_run: return [{"dry_run":True, "url":url, "params":params}], {} if not self.client_id: die("TRAKT_CLIENT_ID not set. Get one from trakt.tv/oauth/applications.") try: r = requests.get(url, params=params, headers={ @@ -53,7 +83,7 @@ class TraktClient: try: d = r.json() except: d = r.text[:200] die(f"API error ({r.status_code}): {d}") - return r.json() + return r.json(), normalize_pagination(dict(r.headers)) def movie_trending(self, page=1, limit=10): return self._get("/movies/trending", {"page":page, "limit":limit}) def movie_anticipated(self, page=1, limit=10): @@ -92,27 +122,28 @@ def fmt_show(d, idx=None): s_out = f" {'%d. '%idx if idx else ''}{t:35}{net_str} ({y}) tvdb={tvdb}" return s_out -def cmd_movie(client, args, endpoint): - p = argparse.ArgumentParser(prog=f"trakt movie {endpoint}") - p.add_argument("--limit", type=int, default=10) +def cmd_discovery(client, resource, endpoint, args): + fmt = fmt_movie if resource == "movie" else fmt_show + list_key = "movies" if resource == "movie" else "shows" + plural = "movies" if resource == "movie" else "TV shows" + label = "Movie" if resource == "movie" else "TV" + p = argparse.ArgumentParser(prog=f"trakt {resource} {endpoint}") + p.add_argument("--page", type=int, default=1, + help="1-based page number to fetch (default 1)") + p.add_argument("--limit", type=int, default=10, + help="items per page (default 10)") parsed, _ = p.parse_known_args(args) - if client.dry_run: return emit(f"[dry-run] Movie {endpoint}", {"dry_run":True}) - fn = getattr(client, f"movie_{endpoint}") - data = fn(limit=parsed.limit) or [] - if not data: return emit(f"No {endpoint} movies.", {"movies":[]}) - lines = [fmt_movie(d, i+1) for i, d in enumerate(data[:parsed.limit])] - emit(f"{endpoint.title()} movies:\n"+"\n".join(lines), {"movies":data[:parsed.limit]}) - -def cmd_tv(client, args, endpoint): - p = argparse.ArgumentParser(prog=f"trakt tv {endpoint}") - p.add_argument("--limit", type=int, default=10) - parsed, _ = p.parse_known_args(args) - if client.dry_run: return emit(f"[dry-run] TV {endpoint}", {"dry_run":True}) - fn = getattr(client, f"tv_{endpoint}") - data = fn(limit=parsed.limit) or [] - if not data: return emit(f"No {endpoint} TV shows.", {"shows":[]}) - lines = [fmt_show(d, i+1) for i, d in enumerate(data[:parsed.limit])] - emit(f"{endpoint.title()} TV:\n"+"\n".join(lines), {"shows":data[:parsed.limit]}) + if client.dry_run: return emit(f"[dry-run] {label} {endpoint}", {"dry_run":True}) + data, pagination = getattr(client, f"{resource}_{endpoint}")(page=parsed.page, limit=parsed.limit) + data = data or [] + entries = data[:parsed.limit] if isinstance(data, list) else [] + payload: Dict[str, Any] = {list_key: entries, "pagination": pagination} + if not entries: return emit(f"No {endpoint} {plural}.", payload) + lines = [fmt(entry, i+1) for i, entry in enumerate(entries)] + output = f"{endpoint.title()} {'movies' if resource == 'movie' else 'TV'}:\n"+"\n".join(lines) + if pagination.get("page") is not None and pagination.get("page_count"): + output += f"\n Page {pagination['page']} of {pagination['page_count']}" + emit(output, payload) def main(): global GLOBAL_FLAGS, QUIET @@ -125,20 +156,22 @@ def main(): mp = sub.add_parser("movie", help="Movie discovery") ms = mp.add_subparsers(dest="action") for a in ["trending","anticipated","popular"]: - ms.add_parser(a, help=f"{a.title()} movies").add_argument("--limit", type=int, default=10) + ap = ms.add_parser(a, help=f"{a.title()} movies") + ap.add_argument("--page", type=int, default=1, help="page number to fetch (default 1)") + ap.add_argument("--limit", type=int, default=10, help="items per page (default 10)") tp = sub.add_parser("tv", help="TV discovery") ts = tp.add_subparsers(dest="action") for a in ["trending","anticipated","popular"]: - ts.add_parser(a, help=f"{a.title()} TV").add_argument("--limit", type=int, default=10) + ap = ts.add_parser(a, help=f"{a.title()} TV") + ap.add_argument("--page", type=int, default=1, help="page number to fetch (default 1)") + ap.add_argument("--limit", type=int, default=10, help="items per page (default 10)") args = parser.parse_args(filtered_argv[1:]) if not args.resource: parser.print_help(); sys.exit(1) client = TraktClient(dry_run=GLOBAL_FLAGS.get("dry_run",False)) remaining = filtered_argv[filtered_argv.index(args.resource)+1:] - if args.resource == "movie": - if args.action: cmd_movie(client, remaining, args.action) - else: mp.print_help() - elif args.resource == "tv": - if args.action: cmd_tv(client, remaining, args.action) + if args.resource in ("movie","tv"): + if args.action: cmd_discovery(client, args.resource, args.action, remaining) + elif args.resource == "movie": mp.print_help() else: tp.print_help() else: parser.print_help() From cc962bb298062fc436f65d01b16aebfdadd5c9a8 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 18:50:30 -0400 Subject: [PATCH 25/40] docs(trakt): executable page loop and pagination metadata contract SKILL.md essential commands show --page usage, add a seq/jq page-loop recipe driven by the pagination.page_count field of the JSON output (with 429 Retry-After handling), and document that --json emits movies/shows plus the normalized pagination object whose keys mirror X-Pagination-* headers, degrading to {} when headers are absent; human output notes the Page N of M footer rule. References state per-invocation paging and the same degradation fallback. Evals replace the header-only pagination case with an executable loop case and a second-page trending case asserting --page, pagination keys, and array preservation. README Quick Start and test-table rows updated to match. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- trakt/README.md | 4 ++-- trakt/SKILL.md | 27 ++++++++++++++++++---- trakt/evals/evals.json | 12 +++++++--- trakt/references/discovery-endpoints.md | 2 ++ trakt/references/recipes-and-operations.md | 13 +++++++++-- 5 files changed, 46 insertions(+), 12 deletions(-) diff --git a/trakt/README.md b/trakt/README.md index e49363a..f176424 100644 --- a/trakt/README.md +++ b/trakt/README.md @@ -12,7 +12,7 @@ Public discovery reads need an application Client ID, not a user login. OAuth bo |---|---| | `SKILL.md` | Command guide, recipes, gotchas, and routing | | `scripts/trakt` | Executable CLI with JSON and dry-run modes | -| `scripts/test_trakt.py` | Offline pytest and unittest suite, including header injection | +| `scripts/test_trakt.py` | Offline pytest and unittest suite, including header injection and pagination | | `references/auth-and-request-contract.md` | Required headers, OAuth boundary, and errors | | `references/discovery-endpoints.md` | Trending/popular/anticipated semantics and paging | | `references/recipes-and-operations.md` | jq pipelines and rate-safe operations | @@ -23,7 +23,7 @@ Public discovery reads need an application Client ID, not a user login. OAuth bo ```sh export TRAKT_CLIENT_ID="YOUR_TRAKT_CLIENT_ID" trakt movie trending --limit 10 -trakt --json tv anticipated | jq '.shows' +trakt --json tv anticipated --page 2 | jq '.pagination' ``` Create a free Client ID at [trakt.tv/oauth/applications](https://trakt.tv/oauth/applications). Preview commands with `trakt --dry-run --json movie popular` without credentials or network access. diff --git a/trakt/SKILL.md b/trakt/SKILL.md index 0406016..7f8ea3e 100644 --- a/trakt/SKILL.md +++ b/trakt/SKILL.md @@ -28,11 +28,13 @@ Every request must send `trakt-api-key: ` together with the mandatory ## Essential commands +All six discovery commands accept `--page N` alongside `--limit N`; both default to 1 and 10 respectively and are forwarded to the API's query string. + ### Trending: watched in the last 24 hours ```sh trakt movie trending --limit 20 -trakt tv trending --limit 20 --json +trakt tv trending --limit 20 --page 2 --json ``` Trending responses wrap each media object in `movie` or `show` and include a `watchers` count. @@ -41,7 +43,7 @@ Trending responses wrap each media object in `movie` or `show` and include a `wa ```sh trakt movie popular --limit 25 --json -trakt tv popular --limit 25 +trakt tv popular --page 2 --limit 25 ``` Popular is a ranking based on rating percentage and number of ratings, not a personalized recommendation. @@ -49,7 +51,7 @@ Popular is a ranking based on rating percentage and number of ratings, not a per ### Anticipated: upcoming interest ```sh -trakt movie anticipated --limit 10 +trakt movie anticipated --page 3 --limit 10 trakt tv anticipated --limit 10 --json ``` @@ -72,11 +74,25 @@ trakt --json movie trending --limit 20 | ### Compare discovery signals -Fetch matching pages of trending, popular, and anticipated, then label each dataset before combining it. Trending is recent watching, popular is broad ranking, and anticipated is upcoming interest. +Fetch matching pages of trending, popular, and anticipated (e.g. `--page 1` for each), then label each dataset before combining it. Trending is recent watching, popular is broad ranking, and anticipated is upcoming interest. + +### Page through anticipated until the feed ends + +Loop `--page`, read `pagination.page_count` from JSON output to pick the stop page, and break early if a page returns no items: + +```sh +for p in $(seq 1 "$(trakt --json movie anticipated --page 1 --limit 100 | jq -r '.pagination.page_count')"); do + trakt --json movie anticipated --page "$p" --limit 100 | + jq --arg p "$p" '{page: ($p|tonumber), pagination: .pagination, + movies: [.movies[] | {title: (.movie.title // .title), year: (.movie.year // null)}]}' +done +``` + +Keep per-page output as labeled NDJSON; merge afterwards. On 429, wait out `Retry-After` before continuing the loop. ## JSON and pagination -`--json` emits a `movies` or `shows` object suitable for `jq`; trending entries retain their wrapper. The API accepts `page` and `limit`, with compatibility defaults of page 1 and limit 10. API responses include `X-Pagination-Page-Count`; automation should stop at that header rather than assuming a short page is the end. +`--json` emits an object with a `movies` or `shows` array (trending entries retain their wrapper) plus a `pagination` object whose keys mirror the API's `X-Pagination-*` headers: `page`, `limit`, `page_count`, `item_count`. Pagination keys are ints when the headers were present and the object is empty `{}` when they were absent, so jq like `.pagination.page_count // 1` degrades safely. Human output appends a `Page N of M` line when the headers are present and stays silent otherwise. The API defaults to page 1 with limit 10 for compatibility; set both explicitly for reproducible automation, and stop at `page_count` rather than assuming a short page is the end. ## Known gotchas @@ -86,6 +102,7 @@ Fetch matching pages of trending, popular, and anticipated, then label each data - **OAuth refresh:** access tokens last seven days and refresh tokens are single-use. Replace the stored refresh token after a successful refresh; `invalid_grant` requires reauthorization. - **Trakt is not TMDb:** Trakt IDs and discovery rankings are not TMDb metadata. Use the `tmdb` skill for credits, images, provider metadata, and catalog enrichment. - **Trending shape:** read `.movie` or `.show` before title/IDs, while preserving `watchers`. +- **Pagination is per invocation:** one CLI call fetches exactly one page (`--page`); loop invocations reading `pagination.page_count` rather than expecting the script to follow links itself. ## When to use diff --git a/trakt/evals/evals.json b/trakt/evals/evals.json index 1a6ea83..7eb6a3c 100644 --- a/trakt/evals/evals.json +++ b/trakt/evals/evals.json @@ -16,9 +16,15 @@ }, { "id": "anticipated-pagination", - "prompt": "Explain how to collect anticipated movies across pages without overrunning Trakt limits.", - "expected_output": "Use page and limit, stop at X-Pagination-Page-Count, and honor Retry-After on 429.", - "assertions": ["names page and limit parameters", "uses the pagination page count header", "handles Retry-After for rate limiting"] + "prompt": "Explain how to collect anticipated movies across pages using the Trakt CLI without overrunning limits.", + "expected_output": "Loop trakt movie anticipated with --page, stop at pagination.page_count from --json output (empty pagination degrades to page 1), and honor Retry-After on 429.", + "assertions": ["uses --page with each CLI invocation", "stops at the normalized pagination.page_count value", "handles Retry-After for rate limiting"] + }, + { + "id": "second-page-trending", + "prompt": "Fetch page 2 of Trakt trending movies as JSON.", + "expected_output": "Run trakt --json movie trending --page 2; JSON output keeps the movies array beside a pagination object mirroring X-Pagination headers.", + "assertions": ["passes --page 2 to the trending command", "names the pagination object keys page limit page_count item_count", "keeps the movies array intact in JSON output"] }, { "id": "header-pair-gotcha", diff --git a/trakt/references/discovery-endpoints.md b/trakt/references/discovery-endpoints.md index 9f7d1d9..76ba4bc 100644 --- a/trakt/references/discovery-endpoints.md +++ b/trakt/references/discovery-endpoints.md @@ -17,6 +17,8 @@ Trending is a short, current watch signal. Popular is a broad popularity ranking These feeds accept `page` and `limit`; compatibility defaults are page 1 and limit 10. Set both explicitly for reproducible automation. Responses provide `X-Pagination-Page`, `X-Pagination-Limit`, `X-Pagination-Page-Count`, and `X-Pagination-Item-Count`. Stop at the reported page count instead of assuming a short page means completion. +The bundled CLI forwards `--page` and `--limit` to the query string and normalizes those four headers into a JSON `pagination` object with the keys `page`, `limit`, `page_count`, and `item_count`. Keys are integers when the headers were present; the object is `{}` when the headers are missing, so downstream jq can fall back with `.pagination.page_count // 1`. + Endpoint pages also document filters such as `extended`, `watchnow`, `genres`, `years`, `ratings`, date ranges, countries, and `ignore_watched`, `ignore_collected`, and `ignore_watchlisted` where supported. Encode comma-separated values as query parameters. `watchnow=any` means any service, while `any_all` and the `free_all`/`subscriptions_all` forms have stricter all-country semantics. ## Result normalization diff --git a/trakt/references/recipes-and-operations.md b/trakt/references/recipes-and-operations.md index b6b84a7..393f103 100644 --- a/trakt/references/recipes-and-operations.md +++ b/trakt/references/recipes-and-operations.md @@ -17,11 +17,20 @@ Fetch the same page of `movie trending`, `movie popular`, and `movie anticipated ## Paginate anticipated releases -Use `page` and `limit` in API clients, inspect `X-Pagination-Page-Count`, and stop at that count. If the response is 429, wait at least the numeric `Retry-After` value and cap retries. The CLI intentionally exposes one page per invocation; shell automation can iterate pages while retaining the response headers in a real HTTP client. +The CLI fetches exactly one page per invocation; loop it. Drive the bound from the normalized pagination metadata: `--json` output carries `.pagination.page_count` (empty `{}` if a response lacked the headers, so fall back with jq's `// 1`). + +```sh +pages=$(trakt --json movie anticipated --page 1 --limit 100 | jq -r '.pagination.page_count // 1') +for p in $(seq 1 "$pages"); do + trakt --json movie anticipated --page "$p" --limit 100 > "anticipated-$p.json" +done +``` + +If you call the API directly instead of through the script, inspect the raw `X-Pagination-Page-Count` header and stop at that count; do not stop merely because a page returned fewer items than `--limit`. If the response is 429, wait at least the numeric `Retry-After` value and cap retries before continuing the loop. ## JSON processing -`--json` emits an object with `movies` or `shows`; trending elements retain their wrapper shape. Use `jq` for selection and `@csv` only after explicitly handling null IDs. Human output is for inspection, JSON output is for pipelines. +`--json` emits an object with `movies` or `shows` plus a `pagination` object (`page`, `limit`, `page_count`, `item_count`); trending elements retain their wrapper shape, and human output adds a `Page N of M` footer only when the headers were present. Use `jq` for selection and `@csv` only after explicitly handling null IDs. Human output is for inspection, JSON output is for pipelines. ## Sources From 1ca147eecdb693280e00005a4d50b7a258449d10 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 19:32:59 -0400 Subject: [PATCH 26/40] fix(openlibrary): harden work/isbn JSON output shapes cmd_work tolerates explicit "authors": null (and non-dict entries) via (data.get("authors") or []) guarded iteration, returning an empty array instead of raising TypeError. cmd_isbn --json now emits authors as an array of bare OLA keys matching `work --json` under the same field: the edition path uses tolerant key extraction (flat refs, stray double-nested refs), dedupes, and falls back to the linked work's double-nested author keys when the edition ships none; publishers likewise become a real list. Human output still renders comma-joined labels. Refreshes the recipe gotcha note that still described the old raw-curl work-record shape. Adds mocked coverage: null-author work record (JSON [] + human '?'), cross-command symmetric author arrays, tolerant edition key-only and mixed-shape refs, ISBN handoff type stability, human label rendering, and label-vs-key separation on mixed records. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- openlibrary/SKILL.md | 5 +- openlibrary/references/recipes-and-gotchas.md | 7 +- openlibrary/scripts/openlibrary | 46 +++++--- openlibrary/scripts/test_openlibrary.py | 108 +++++++++++++++++- 4 files changed, 146 insertions(+), 20 deletions(-) diff --git a/openlibrary/SKILL.md b/openlibrary/SKILL.md index 490af1d..2134a6d 100644 --- a/openlibrary/SKILL.md +++ b/openlibrary/SKILL.md @@ -149,8 +149,9 @@ openlibrary work OL1168083W --json | jq -r '.authors[0]' # bare OL…A ke openlibrary author OL118077A # bio, dates ``` -The CLI keeps this pipeline type-safe: in JSON, `.authors` is an array of bare -OL…A keys; human output renders the same keys as a comma-separated label. Each hop +The CLI keeps this pipeline type-safe: every command that emits an `authors` +field under `--json` (`isbn`, `work`) uses the same shape — an array of bare +OL…A keys — while human output renders comma-separated labels instead. Each hop uses a different key suffix (M → W → A); see Known Gotchas before hand-assembling these URLs yourself. diff --git a/openlibrary/references/recipes-and-gotchas.md b/openlibrary/references/recipes-and-gotchas.md index 64ae929..9afa874 100644 --- a/openlibrary/references/recipes-and-gotchas.md +++ b/openlibrary/references/recipes-and-gotchas.md @@ -30,8 +30,11 @@ openlibrary author "$AUTHOR" --json | jq -r ' ``` Failure modes at each hop: step 1 needs `-L` in curl or you parse an HTML 302 page; -step 3's `.authors[0].author.key` is wrong on *edition* records (flat `.authors[0].key` -there); step 4 crashes naive parsers when `bio` is an object. +the edition record behind step 1 frequently ships `"authors": null` outright — the +CLI absorbs that plus the nested-vs-flat split (works double-nest +`.authors[].author.key`, editions stay flat `.authors[].key`) by falling back to the +linked work, so build pipelines on its `--json` handoff instead of hand-probing +record fields; step 4 crashes naive parsers when `bio` is an object. ## Recipe 2: search → filter to readable ebooks → fetch editions of the top hit diff --git a/openlibrary/scripts/openlibrary b/openlibrary/scripts/openlibrary index 49c7817..af1a08a 100755 --- a/openlibrary/scripts/openlibrary +++ b/openlibrary/scripts/openlibrary @@ -241,30 +241,43 @@ def cmd_isbn(client, args): return title = data.get("title", "?") - authors_list = data.get("authors", []) + work_keys = [normalize_olid(w.get("key", "")) for w in data.get("works", [])] + edition_authors = [a for a in (data.get("authors") or []) if isinstance(a, dict)] + + def _edition_author_key(a: Dict) -> str: + # Canonical editions nest flat ({"key": "/authors/OL…A"}); tolerate + # stray double-nested refs ({"author": {"key": ...}}) so one accessor + # survives both wiki shapes. + ref = a.get("key") or (a.get("author") or {}).get("key") or "" + return normalize_olid(ref) def _author_label(a: Dict) -> str: # Edition records often carry key-only author refs (no embedded name); # fall back to the bare OL…A so output is never just '?'. - label = a.get("name") or normalize_olid(a.get("key", "")) + label = a.get("name") or _edition_author_key(a) return label or "?" - author_names = ", ".join(_author_label(a) for a in authors_list) if authors_list else "" - work_keys = [normalize_olid(w.get("key", "")) for w in data.get("works", [])] + author_names = ", ".join(_author_label(a) for a in edition_authors) + # JSON hands off bare OL…A keys (same shape as `work --json`); display + # labels remain a human-surface concern only. + author_keys = sorted({ + k for k in (_edition_author_key(a) for a in edition_authors) if k + }) - if not author_names and work_keys: + if not author_keys and work_keys: # Some editions ship authors:null entirely. The authoritative author # links live on the work (double-nested authors[].author.key) — one # extra read beats reporting an unknown author. work_data = client._get(f"/works/{work_keys[0]}.json") or {} - work_authors = [ - normalize_olid(a.get("author", {}).get("key", "")) + author_keys = [ + normalize_olid((a.get("author") or {}).get("key", "")) for a in (work_data.get("authors") or []) if isinstance(a, dict) ] - author_names = ", ".join(k for k in work_authors if k) - if not author_names: - author_names = "?" + author_keys = [k for k in author_keys if k] + if not author_names: + author_names = ", ".join(author_keys) + author_names = author_names or "?" pages = data.get("number_of_pages", data.get("pagination", "?")) publishers = ", ".join(data.get("publishers", [])) or "?" publish_date = data.get("publish_date", "?") @@ -283,9 +296,10 @@ def cmd_isbn(client, args): f" Subjects: {subjects}" f"{desc_short}\n" f" Edition: {edition_key} Works: {', '.join(work_keys) or '?'}", - {"isbn": parsed.isbn, "title": title, "authors": author_names, + {"isbn": parsed.isbn, "title": title, "authors": author_keys, "pages": pages, "publish_date": publish_date, - "publishers": publishers, "subjects": data.get("subjects", []), + "publishers": [p for p in (data.get("publishers") or []) if p], + "subjects": data.get("subjects", []), "description": description, "edition_key": edition_key, "work_keys": work_keys, @@ -348,11 +362,13 @@ def cmd_work(client, args): return title = data.get("title", "?") - authors = data.get("authors", []) + # Rare records carry an explicit "authors": null; a default-value .get + # alone does not protect against iterating None. + authors = [a for a in (data.get("authors") or []) if isinstance(a, dict)] author_keys = [ - normalize_olid(a.get("author", {}).get("key", "")) + normalize_olid((a.get("author") or {}).get("key", "")) for a in authors - if isinstance(a, dict) and a.get("author", {}).get("key") + if (a.get("author") or {}).get("key") ] author_str = ", ".join(author_keys) or "?" desc = unwrap_text(data.get("description", "")) diff --git a/openlibrary/scripts/test_openlibrary.py b/openlibrary/scripts/test_openlibrary.py index cb477c3..4d4b2d9 100644 --- a/openlibrary/scripts/test_openlibrary.py +++ b/openlibrary/scripts/test_openlibrary.py @@ -215,7 +215,90 @@ class MockedClientTests(unittest.TestCase): self.assertEqual(code, 0) self.assertEqual(req.call_count, 2) self.assertTrue(req.call_args_list[1].args[0].endswith(WORK_KEY + ".json")) - self.assertEqual(json.loads(out)["authors"], "OL118077A") + self.assertEqual(json.loads(out)["authors"], ["OL118077A"]) + + def test_work_with_explicit_null_authors_returns_empty_array(self): + # Real-world work records sometimes carry an explicit "authors": null; + # the CLI must render '?' and hand off [] instead of raising TypeError. + record = dict(WORK_RECORD, authors=None) + with mock.patch.object(requests, "get", + return_value=FakeResponse(200, record)): + code, out, _ = run_cli("--json", "work", WORK_KEY) + self.assertEqual(code, 0) + data = json.loads(out) + self.assertIsInstance(data["authors"], list) + self.assertEqual(data["authors"], []) + ol_cli.GLOBAL_FLAGS.update(json=False) + try: + with mock.patch.object(requests, "get", + return_value=FakeResponse(200, record)): + human_code, human_out, _ = run_cli("work", WORK_KEY) + finally: + ol_cli.GLOBAL_FLAGS.update(json=True) + self.assertEqual(human_code, 0) + self.assertIn("?", human_out) + + def test_isbn_and_work_emit_same_json_type_under_authors_key(self): + # Symmetry contract: any "authors"-keyed field across CLI commands is a + # JSON array of bare OL…A key strings. Downstream jq pipelines can + # treat .authors identically regardless of the entry command. + edition_authored = { + "type": {"key": "/type/edition"}, + "key": EDITION_KEY, + "title": "Nineteen Eighty-Four", + "authors": [{"key": AUTHOR_KEY}], + "works": [{"key": WORK_KEY}], + } + edition = FakeResponse(200, edition_authored) + with mock.patch.object(requests, "get", + return_value=FakeResponse(200, WORK_RECORD)): + _, work_out, _ = run_cli("--json", "work", WORK_KEY) + _, isbn_out, _ = run_cli("--json", "isbn", "9780451524935") + work_data = json.loads(work_out) + isbn_data = json.loads(isbn_out) + for data in (work_data, isbn_data): + self.assertIsInstance(data["authors"], list) + self.assertNotIsInstance(data["authors"], str) + self.assertTrue(all( + isinstance(k, str) and k.endswith("A") + for k in data["authors"])) + self.assertEqual(isbn_data["authors"], ["OL118077A"]) + self.assertEqual(work_data["authors"], ["OL118077A"]) + + def test_edition_key_only_author_refs_become_bare_keys_in_json(self): + edition_authored = { + "type": {"key": "/type/edition"}, + "key": EDITION_KEY, + "title": "Nineteen Eighty-Four", + "authors": [{"key": "/authors/" + "OL118077A"}, + {"key": "/authors/" + "OL7862984A"}], + "works": [{"key": WORK_KEY}], + } + edition = FakeResponse(200, edition_authored) + with mock.patch.object(requests, "get", return_value=edition) as req: + code, out, _ = run_cli("--json", "isbn", "9780451524935") + self.assertEqual(code, 0) + self.assertEqual(req.call_count, 1) # no fallback read needed + data = json.loads(out) + self.assertEqual(data["authors"], + sorted(["OL118077A", "OL7862984A"])) + + def test_isbn_human_output_still_shows_comma_joined_labels(self): + # The display surface is unchanged: comma-joined names on stdout while + # --json carries the array shape. + edition = FakeResponse(200, { + "type": {"key": "/type/edition"}, "key": EDITION_KEY, + "title": "Nineteen Eighty-Four", + "authors": [{"name": "George Orwell"}, {"name": "Thomas Pynchon"}], + "works": [{"key": WORK_KEY}]}) + ol_cli.GLOBAL_FLAGS.update(json=False) + try: + with mock.patch.object(requests, "get", return_value=edition): + code, out, _ = run_cli("isbn", "9780451524935") + finally: + ol_cli.GLOBAL_FLAGS.update(json=True) + self.assertEqual(code, 0) + self.assertIn("George Orwell, Thomas Pynchon", out) def test_work_json_authors_are_bare_keys_array(self): with mock.patch.object(requests, "get", @@ -257,6 +340,29 @@ class MockedClientTests(unittest.TestCase): self.assertTrue(all(isinstance(key, str) for key in data["authors"])) self.assertIsInstance(data["subjects"], list) + def test_work_record_with_non_dict_author_entries_is_tolerated(self): + # Malformed wiki payloads can smuggle bare strings into authors[]; + # tolerate-and-filter beats crash. + record = dict(WORK_RECORD, + authors=[{"author": {"key": AUTHOR_KEY}}, None]) + with mock.patch.object(requests, "get", + return_value=FakeResponse(200, record)): + code, out, _ = run_cli("--json", "work", WORK_KEY) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out)["authors"], ["OL118077A"]) + + def test_isbn_handoff_fields_have_stable_json_types(self): + with mock.patch.object(requests, "get", + return_value=FakeResponse( + 200, dict(EDITION_RECORD, authors=None), + url="https://openlibrary.org/books/x.json")): + _, out, _ = run_cli("--json", "isbn", "9780451524935") + data = json.loads(out) + for key in ("edition_key", "title", "description"): + self.assertIsInstance(data[key], str) + for key in ("authors", "work_keys", "publishers", "subjects"): + self.assertIsInstance(data[key], list) + def test_merge_redirect_stub_in_http_200_is_followed_with_json_suffix(self): # Merged-away keys answer 200 with {type:/type/redirect, location}, # NOT a 3xx — the client must detect the stub and refetch. Stub From 3bcf4762022abf701997b1bfb41a6850c7aba8e0 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 19:35:15 -0400 Subject: [PATCH 27/40] fix(openlibrary): warn when merge-stub redirect walk exhausts its budget Addresses the tracked follow-up debt from thicken-openlibrary: the client already bounds /type/redirect stub chasing at MAX_REDIRECT_HOPS, but a chain that outlives the budget silently handed back an opaque stub, which downstream commands rendered as an empty-shaped record with no hint why. The walk now emits a stderr warning naming the unresolved location before returning; mocked test drives HOPS+1 chained stubs end to end. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- openlibrary/scripts/openlibrary | 5 +++++ openlibrary/scripts/test_openlibrary.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/openlibrary/scripts/openlibrary b/openlibrary/scripts/openlibrary index af1a08a..abb82ae 100755 --- a/openlibrary/scripts/openlibrary +++ b/openlibrary/scripts/openlibrary @@ -148,6 +148,11 @@ class OpenLibraryClient: params = None hops += 1 continue + if is_redirect_stub(data): + # Bounded stub walk exhausted (>N chained merges); make that + # visible instead of handing back an opaque stub silently. + warn(f"Redirect chain did not resolve within {MAX_REDIRECT_HOPS}" + f" hops (still stuck at {data.get('location', '?')})") self.last_url = resp.url return data diff --git a/openlibrary/scripts/test_openlibrary.py b/openlibrary/scripts/test_openlibrary.py index 4d4b2d9..f892d0e 100644 --- a/openlibrary/scripts/test_openlibrary.py +++ b/openlibrary/scripts/test_openlibrary.py @@ -379,6 +379,25 @@ class MockedClientTests(unittest.TestCase): "https://openlibrary.org" + WORK_KEY + ".json") self.assertEqual(json.loads(out)["title"], "Nineteen Eighty-Four") + def test_redirect_stub_chain_beyond_hop_budget_warns_instead_of_silence(self): + # A work that keeps resolving into further merge stubs exhausts the + # bounded walk; the CLI must say so (stderr warning) rather than emit + # an unexplained /type/redirect payload. + stub = FakeResponse(200, {"type": {"key": "/type/redirect"}, + "location": WORK_KEY}) + responses = [stub] * (ol_cli.MAX_REDIRECT_HOPS + 1) + with mock.patch.object(requests, "get", + side_effect=responses) as req: + code, out, err = run_cli("--json", "work", "OL24776360W") + self.assertEqual(code, 0) + self.assertEqual(req.call_count, ol_cli.MAX_REDIRECT_HOPS + 1) + self.assertIn("did not resolve", err) + # Without the resolution the command degrades to an empty-shaped + # record; the stderr warning is what keeps that from being silent. + self.assertEqual(json.loads(out), { + "key": "OL24776360W", "title": "?", "authors": [], + "description": "", "subjects": [], "cover_url": None}) + def test_text_wrapper_dict_is_unwrapped(self): self.assertEqual(ol_cli.unwrap_text({"type": "/type/text", "value": "hi"}), "hi") self.assertEqual(ol_cli.unwrap_text("plain"), "plain") From cc7e61ed30127f4b8b27dbb5573de5739b70cba2 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Wed, 26 Aug 2026 21:24:20 -0400 Subject: [PATCH 28/40] docs(ghost): thicken Admin API skill Full lastfm-model rebuild of the ghost skill against current docs.ghost.org research: - Fix JWT signer correctness: hex-decode the secret half before HMAC-SHA256 signing (official contract; literal-hex signing produced invalid tokens), document HS256 + kid header + aud /admin/ + 5-minute token window, add admin_api_audience() derivation and Ghost-scheme error handling with researched signatures (409 UPDATE_COLLISION, 404 non-public guidance, INVALID_AUTH_HEADER hint, 204 delete tolerance). - Extend CLI surface: get-post, update-post (updated_at collision guard), delete-post, create-page, create-tag, posts pagination (--page/--order, meta.pagination surfaced), scheduled posting with --published-at guard; dry-run now previews method/URL/payload exactly as executed. - Add 5 cited reference files (auth/basics, content-vs-admin split incl. draft-visibility asymmetry, endpoint guide, worked recipes, gotchas). - Add scripts/test_ghost.py: 28 offline tests incl. fixed-vector JWT known-answer checks and jq-executed pipeline-consumability chains. - Add evals/evals.json (6 cases incl. npm ghost-cli negative probe). - Rewrite SKILL.md (155-line body) and README in lastfm model. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/marketplace.json | 2 +- ghost/README.md | 46 +- ghost/SKILL.md | 190 +++++--- ghost/evals/evals.json | 42 ++ ghost/references/admin-auth-and-basics.md | 115 +++++ ghost/references/content-vs-admin-api.md | 53 +++ ghost/references/gotchas-field-guide.md | 62 +++ .../references/posts-pages-tags-endpoints.md | 121 +++++ ghost/references/worked-recipes.md | 132 ++++++ ghost/scripts/ghost | 438 +++++++++++++---- ghost/scripts/test_ghost.py | 446 ++++++++++++++++++ llms.txt | 2 +- 12 files changed, 1464 insertions(+), 185 deletions(-) create mode 100644 ghost/evals/evals.json create mode 100644 ghost/references/admin-auth-and-basics.md create mode 100644 ghost/references/content-vs-admin-api.md create mode 100644 ghost/references/gotchas-field-guide.md create mode 100644 ghost/references/posts-pages-tags-endpoints.md create mode 100644 ghost/references/worked-recipes.md create mode 100644 ghost/scripts/test_ghost.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index ba6d21b..b0a4d98 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -489,7 +489,7 @@ "./ghost" ], "strict": false, - "description": "Manage Ghost CMS content from the terminal — create and list posts, pages, and tags, and fetch site info via the Ghost Admin API (v5/v6). Use when the user asks about ghost, cms, blog, blogging, posts, pages, tags, publishing, or site configuration." + "description": "Manage Ghost CMS content over the Admin API — browse posts, pages, and tags, draft and publish content, schedule posts, and inspect site info from the terminal. Do not use this skill for Ghost server installation or site administration (installing, nginx, SSL, systemd, updates); those belong to the official npm ghost-cli tooling." }, { "name": "github-runner", diff --git a/ghost/README.md b/ghost/README.md index e600f7e..7b48cdd 100644 --- a/ghost/README.md +++ b/ghost/README.md @@ -1,36 +1,52 @@ -# Ghost CMS from the Terminal +# Ghost CMS content management from the terminal -Manage content on a Ghost CMS site: view site info, list and create posts and pages, manage tags — all via the Ghost Admin API (v5/v6). +Let your agent browse, draft, publish, and schedule content on a Ghost blog or newsletter site through the Admin API — no web editor required. ## Why Install This Skill -When your agent loads this skill, it can **manage your Ghost CMS content** without the web editor. That means: +Editing a Ghost site usually means clicking through the admin UI. This skill hands your agent direct, scripted control instead: -- **List posts and pages** — by status (published, draft, scheduled) -- **Create content** — write and publish blog posts from the terminal -- **Manage tags** — list and browse tags -- **Check site info** — title, URL, description, version +- **See the whole editorial state** — published posts, plus the drafts and scheduled queue that public site feeds never show. +- **Publish programmatically** — create posts and pages as drafts, then flip them live after review, with safe collision-checked updates. +- **Schedule content** — stage posts to appear at future times. +- **Keep tags tidy** — list tags with usage counts, add new ones, drive exports. + +It speaks Ghost's exact authentication dialect automatically: your Admin API key (`id:secret`) becomes a fresh short-lived signed JWT for every command, so there are no tokens to mint, rotate, or paste anywhere. + +Not to be confused with Ghost's official npm `ghost-cli` tool, which installs and operates Ghost servers (`ghost install`, nginx/SSL setup, upgrades). This skill manages *content* on an already-running site; that one manages *servers*. ## What You Get -| Directory | Purpose | -|-----------|---------| -| `SKILL.md` | Complete command reference with setup and examples | -| `scripts/ghost` | CLI tool for Ghost Admin API operations | +| Path | Purpose | +|------|---------| +| `SKILL.md` | Command reference: setup, browse/create/publish recipes, gotchas | +| `scripts/ghost` | CLI tool covering site info, post/page/tag operations, JSON + dry-run modes | +| `scripts/test_ghost.py` | Offline test suite for the CLI (all network mocked) | +| `references/admin-auth-and-basics.md` | Full JWT signing walkthrough with auth error signatures | +| `references/content-vs-admin-api.md` | Content vs Admin API choice guide and draft-visibility trap | +| `references/posts-pages-tags-endpoints.md` | Endpoint map, pagination loop patterns, error envelope | +| `references/worked-recipes.md` | Copy-paste workflows: draft→publish, exports, scheduling | +| `references/gotchas-field-guide.md` | Symptom-first troubleshooting for common failures | ## Quick Start ```bash export GHOST_URL="https://your-ghost-site.com" -export GHOST_ADMIN_KEY="your-id:your-secret" +export GHOST_ADMIN_KEY="" # Ghost Admin → Integrations → Custom integration + +ghost site # connectivity check +ghost posts --status draft +ghost create-post --title "Hello from the terminal" --html "

First!

" ``` -API key from Ghost Admin → Integrations → Create custom integration. +Preview any write safely by adding `--dry-run` before the subcommand. ## Triggers -Load this when working with Ghost CMS — managing posts, pages, tags, or checking site configuration. +Load this skill when working with Ghost CMS content: listing posts/pages/tags, drafting or publishing blog content, scheduling posts, exporting site content, fixing Ghost API authentication errors, or investigating why drafts don't show up in a Ghost site feed. ## Requirements -Python 3.8+ with `requests` library. +- Python 3.8+ with the `requests` library. +- A running Ghost 5.x/6.x site where you can create a Custom Integration (Ghost Admin → Settings → Integrations). +- The integration's **Admin API Key** exported as `GHOST_ADMIN_KEY`, plus the site URL as `GHOST_URL`. Keep the key server-side; it signs mutations and must never ship in client code or CI logs. diff --git a/ghost/SKILL.md b/ghost/SKILL.md index 701ce72..02e4a6d 100644 --- a/ghost/SKILL.md +++ b/ghost/SKILL.md @@ -1,128 +1,170 @@ --- name: ghost -description: Manage Ghost CMS content from the terminal — create and list posts, pages, - and tags, and fetch site info via the Ghost Admin API (v5/v6). Use when the user - asks about ghost, cms, blog, blogging, posts, pages, tags, publishing, or site configuration. +description: Manage Ghost CMS content over the Admin API — browse posts, pages, + and tags, draft and publish content, schedule posts, and inspect site info from + the terminal. Do not use this skill for Ghost server installation or site + administration (installing, nginx, SSL, systemd, updates); those belong to the + official npm ghost-cli tooling. license: MIT compatibility: Requires GHOST_URL and GHOST_ADMIN_KEY env vars. Admin key in "id:secret" format from Ghost Admin → Integrations. Python 3.8+ and the `requests` library. metadata: tags: ghost, cms, blog, blogging, post, page, tag, ghost-cms, content-management, api-client - sources: https://ghost.org/docs/admin-api/, https://ghost.org/docs/ + sources: https://docs.ghost.org/admin-api/, https://docs.ghost.org/content-api/ --- -# ghost — Ghost CMS from the Terminal +# ghost — Ghost CMS content from the terminal -Manage content on a Ghost CMS site: view site info, list and create posts and pages, manage tags — all via the Ghost Admin API (v5/v6). +Drive a Ghost CMS site's Admin API (v5/v6, `Accept-Version: v6.0`): list posts by status including drafts, create and publish pages and posts, manage tags, and check site info. Drafts, scheduled posts, and published content are all visible here because every call authenticates with a per-request Admin JWT built from your `id:secret` integration key. ## Setup -1. Get your Admin API key from **Ghost Admin → Settings → Advanced → Integrations** (or **Ghost Admin → Integrations**). Create a custom integration to get a key in `id:secret` format. -2. Set these environment variables: - ```bash -export GHOST_URL="https://your-ghost-site.com" # your Ghost site URL -export GHOST_ADMIN_KEY="your-id:your-secret" # from Ghost Admin → Integrations +export GHOST_URL="https://your-ghost-site.com" +export GHOST_ADMIN_KEY="" # id:secret from Ghost Admin → Integrations ``` -`--help` and `--dry-run` work without credentials (lazy auth). +1. In **Ghost Admin → Settings → Integrations**, create (or open) a Custom Integration. +2. Copy its **Admin API Key** — one string, two colon-separated hex halves (`id:secret`). The separate **Content API key** from the same screen will NOT let you see drafts; see Known Gotchas. +3. At request time the CLI signs a short-lived JWT per call: HS256 signature keyed by the secret half **after hex-decoding it to raw bytes**, `kid` header carrying the id half, audience `/admin/`, `exp` five minutes after `iat`, sent as `Authorization: Ghost `. You never handle the token yourself. +4. `--help` and `--dry-run` work without credentials (lazy auth). -## Essential Commands +## Essential commands -### site — Get site information +### Inspect ```bash -ghost site # show site title, URL, description -ghost --json site # machine-readable JSON -ghost --dry-run site # preview without API call +ghost site # title, url, description, version +ghost get-post POST_ID # full record incl. exact updated_at for edits ``` -Shows: site title, URL, description. - -### posts — List blog posts +### Browse (intent: find content) ```bash -ghost posts # 20 most recent posts -ghost posts --limit 50 # more results -ghost posts --status published # only published posts -ghost posts --status draft # only draft posts -ghost posts --status scheduled # only scheduled posts -ghost posts --limit 10 --json # 10 most recent as JSON +ghost posts # latest 20 +ghost posts --status draft # unpublished work queue +ghost posts --status scheduled # what publishes next +ghost posts --limit 100 --page 2 # paginate (max page size 100) +ghost posts --order "updated_at desc" # SQL-style ordering +ghost pages # static pages +ghost tags # tags with usage counts ``` -Shows: title, status, slug, and last-updated date for each post. - -### create-post — Create a new blog post +### Create and publish ```bash -ghost create-post --title "My First Post" # draft, no HTML -ghost create-post --title "Hello World" --html "

Hello!

" # with HTML content -ghost create-post --title "Ready" --html "

Published

" --status published # publish immediately -ghost create-post --title "Scheduled" --html "

Later

" --status scheduled # schedule -ghost create-post --title "Custom Slug" --slug "my-custom-url" # custom URL slug -ghost create-post --title "Draft" --dry-run # preview without creating +ghost create-post --title "Notes" # safe default: draft +ghost create-post --title "Hello" --html "

Hi

" +ghost create-post --title "Launch" --status published --html "

We're live

" +ghost create-post --title "Later" --status scheduled \ + --published-at "2026-09-01T09:00:00.000Z" # future ISO-8601 required together +ghost create-page --title "About" --html "

…

" --slug about +ghost create-tag --name "Engineering" --description "Technical posts" ``` -Creates the post and returns its title, slug, and status. - -### pages — List pages +### Edit and remove ```bash -ghost pages # 20 most recent pages -ghost pages --limit 50 # more results -ghost pages --json # machine-readable JSON +ghost update-post POST_ID --title "New title" \ + --updated-at "" # REQUIRED: latest updated_at, re-read first +ghost update-post POST_ID --status published --updated-at "" +ghost delete-post POST_ID # permanent, 204-style removal ``` -Shows: title, status, slug, and last-updated date for each page. +## Pipeline recipes -### tags — List tags +### Draft now, publish after review ```bash -ghost tags # 50 tags with post counts -ghost tags --limit 100 # more results -ghost tags --json # machine-readable JSON +ghost --json create-post --title "Release notes" > /tmp/post.json +id=$(jq -r '.post.id // .post_id // empty' /tmp/post.json) +ghost get-post "$id" # read fresh updated_at +ghost update-post "$id" --status published \ + --updated-at "" ``` -Shows: tag name, slug, and number of posts using each tag. +Never fabricate `updated_at`; copy it verbatim from a fresh read or Ghost rejects the edit with HTTP 409 `UpdateCollisionError`. -## Global Flags - -These flags work anywhere in the command — before or after the subcommand: +### Review queue across statuses ```bash -ghost --json posts # JSON output -ghost posts --json # same result, after subcommand -ghost --dry-run create-post --title "Test" # preview without API call -ghost --quiet posts # suppress diagnostic output -ghost --verbose site # verbose logging +for s in draft scheduled; do + ghost posts --status "$s" --json | jq -r '.posts[] | "\(.status)\t\(.title)\t\(.slug)"' +done ``` +### Complete export + +Loop pages by `meta.pagination.next` (surfaced as `.page.next`) instead of trusting totals: + +```bash +page=1 +while :; do + ghost posts --limit 100 --page "$page" --json > "/tmp/posts-$page.json" + jq -r '.posts[].id' "/tmp/posts-$page.json" + next=$(jq -r '.page.next // empty' "/tmp/posts-$page.json") + [ -z "$next" ] && break + page=$next; sleep 0.2 +done +``` + +## JSON output and jq + +`--json` works before or after the subcommand: + +```bash +ghost --json posts # same as: ghost posts --json +``` + +JSON shapes worth knowing: + +- Lists emit `{"total", "page": {pagination}, "posts": [...]}`; detail/create emit the resource under its noun (`post`, `page`, `tag`, `site`). +- Pagination mirrors the API: `.page = {"page", "limit", "pages", "total", "next", "prev"}`; `next`/`prev` are numbers or `null`. +- `--dry-run --json` emits the executed plan instead of results: `{"dry_run": true, "method", "url", "params"/"json"}` — preview the exact request before running it live. +- Errors exit non-zero with the API's own message plus code on stderr; JSON mode never wraps errors in stdout JSON. + +## Global flags + | Flag | Effect | |------|--------| -| `--json` | Output machine-readable JSON instead of human-readable text | -| `--dry-run` | Show what API call would be made without executing it | -| `--quiet` | Suppress non-essential diagnostic output | -| `--verbose` | Enable verbose/debug logging | +| `--json` | Machine-readable JSON (position-independent) | +| `--dry-run` | Print the planned API call (method, URL, payload) without executing | +| `--quiet` | Suppress diagnostics | +| `--verbose` | Debug logging | -## Known Gotchas +## Known gotchas -- **Admin API key format** — The `GHOST_ADMIN_KEY` must be in `id:secret` format (e.g. `644a4c1a2b3c4d5e6f7g8h9i:abcd1234efgh5678ijkl9012`). This is the format Ghost generates when you create a Custom Integration. A plain token or JWT will not work. -- **JWT token auto-generated** — The CLI generates a short-lived JWT (HS256, 5-minute expiry) internally from the Admin API key on each request. You don't need to create or manage JWT tokens yourself. -- **5-minute JWT window** — Each JWT is valid for 300 seconds (5 minutes). If your system clock is significantly skewed, requests may fail. Ensure NTP is synced. -- **API version v6** — The CLI sends `Accept-Version: v6.0` on all requests, targeting the Ghost Admin API v6. Response shapes follow the v6 spec. May also work against v5 sites. -- **HTML content format** — Post and page content must be provided as raw HTML strings via `--html`. Markdown is not auto-converted. If you write in Markdown, convert it to HTML first (e.g. with a markdown-to-html tool). -- **No update or delete commands** — The current CLI supports listing and creating posts/pages/tags, but does not include update or delete operations. Use the Ghost Admin UI or direct API calls for those. -- **No tag creation via CLI** — Tag listing works, but `create-tag` is not exposed as a subcommand. The GhostClient class has a `create_tag` method internally but it is not wired to a CLI command. -- **Rate limiting** — Ghost Admin API enforces rate limits. For heavy operations, stagger your requests. -- **Error output** — API errors (4xx/5xx) include the response body in the error message for debugging. Auth errors (401/403) explicitly tell you to check `GHOST_ADMIN_KEY`. +- **Drafts need the Admin plane.** The public Content API (that key-as-query-param API) serves published posts only and hides drafts silently — no error, just absent, even with a perfectly valid key. Its filters like `status:draft` are ignored rather than rejected. Everything this CLI does goes through the Admin API precisely so drafts and scheduled posts stay reachable. +- **Two keys, same integration screen.** The Content key is browser-safe but read-only-public; the Admin key (`GHOST_ADMIN_KEY`) signs mutations and reaches drafts. Never point scripts at the Content key and expect draft visibility. +- **Five-minute tokens.** Each JWT lives at most 300 seconds (`exp ≤ iat + 300`) and the verifier caps token age too, so long batch jobs must re-sign per request (the CLI does). Skewed clocks break signing windows; keep NTP healthy. +- **HS256 only, decoded-secret keying.** Tokens signed with HS512 are refused ("invalid algorithm"); signing without first hex-decoding the secret half produces "valid-looking" garbage that 401s. The CLI handles both rules. +- **`Authorization: Ghost`, not Bearer.** `Bearer` scheme answers 401 `INVALID_AUTH_HEADER`. +- **Edits require collision guards.** PUTs without the post's current `updated_at` fail with 409; relation arrays (`tags`, `authors`) replace wholesale rather than merge. +- **HTML ingestion is lossy without cards.** Send proper Lexical, or wrap fixed markup in HTML card comments when using `--html`. +- **Pagination caps at 100** since Ghost 6 removed `limit=all`; oversized limits silently return ≤100 rows, so always loop by `next`. +- **Deletion is permanent** and takes effect on the public site immediately. -## References +## When to use -- [scripts/ghost](scripts/ghost) — The CLI binary. Built following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, structured logging. -- [Ghost Admin API Docs](https://ghost.org/docs/admin-api/) — Official Ghost Admin API documentation. -- [Ghost Integrations](https://ghost.org/docs/integrations/) — How to create Custom Integrations and get your Admin API key. +Use this skill whenever the task is content workflow against a running Ghost site: browsing or exporting posts, drafting, publishing, scheduling, tag upkeep, page creation, or diagnosing those flows (auth errors, pagination, missing drafts). ## When not to use -Do not use this skill for Ghost site administration that requires the admin dashboard (themes, staff accounts, membership tiers, sending settings), for front-end theme development, or for other publishing platforms — WordPress, Hugo, and Jekyll each have their own tooling. +Do not use it to install, host, or operate a Ghost server (`ghost install`, nginx/SSL/systemd setup, upgrades, backups) — that is Ghost's official npm ghost-cli site-management tool, unrelated despite the shared name. Not for other publishing platforms (WordPress, Hugo have their own tooling), not for theme development, and not for site configuration better done once in the Admin dashboard (staff accounts, membership tiers). + +## Reference files + +| File | Use it for | +| ---- | ---------- | +| [references/admin-auth-and-basics.md](references/admin-auth-and-basics.md) | Full JWT signing walkthrough, key format, audience/expiry rules, auth error table | +| [references/content-vs-admin-api.md](references/content-vs-admin-api.md) | Choosing between planes; the draft-visibility trap; diagnostic checklist | +| [references/posts-pages-tags-endpoints.md](references/posts-pages-tags-endpoints.md) | Endpoint map, field semantics, pagination loop, error envelope | +| [references/worked-recipes.md](references/worked-recipes.md) | Copy-paste workflows: draft→publish, exports, scheduling, triage | +| [references/gotchas-field-guide.md](references/gotchas-field-guide.md) | Symptom-first incident lookup for auth, editing, volume problems | + +## Scripts and prerequisites + +- `scripts/ghost` — executable Python CLI (stdlib + requests only). Flags above; lazy auth; structured logging. +- `scripts/test_ghost.py` — offline test suite (mocked HTTP, zero network). +- Python 3.8+, `requests`. Nothing listens, nothing installs; scope limited to one configured site via `GHOST_URL`. diff --git a/ghost/evals/evals.json b/ghost/evals/evals.json new file mode 100644 index 0000000..d65aa5e --- /dev/null +++ b/ghost/evals/evals.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "skill_name": "ghost", + "evals": [ + { + "id": "list-draft-posts", + "prompt": "Show me the draft posts on my Ghost blog so I can see what is waiting to be finished.", + "expected_output": "Use ghost posts --status draft, which routes to the Admin API because drafts are invisible to the Content API.", + "assertions": ["uses ghost posts command", "filters status draft", "mentions Admin API for drafts"] + }, + { + "id": "draft-then-publish-pipeline", + "prompt": "Create a post called Release notes on my Ghost site, then publish it once the content is reviewed.", + "expected_output": "Create with ghost create-post --title 'Release notes', then re-read it and run ghost update-post --updated-at --status published.", + "assertions": ["creates as draft first", "update sends latest updated_at", "publishes via status published"] + }, + { + "id": "jwt-auth-gotcha", + "prompt": "My Ghost Admin API script keeps failing with 401 INVALID_JWT even though my id:secret key looks right. What is going on?", + "expected_output": "Explain the Admin JWT contract: HS256 signature over the hex-decoded secret half, kid header set to the ID half, aud /admin/, exp at most five minutes after iat; check clock skew and that the secret is hex-decoded before signing. HS512-signed tokens are rejected with invalid algorithm.", + "assertions": ["explains HS256 with hex-decoded secret", "names kid header and aud", "states five minute token window"] + }, + { + "id": "content-vs-admin-keys", + "prompt": "I have a Ghost Content API key. Can I use it to list my unpublished posts?", + "expected_output": "No. The Content API serves published content only and ignores non-public filters, so drafts never appear no matter how valid the key is; switch to an Admin API key with ghost posts --status draft.", + "assertions": ["states content key cannot see drafts", "recommends admin api"] + }, + { + "id": "not-for-ghost-cli-install", + "prompt": "Run ghost install to set up a new Ghost production server with nginx and ssl on this machine.", + "expected_output": "Do not route this here: ghost install belongs to Ghost's official npm ghost-cli site-management tooling, not this Ghost Admin API content skill.", + "assertions": ["must not trigger ghost skill", "routes server installs to npm ghost-cli"] + }, + { + "id": "pagination-loop", + "prompt": "Export every post from my Ghost site as JSON, there are several hundred.", + "expected_output": "Loop with page and limit 100 following meta.pagination.next until next is null, since limit=all is gone in Ghost 6 and pages cap at 100.", + "assertions": ["uses pagination next loop", "caps limit at 100"] + } + ] +} diff --git a/ghost/references/admin-auth-and-basics.md b/ghost/references/admin-auth-and-basics.md new file mode 100644 index 0000000..db8c1a6 --- /dev/null +++ b/ghost/references/admin-auth-and-basics.md @@ -0,0 +1,115 @@ +# Ghost Admin API Authentication and Basics + +The Admin API is Ghost's management plane at `https://{admin_domain}/ghost/api/admin/`. It handles full CRUD on posts, pages, tags, and more, including drafts and scheduled content. Every request below assumes you have an Admin API key from **Ghost Admin → Settings → Integrations → Custom Integration**. + +## The Admin API key + +An Admin API key is a single string of two colon-separated halves: + +``` +{id}:{secret} +``` + +- `{id}` — a 24-character hexadecimal identifier (a Ghost ObjectID). +- `{secret}` — a 64-character hexadecimal string encoding 32 random bytes. + +Parse the key by splitting on the first `:`. Never assume total length; both halves are hex, but treat them as opaque strings until the moment you use them. Regenerating the key in Ghost Admin immediately invalidates every script holding the old one. Treat the whole key as a server-side secret: it signs tokens that can create, edit, and delete content. Use placeholders like `` in examples and CI; never paste real keys into code review tools. + +## JWT token contract, end to end + +Ghost does not accept the Admin API key directly. You exchange it for a short-lived JSON Web Token per request: + +1. Split the key on `:` into `id` and `secret`. +2. **Hex-decode the secret** into its 32 raw bytes. Signing with the literal hex characters produces an invalid signature; this is the single most common integration bug. +3. Build a JWT header with `alg: HS256`, `kid: `, `typ: JWT`. + +```json +{ + "alg": "HS256", + "kid": "", + "typ": "JWT" +} +``` + +4. Build a payload with integer-second timestamps and the audience claim: + +```json +{ + "iat": 1700000000, + "exp": 1700000300, + "aud": "/admin/" +} +``` + +5. Base64url-encode each segment without padding (`=` stripped), sign the `header.payload` string with HMAC-SHA256 keyed by the decoded bytes, append the base64url signature as the third dot-separated segment. +6. Send it as `Authorization: Ghost ` — the scheme is `Ghost`, not `Bearer`. +7. Include `Accept-Version: v6.0` and, for JSON writes, `Content-Type: application/json`. + +Python equivalent of the bundled script's signer: + +```python +import base64, hashlib, hmac, json, time + +def admin_token(key: str, request_path: str = "/ghost/api/admin/") -> str: + key_id, secret_hex = key.split(":", 1) + hmac_key = bytes.fromhex(secret_hex) # decode hex to raw bytes + now = int(time.time()) + def b64url(obj) -> str: + return base64.urlsafe_b64encode( + json.dumps(obj, separators=(",", ":")).encode()).rstrip(b"=").decode() + header = b64url({"alg": "HS256", "typ": "JWT", "kid": key_id}) + audience = "/admin/" # see audience rules below + payload = b64url({"iat": now, "exp": now + 300, "aud": audience}) + signing_input = f"{header}.{payload}".encode() + signature = hmac.new(hmac_key, signing_input, hashlib.sha256).digest() + return f"{header}.{payload}." + base64.urlsafe_b64encode(signature).rstrip(b"=").decode() +``` + +### Rules that decide whether a token works + +- **Algorithm must be HS256.** A token signed with HS512 is rejected outright (`Invalid token: invalid algorithm`). Do not "upgrade" the algorithm; Ghost's verifier allow-lists HS256 only. +- **aud (audience)** for current unversioned URLs (`/ghost/api/admin/...`) is exactly `/admin/`. Legacy versioned routes scope the audience to their URL version (`/v3/admin/`, `/v4/admin/`; v5 has no such form — Ghost 5 removed versioned URLs entirely). Sending `Accept-Version: v6.0` does not change the audience. +- **exp ≤ iat + 300.** Five minutes is the documented maximum token lifetime. The server additionally enforces a five-minute maximum age measured from `iat`, so a long-lived token fails even mid-window. Mint a fresh token for each request rather than caching them. +- **Timestamps are seconds**, not milliseconds. Millisecond values produce oversized `iat`/`exp` and fail validation. +- **NTP matters.** A skewed system clock shifts `iat` outside the acceptance window even though your code looks correct. + +## Error signatures for auth failures + +Ghost returns JSON errors shaped like `{"errors": [{"message", "context", "type", "code", ...}]}`. Distinct auth failure modes have distinct signatures worth memorizing: + +| Symptom | Status | Meaning | +| --- | --- | --- | +| `Invalid token: jwt expired` / `maxAge exceeded`, code `INVALID_JWT` | 401 | Token lifetime violated — mint fresher tokens | +| `Invalid token: invalid algorithm`, `INVALID_JWT` | 401 | Wrong alg (e.g. HS512); sign HS256 | +| `jwt audience invalid`, `INVALID_JWT` | 401 | Wrong aud; use `/admin/` for unversioned URLs | +| `Admin API kid missing.`, `MISSING_ADMIN_API_KID` | 400 | JWT header lacks `kid` | +| `Unknown Admin API Key`, `UNKNOWN_ADMIN_API_KEY` | 401 | kid does not match any integration; key regenerated? | +| `Authorization header format is "Authorization: Ghost [token]"`, `INVALID_AUTH_HEADER` | 401 | Used `Bearer` instead of the `Ghost` scheme | +| Malformed token JSON/base64, `INVALID_JWT` | 400 | Structurally undecodable token | +| No auth at all → `Authorization failed`, type `NoPermissionError` | 403 | Missing `Authorization` header entirely | + +The CLI maps each of these to exit code 2 with the server message plus a hint. + +## Request conventions + +```http +GET /ghost/api/admin/posts/?limit=15&page=1 HTTP/1.1 +Host: example.com +Authorization: Ghost +Accept-Version: v6.0 +Accept: application/json +``` + +- All resources ride in plural envelopes: `{"posts": [...], "meta": {...}}`. Writes must wrap payloads the same way: `{"posts": [{...}]}`. `/site/` and `/settings/` are the sole exceptions (single objects). +- Pagination defaults to `page=1&limit=15`; Ghost 6 caps page size at 100 and no longer honors `limit=all`. +- Filter syntax follows NQL: `filter=status:draft` uses URL-encoded `property:value`, comma is OR, parentheses group, `-` negates. +- `include=tags,authors` hydrates relations; `fields=title,slug,status` slims responses. + +## Sources + +- https://docs.ghost.org/admin-api +- https://docs.ghost.org/admin-api/#token-generation +- https://docs.ghost.org/admin-api/#accept-version-header +- https://docs.ghost.org/faq/api-versioning +- https://docs.ghost.org/content-api/pagination +- https://github.com/TryGhost/Ghost/blob/main/ghost/core/core/server/services/auth/api-key/admin.js diff --git a/ghost/references/content-vs-admin-api.md b/ghost/references/content-vs-admin-api.md new file mode 100644 index 0000000..fc6b1f4 --- /dev/null +++ b/ghost/references/content-vs-admin-api.md @@ -0,0 +1,53 @@ +# Content API vs Admin API: Which Plane, Which Key + +Ghost exposes two REST APIs with different credentials, scopes, and content visibility. Picking the wrong one produces the classic failure: everything looks configured, yet drafts are nowhere to be found and nothing errors. + +## The split at a glance + +| Aspect | Content API | Admin API | +| --- | --- | --- | +| Base path | `/ghost/api/content/` | `/ghost/api/admin/` | +| Credential | Content key as `?key=` query param | JWT in `Authorization: Ghost ` | +| Verbs | GET only (Browse, Read) | Full REST per resource | +| Scope | Published posts/pages/tags/authors/tiers/settings | Everything public **plus drafts, scheduled posts, members, webhooks, images, themes** | +| Key safety | Safe for browsers and clients (public data only) | Server-side only; signs mutations | +| Cacheability | Designed to be cached/CDN-fronted | Mutating; publish busts front-end caches | +| Typical consumers | Site themes, headless frontends, mobile apps | Editorial automation, migrations, scheduling bots | + +Both key types come from the same Custom Integration screen; an integration has a Content API key and an Admin API key side by side. They are not interchangeable. + +## Draft-visibility asymmetry (the trap) + +The Content API **delivers published content only**. Its docs state the key "only ever provide[s] access to public data," and Ghost enforces this at the model layer: public-context post queries carry a non-overridable `status:published` filter. + +Consequences worth internalizing: + +1. **Drafts are unreachable via Content API regardless of key validity.** A valid key does not make drafts visible; the request simply never matches them. +2. **It fails silent, not loud.** Browsing with a valid Content key returns HTTP 200 with only published posts — an empty or partial list, no error, no hint. There is no 403 saying "you can't see drafts." +3. **Filtering does not bypass it.** `filter=status:draft` against the Content API returns the same published collection; the disallowed filter is ignored rather than rejected. +4. **Direct reads of non-public posts 404.** Reading `/content/posts//` behaves as if the post does not exist — consistent with the documented 404 category "data which is not public." + +The bundled CLI is Admin-API-first precisely because of this asymmetry: `ghost posts --status draft` works only because it authenticates with the Admin JWT, which sees drafts, scheduled, and published posts alike. + +### Diagnostic checklist when "posts are missing" + +- Authenticated with the **Content** key? Switch to the Admin key workflow (`GHOST_ADMIN_KEY`); drafts will appear. +- Using Admin and still missing them? Check `filter=status:` values (`draft`, `scheduled`, `published`) and page through with `--page`. +- Post visible in Admin UI but 404s from your site code? That is the same asymmetry in reverse: unpublished content never appears on the public plane. + +## When to use which + +Use the **Content API** for anything that renders your site to the world: headless frontend builds, static-site generators, search indexes, feeds, mobile apps. It is read-only, key-as-query-param, and cache-friendly. + +Use the **Admin API** for anything that changes content or needs non-public data: creating and editing posts, publishing/scheduling, managing tags and pages, uploading images, working with drafts before they go live. Keep Admin keys out of browsers, client bundles, and CI logs. + +A frequent pattern pairs both: editorial automation writes through Admin; the public site reads through Content plus CDN. If a workflow only ever reads published content, prefer Content — smaller blast radius, browser-safe key. + +## Sources + +- https://docs.ghost.org/content-api +- https://docs.ghost.org/content-api/#key +- https://docs.ghost.org/admin-api +- https://docs.ghost.org/admin-api/#choosing-an-authentication-method +- https://docs.ghost.org/content-api/errors +- https://github.com/TryGhost/Ghost/blob/main/ghost/core/core/server/models/post.js diff --git a/ghost/references/gotchas-field-guide.md b/ghost/references/gotchas-field-guide.md new file mode 100644 index 0000000..9331fa7 --- /dev/null +++ b/ghost/references/gotchas-field-guide.md @@ -0,0 +1,62 @@ +# Ghost CMS Gotchas Field Guide + +Real-world failure modes, their symptoms, and their fixes. Each entry states the symptom first so this file can be scanned mid-incident. + +## Auth and keys + +**Symptom: 401 with `Invalid token: jwt expired` or `maxAge exceeded`.** +Your token outlived its five-minute window. Ghost both rejects `exp` more than 300 seconds past `iat` *and* independently caps token age at five minutes from `iat`, so caching tokens across a long batch job fails midway. Mint a fresh token per request — the bundled CLI does this automatically. + +**Symptom: 401 `Invalid token: invalid algorithm`.** +The token was signed HS512 (or another algorithm). Ghost's verifier allow-lists exactly `['HS256']`; "stronger" algorithms are rejected, not gracefully accepted. Sign HS256. + +**Symptom: signature looks right but still 401.** +You signed the HMAC key with the literal hex characters of the secret half instead of the raw bytes they encode. Hex-decode first (`bytes.fromhex(secret_hex)` in Python; `-macopt hexkey:$SECRET` in the official OpenSSL example). This is the most common hand-rolled signer bug. + +**Symptom: used `Authorization: Bearer ...` → 401 `INVALID_AUTH_HEADER`.** +Ghost's scheme is `Ghost`: `Authorization: Ghost `. + +**Symptom: worked for months, suddenly 401 `UNKNOWN_ADMIN_API_KEY`.** +Someone regenerated the integration key in Ghost Admin. Old scripts keep signing tokens under a kid the server no longer knows. Update every deployment holding the old key. + +**Symptom: fails only on one machine.** +Clock skew. Tokens are valid within tight iat/exp windows and NTP drift breaks them. Sync the clock. + +## Content visibility + +**Drafts invisible even though everything authenticates:** you are hitting the Content API (Content key, `/ghost/api/content/`). It serves published posts only, silently ignoring filters like `status:draft`, and it never errors about what it hides. Use the Admin API (Admin key + JWT) to reach drafts and scheduled posts. See [content-vs-admin-api.md](content-vs-admin-api.md). + +**Reading a known post id returns 404 from site code but renders in Admin:** same asymmetry from the other side — non-public content simply does not exist on the public plane. + +## Editing + +**409 `UpdateCollisionError` ("Saving failed! Someone else is editing this post."):** your PUT carried a stale `updated_at`. Every edit payload must include the version you actually read; re-GET immediately before PUT and pass its exact timestamp string. Concurrent editors and parallel automation make this likelier, not less. + +**Tags/authors vanished after an edit:** relation arrays REPLACE on update rather than merge. `PUT {"tags":["news"]}` deletes every other tag. Fetch-modify-send the complete array. + +**HTML came through mangled or stripped:** native post source is Lexical. Passing `html` requires the `?source=html` flag, and Ghost's HTML→Lexical conversion is lossy — inline styles and exotic tags get normalized. Preserve verbatim markup inside an HTML card (`…`) or send proper Lexical JSON. + +**Slug differed from what you sent:** slugs are uniquified (`my-post-2`) and sanitized (lowercase, hyphens) server-side. Read back `slug` from the create/edit response rather than assuming echo. + +## Pagination and volume + +**Only ever see ~15 (or at most 100) results:** default page size is 15; Ghost 6 caps limit at 100 and removed `limit=all` (oversized limits no longer error — they silently return ≤100 rows). Loop pages via `meta.pagination.next` until null. Treat totals as advisory; iterate by `next`. + +**Bulk script gets slow/rate-limited on big exports:** stagger requests between pages. There is no single documented universal rate limit; throttling is host-dependent and appears as 429 `TooManyRequestsError`. Back off exponentially and honor any Retry-After header seen in practice. + +## Versions + +**Requests work locally but docs examples conflict:** send `Accept-Version: v6.0` on every request. Versioning lives in headers since Ghost 5 removed versioned URLs; legacy URLs redirect internally and mark responses `Deprecation`. Breaking changes arrive only with major versions (~annually), and Ghost emails admins if a client sends unservable versions — another reason stale CI integrations suddenly "stop working" after upgrades. + +**Mobiledoc field errors after an upgrade:** Ghost 5+ replaced Mobiledoc with Lexical as the canonical content format. Integrations writing Mobiledoc must migrate to `lexical` (or use HTML cards). + +## Skill boundary reminder + +This skill drives the REST APIs. Server installation, nginx/SSL/systemd setup, `ghost start/stop/update/backup/doctor`, and theme-file editing belong to Ghost's separate npm `ghost-cli` ops tooling — different tool entirely. + +## Sources + +- https://docs.ghost.org/admin-api +- https://docs.ghost.org/content-api +- https://docs.ghost.org/changes +- https://docs.ghost.org/faq/api-versioning diff --git a/ghost/references/posts-pages-tags-endpoints.md b/ghost/references/posts-pages-tags-endpoints.md new file mode 100644 index 0000000..d4af400 --- /dev/null +++ b/ghost/references/posts-pages-tags-endpoints.md @@ -0,0 +1,121 @@ +# Admin Endpoint Guide: Posts, Pages, Tags, Pagination, Errors + +All paths relative to `https://{admin_domain}/ghost/api/admin/`. Authentication per [admin-auth-and-basics.md](admin-auth-and-basics.md). + +## Posts + +```text +GET /posts/ browse (filter, limit, page, order, include, fields, formats) +GET /posts/{id}/ read by id +GET /posts/slug/{slug}/ read by slug +POST /posts/ create +PUT /posts/{id}/ edit +DELETE /posts/{id}/ delete (204 No Content) +``` + +Post fields include `id`, `uuid`, `title`, `slug`, `html` (rendered), `lexical` (Ghost 5+ source format, replacing Mobiledoc), `status`, `visibility`, `created_at`, `updated_at`, `published_at`, plus tag/author relations and computed `url`/`excerpt`. Ghost 6 returns `lexical` by default; pass `formats=html,lexical` when you need rendered HTML alongside the source. + +### Creating posts + +Only `title` is required. Omitting `status` creates a draft — the safest default for automation. + +```bash +curl -sS -X POST "$BASE/posts/" \ + -H "Authorization: Ghost $TOKEN" \ + -H "Content-Type: application/json" \ + -H "Accept-Version: v6.0" \ + --data '{"posts":[{"title":"Release notes"}]}' +``` + +Add content either as a JSON-encoded Lexical document in `lexical`, or with `html` **plus the `?source=html` query flag**, which converts HTML to Lexical server-side. The conversion is lossy; wrap markup you must preserve verbatim in an HTML card (` ... `). Scheduled posts require `status: scheduled` and a future ISO 8601 `published_at`. + +### Editing posts: the collision guard + +PUT updates are partial, but every edit payload MUST carry the resource's current `updated_at`. Treat it as proof you read the latest version. The recommended sequence is GET immediately before PUT. + +```json +{"posts": [{"updated_at": "", "status": "published"}]} +``` + +Sending a stale `updated_at` fails with HTTP 409 `UpdateCollisionError` ("Saving failed! Someone else is editing this post."). A second editor (or another script) racing your automation is the usual trigger; re-GET, merge, retry. + +**Tag and author relations replace, never merge:** PUTting `tags:["news"]` removes all other tags. Fetch the post, modify the complete array, send it back whole. + +### Status lifecycle + +`draft` → editable, invisible to public plane → `scheduled` (requires future `published_at`; Ghost flips it to `published` automatically) → `published` → back to `draft` via PUT if needed. Email-only posts report `sent` after dispatch. + +## Pages + +Same verb set as posts at `/pages/`, plus `POST /pages/{id}/copy/` to duplicate. Pages are API-shape-identical to posts but render outside collection channels (about pages, landing pages). Creation is identical: only `title` required, omitting status drafts it. + +## Tags + +```text +GET /tags/ browse (add include=count.posts for usage counts) +POST /tags/ create +PUT /tags/{id}/ edit +DELETE /tags/{id}/ delete +``` + +Creating a tag that already exists by name/slug errors with a validation message rather than deduplicating; browse first when unsure. Hidden/internal tags carry `visibility: internal` and code-style slugs (`hash-...`). + +## Site info + +`GET /site/` is unauthenticated and returns a single object (no envelope): `{title, description, logo, url, version}`. Handy as a connectivity check before sending authenticated requests. + +## Pagination + +Browse endpoints default to `page=1&limit=15`. Ghost 6 caps page size at 100 — `limit=all` and `limit=9999` no longer error but silently return at most 100 rows. Consumers MUST loop: + +```python +page = 1 +while True: + doc = get_posts(page=page, limit=100) + yield from doc["posts"] + nxt = doc["meta"]["pagination"]["next"] + if nxt is None: + break + page = nxt +``` + +`meta.pagination` shape: `{"page": 1, "limit": 100, "pages": 7, "total": 624, "next": 2, "prev": null}`. Drive loops from `next` (null terminates), never from precomputed arithmetic on `total`, which can move mid-run under concurrent edits. Add a small delay between pages on large exports; hosts throttle aggressive crawlers even though Ghost itself documents no fixed rate limit. + +## Error envelope and status codes + +```json +{ + "errors": [{ + "message": "...", "context": null, + "type": "NotFoundError", + "details": null, "property": null, + "help": null, "code": null, + "id": "...", "ghostErrorCode": null + }] +} +``` + +| Status | type | When | +| --- | --- | --- | +| 400 | ValidationError / BadRequestError | Malformed query or payload, invalid field values | +| 401 | UnauthorizedError | Bad/expired/malformed JWT (see auth reference table) | +| 403 | NoPermissionError | Missing header, insufficient integration permissions | +| 404 | NotFoundError | Unknown id/slug, or non-public resource via Content API | +| 409 | UpdateCollisionError | Stale `updated_at` on PUT | +| 429 | TooManyRequestsError | Host throttling; back off, honor any Retry-After | +| 500 |ServerError | Ghost-side failure; safe to retry idempotent reads | + +Match errors on `errors[].code` where present (`UPDATE_COLLISION`, `INVALID_JWT`); messages change copy between releases less often than types, but codes are most stable of all. + +## Sources + +- https://docs.ghost.org/admin-api/posts/overview +- https://docs.ghost.org/admin-api/posts/creating-a-post +- https://docs.ghost.org/admin-api/posts/updating-a-post +- https://docs.ghost.org/admin-api/posts/publishing-a-post +- https://docs.ghost.org/admin-api/posts/scheduling-a-post +- https://docs.ghost.org/admin-api/pages/overview +- https://docs.ghost.org/admin-api/site/overview +- https://docs.ghost.org/content-api/pagination +- https://docs.ghost.org/content-api/errors +- https://docs.ghost.org/changes diff --git a/ghost/references/worked-recipes.md b/ghost/references/worked-recipes.md new file mode 100644 index 0000000..8755869 --- /dev/null +++ b/ghost/references/worked-recipes.md @@ -0,0 +1,132 @@ +# Worked Recipes: CLI and Raw API + +Recipes combining the bundled `scripts/ghost` CLI with raw Admin API calls. Every recipe is executable end-to-end with only `GHOST_URL` and `GHOST_ADMIN_KEY` set (and `jq` for JSON plumbing). HTML/Lexical examples use placeholder tokens, never real credentials. + +## Recipe 1: Draft a post now, publish after review + +Draft-first keeps half-written work off the public site while still letting you preview with admin themes. + +```bash +# 1. Create the draft +ghost --json create-post --title "Release notes" \ + --html "

What shipped this week…

" > /tmp/draft.json + +post_id=$(jq -r '.post.id' /tmp/draft.json) + +# 2. Later: re-read to fetch the CURRENT updated_at (collision guard) +ghost get-post "$post_id" | grep updated_at + +# 3. Publish, passing the fresh timestamp +ghost update-post "$post_id" \ + --status published \ + --updated-at "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" # NO — use the value from step 2 +``` + +The last line above is deliberately wrong: never synthesize `updated_at`. Copy the exact string from step 2's output; the server compares timestamps literally, and a mismatch means 409. + +Raw-API equivalent of step 3: + +```bash +curl -sS -X PUT "$BASE/posts/$POST_ID/" \ + -H "Authorization: Ghost $TOKEN" \ + -H "Content-Type: application/json" \ + -H "Accept-Version: v6.0" \ + --data "{\"posts\":[{\"updated_at\":\"$UPDATED_AT\",\"status\":\"published\"}]}" +``` + +## Recipe 2: Review queue — everything unpublished + +Drafts, scheduled, and published live on different filters; one call per status: + +```bash +ghost posts --status draft --limit 50 --json | jq -r '.posts[] | "\(.title)\t\(.slug)"' +ghost posts --status scheduled --json | jq -r '.posts[] | "\(.title)\t(.published_at // "-")"' +``` + +Pair it with an authoring-wide sanity check via jq types before feeding slugs onward: + +```bash +ghost posts --status draft --json | jq '{count: (.posts|length), all_slugs_strings: ([.posts[].slug | type] | all(. == "string"))}' +``` + +## Recipe 3: Full export, page by page + +Ghost 6 caps pages at 100 rows; `limit=all` is gone. + +```bash +page=1 +while :; do + ghost posts --limit 100 --page "$page" --json > "/tmp/posts-$page.json" + next=$(jq -r '.page.next // empty' "/tmp/posts-$page.json") + jq -r '.posts[].id' "/tmp/posts-$page.json" + [ -z "$next" ] && break + page=$next + sleep 0.2 +done +``` + +Note the loop reads `meta.pagination.next` from the CLI's `.page` field rather than computing `(page * limit) < total`; totals shift under concurrent edits. + +## Recipe 4: Publish at a future time (scheduling) + +```bash +ghost create-post --title "Launch day" \ + --html "

We're live.

" \ + --status scheduled \ + --published-at "2026-09-01T09:00:00.000Z" +``` + +Requirements: the timestamp must be in the future and ISO 8601; Ghost processes the queue on its own schedule (typically within five minutes of the mark). The post remains visible as `scheduled` in the Admin plane, invisible publicly until flip time. CLI enforces the pairing (`--status scheduled` without `--published-at` errors before any request is made). + +## Recipe 5: Slug-based handoff between systems + +External systems key content by slug. Resolve slug → id → full record: + +```bash +curl -sS "$BASE/posts/slug/$SLUG/" \ + -H "Authorization: Ghost $TOKEN" \ + -H "Accept-Version: v6.0" | jq -r '.posts[0].id' +``` + +Then read or edit by that id. The CLI reads by id (`ghost get-post `); for slug lookups use the curl form above. + +## Recipe 6: Tag hygiene pass + +Find tags nobody uses, then create missing ones for a new series: + +```bash +# Usage-counted listing +ghost tags --limit 200 --json | jq -r '.tags[] | select((.count.posts // 0) == 0) | .slug' + +# Create two series tags (idempotency: check existence first, creation duplicates error out) +ghost create-tag --name "Engineering" --slug engineering --description "Technical posts" +``` + +Remember relation semantics when tagging posts programmatically: PUT replaces the tag array wholesale, so send `[...existing_slugs, "engineering"]`, not just the addition. + +## Recipe 7: Connectivity triage + +When nothing works, descend this ladder: + +```bash +# 1. Is Ghost up? (no auth required) +curl -sS "$GHOST_URL/ghost/api/admin/site/" | jq . + +# 2. Does our JWT authenticate? +ghost site + +# 3. Can we browse? (exercises query params + permissions) +ghost posts --limit 1 +``` + +Step 1 failing = wrong URL/site down. Step 2 failing = auth contract problem (see the auth reference's signature table: expired vs invalid-algorithm vs audience). Step 3 failing while step 2 passes usually means integration permission gaps rather than token problems. + +## Sources + +- https://docs.ghost.org/admin-api/#token-generation-examples +- https://docs.ghost.org/admin-api/posts/creating-a-post +- https://docs.ghost.org/admin-api/posts/updating-a-post +- https://docs.ghost.org/admin-api/posts/publishing-a-post +- https://docs.ghost.org/admin-api/posts/scheduling-a-post +- https://docs.ghost.org/content-api/pagination +- https://docs.ghost.org/changes diff --git a/ghost/scripts/ghost b/ghost/scripts/ghost index 56a9068..d9057d6 100755 --- a/ghost/scripts/ghost +++ b/ghost/scripts/ghost @@ -3,6 +3,8 @@ Manage content on a Ghost CMS site: create and edit posts and pages, manage tags, and configure metadata. Requires GHOST_URL and GHOST_ADMIN_KEY. +Admin JWT authentication (HS256, kid header, 5-minute tokens) is built in +per https://docs.ghost.org/admin-api/. """ import argparse @@ -14,7 +16,7 @@ import os import sys import time import warnings -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict warnings.simplefilter("ignore") @@ -26,6 +28,28 @@ ENV_ADMIN_KEY = os.getenv("GHOST_ADMIN_KEY", "") QUIET = False GLOBAL_FLAGS: Dict[str, Any] = {"json": False, "dry_run": False, "quiet": False, "verbose": False} +# Ghost rejects JWTs whose exp is more than five minutes after iat, and its +# verifier additionally enforces a five-minute maxAge on iat itself. +TOKEN_TTL_SECONDS = 300 +VERSION = "v6.0" + + +def admin_api_audience(request_path: str = "/ghost/api/admin/") -> str: + """Derive the JWT audience claim from an Admin API request path. + + Current unversioned routes (/ghost/api/admin/...) use audience "/admin/". + Legacy versioned routes (/ghost/api/v3/admin/..., accepted up to v4) + scope the audience to the URL version, e.g. "/v3/admin/". + """ + marker = "/ghost/api" + idx = request_path.find(marker) + if idx == -1: + return "/admin/" + segments = [s for s in request_path[idx + len(marker):].split("/") if s] + if segments and len(segments[0]) > 1 and segments[0][0] == "v" and segments[0][1:].isdigit(): + return f"/{segments[0]}/admin/" + return "/admin/" + def log(msg): if not QUIET and not GLOBAL_FLAGS.get("json", False): @@ -41,6 +65,54 @@ def die(msg, exit_code=1): sys.exit(exit_code) +def _error_detail(resp): + """Extract Ghost's {errors: [...]} envelope text when present.""" + try: + body = resp.json() + except Exception: + return resp.text[:200] + if isinstance(body, dict) and isinstance(body.get("errors"), list): + parts = [] + for err in body["errors"][:3]: + code = err.get("code") or err.get("ghostErrorCode") or "" + message = err.get("message", "") + parts.append(f"{message} ({code})" if code else message) + return "; ".join(parts) + return json.dumps(body)[:200] + + +def _handle_response(resp): + """Map researched Ghost error signatures to actionable CLI messages.""" + status = resp.status_code + if status == 204: + # DELETE succeeds with 204 No Content — an empty body is not an error. + return {} + if status < 400: + try: + return resp.json() + except Exception: + die(f"API returned non-JSON response ({status}) from {resp.url}") + if status == 401: + hint = ("A structurally invalid or expired JWT usually returns 401 INVALID_JWT; " + "re-check GHOST_ADMIN_KEY and system clock (NTP skew breaks the 5-minute token window).") + if "INVALID_AUTH_HEADER" in resp.text: + hint = 'Authorization header must be "Authorization: Ghost [token]", not Bearer.' + die(f"Auth failed ({status}): {_error_detail(resp)} {hint}", 2) + if status == 403: + detail = _error_detail(resp) + if "NoPermissionError" in str(detail): + detail += " — requests without a valid Admin JWT receive 403 Authorization failed." + die(f"Forbidden ({status}): {detail} Check GHOST_ADMIN_KEY and integration permissions.", 2) + if status == 404: + die(f"Not found ({status}): {_error_detail(resp)} Admin draft reads need the Admin API — " + f"the Content API 404s non-public posts.", 3) + if status == 409: + die(f"Conflict ({status}): {_error_detail(resp)} Re-GET the post and send its latest updated_at.", 4) + if status == 429: + die(f"Rate limited ({status}): back off and retry; stagger paginated requests.", 5) + die(f"API error ({status}): {_error_detail(resp)}") + + def emit(human, data): if GLOBAL_FLAGS.get("json", False): print(json.dumps(data, default=str)) @@ -69,7 +141,7 @@ def _preparse_global_flags(argv): class GhostClient: - """Ghost Admin API client (v5/v6).""" + """Ghost Admin API client (v5/v6, Accept-Version v6.0).""" def __init__(self, url="", key="", dry_run=False): self.url = (url or ENV_URL).rstrip("/") @@ -77,100 +149,111 @@ class GhostClient: self.dry_run = dry_run def _jwt_token(self): - """Generate a short-lived JWT from the Admin API key (id:secret format).""" + """Generate a short-lived JWT from the Admin API key (id:secret format). + + Follows https://docs.ghost.org/admin-api/#token-generation: + HS256 over base64url segments, kid = key ID half, + aud = admin route audience, exp at most five minutes after iat. + The secret half is hex-decoded to raw bytes before signing; signing + the literal hex characters produces an INVALID signature. + """ if not self.key or ":" not in self.key: die("GHOST_ADMIN_KEY must be in 'id:secret' format. Get it from Ghost Admin → Integrations.") - key_id, secret = self.key.split(":", 1) + key_id, secret_hex = self.key.split(":", 1) + try: + hmac_key = bytes.fromhex(secret_hex) + except ValueError: + die("GHOST_ADMIN_KEY secret half must be hexadecimal. Copy the key verbatim from Ghost Admin → Integrations.") now = int(time.time()) - header = base64.urlsafe_b64encode(json.dumps({"alg": "HS256", "kid": key_id, "typ": "JWT"}).encode()).rstrip(b"=").decode() - payload = base64.urlsafe_b64encode(json.dumps({"iat": now, "exp": now + 300, "aud": "/admin/"}).encode()).rstrip(b"=").decode() - sig = hmac.new(secret.encode(), f"{header}.{payload}".encode(), hashlib.sha256).digest() - sig_b64 = base64.urlsafe_b64encode(sig).rstrip(b"=").decode() - return f"{header}.{payload}.{sig_b64}" + header = { + "alg": "HS256", + "typ": "JWT", + "kid": key_id, + } + payload = { + "iat": now, + "exp": now + TOKEN_TTL_SECONDS, + "aud": admin_api_audience(), + } - def _get(self, path, params=None): + def b64url(obj): + raw = json.dumps(obj, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + header_b64 = b64url(header) + payload_b64 = b64url(payload) + signing_input = f"{header_b64}.{payload_b64}".encode() + signature = hmac.new(hmac_key, signing_input, hashlib.sha256).digest() + sig_b64 = base64.urlsafe_b64encode(signature).rstrip(b"=").decode() + return f"{header_b64}.{payload_b64}.{sig_b64}" + + def _admin_headers(self): + return { + "Authorization": f"Ghost {self._jwt_token()}", + "Accept-Version": VERSION, + "Accept": "application/json", + } + + def _request(self, method, path, params=None, json_data=None): url = f"{self.url}/ghost/api/admin{path}" if self.dry_run: - return {"dry_run": True, "url": url, "params": params} + return {"dry_run": True, "method": method, "url": url, + "params": params, "json": json_data} + request_fn = getattr(requests, method) try: - resp = requests.get(url, params=params, - headers={"Authorization": f"Ghost {self._jwt_token()}", - "Accept-Version": "v6.0", "Accept": "application/json"}, - timeout=30) + resp = request_fn(url, params=params, json=json_data, + headers=self._admin_headers(), timeout=30) except requests.ConnectionError as e: die(f"Cannot connect to {self.url}: {e}") - if resp.status_code in (401, 403): - die(f"Auth failed ({resp.status_code}). Check GHOST_ADMIN_KEY.") - if resp.status_code >= 400: - try: - detail = resp.json() - except Exception: - detail = resp.text[:200] - die(f"API error ({resp.status_code}): {detail}") - return resp.json() + return _handle_response(resp) + + def _get(self, path, params=None): + return self._request("get", path, params=params) def _post(self, path, json_data): - url = f"{self.url}/ghost/api/admin{path}" - if self.dry_run: - return {"dry_run": True, "url": url, "json": json_data} - try: - resp = requests.post(url, json=json_data, - headers={"Authorization": f"Ghost {self._jwt_token()}", - "Accept-Version": "v6.0", - "Content-Type": "application/json"}, - timeout=30) - except requests.ConnectionError as e: - die(f"Cannot connect: {e}") - if resp.status_code >= 400: - try: - detail = resp.json() - except Exception: - detail = resp.text[:200] - die(f"API error ({resp.status_code}): {detail}") - return resp.json() + return self._request("post", path, json_data=json_data) def _put(self, path, json_data): - url = f"{self.url}/ghost/api/admin{path}" - if self.dry_run: - return {"dry_run": True, "url": url, "json": json_data} - try: - resp = requests.put(url, json=json_data, - headers={"Authorization": f"Ghost {self._jwt_token()}", - "Accept-Version": "v6.0", - "Content-Type": "application/json"}, - timeout=30) - except requests.ConnectionError as e: - die(f"Cannot connect: {e}") - if resp.status_code >= 400: - try: - detail = resp.json() - except Exception: - detail = resp.text[:200] - die(f"API error ({resp.status_code}): {detail}") - return resp.json() + return self._request("put", path, json_data=json_data) - def get_posts(self, limit=20, status=None): + def _delete(self, path): + return self._request("delete", path) + + def get_posts(self, limit=20, status=None, page=None, order=None): params: Dict[str, Any] = {"limit": limit} if status: params["filter"] = f"status:{status}" + if page is not None: + params["page"] = page + if order: + params["order"] = order return self._get("/posts", params) def get_post(self, post_id): - return self._get(f"/posts/{post_id}") + # Admin post reads default to Lexical source; request rendered HTML too. + return self._get(f"/posts/{post_id}", {"formats": "html,lexical"}) - def create_post(self, title, html="", status="draft", slug=""): - post = {"title": title, "status": status} + def create_post(self, title, html="", status="draft", slug="", published_at=None): + post: Dict[str, Any] = {"title": title, "status": status} if html: post["html"] = html if slug: post["slug"] = slug + if published_at: + post["published_at"] = published_at return self._post("/posts", {"posts": [post]}) def update_post(self, post_id, **kwargs): return self._put(f"/posts/{post_id}", {"posts": [kwargs]}) - def get_pages(self, limit=20): - return self._get("/pages", {"limit": limit}) + def delete_post(self, post_id): + return self._delete(f"/posts/{post_id}") + + def get_pages(self, limit=20, page=None): + params: Dict[str, Any] = {"limit": limit} + if page is not None: + params["page"] = page + return self._get("/pages", params) def create_page(self, title, html="", status="draft", slug=""): page = {"title": title, "status": status} @@ -180,8 +263,11 @@ class GhostClient: page["slug"] = slug return self._post("/pages", {"pages": [page]}) - def get_tags(self, limit=50): - return self._get("/tags", {"limit": limit, "include": "count.posts"}) + def get_tags(self, limit=50, page=None): + params: Dict[str, Any] = {"limit": limit, "include": "count.posts"} + if page is not None: + params["page"] = page + return self._get("/tags", params) def create_tag(self, name, slug="", description=""): tag = {"name": name} @@ -203,38 +289,61 @@ def fmt_post(p): return f" {title:45} [{status:7}] /{slug} updated {updated}" +def _plan_payload(data): + """Shape a dry-run plan from a client-level plan dict.""" + return {"dry_run": True, "method": data.get("method"), + "url": data.get("url"), "params": data.get("params"), + "json": data.get("json")} + + def cmd_posts(client, args): p = argparse.ArgumentParser(prog="ghost posts") p.add_argument("--limit", type=int, default=20) p.add_argument("--status", choices=["published", "draft", "scheduled"]) + p.add_argument("--page", type=int) + p.add_argument("--order", default="") parsed, _ = p.parse_known_args(args) - if client.dry_run: - return emit("[dry-run] List posts", {"dry_run": True}) - - data = client.get_posts(limit=parsed.limit, status=parsed.status) or {} + data = client.get_posts(limit=parsed.limit, status=parsed.status, + page=parsed.page, order=parsed.order or None) or {} + if data.get("dry_run"): + return emit("[dry-run] List posts", _plan_payload(data)) posts = data.get("posts", []) if not posts: - return emit("No posts found.", {"posts": []}) + return emit("No posts found.", {"posts": [], "total": 0}) lines = [fmt_post(p) for p in posts] - total = data.get("meta", {}).get("pagination", {}).get("total", len(posts)) - emit(f"{total} post(s):\n" + "\n".join(lines), - {"total": total, "posts": posts}) + pagination = data.get("meta", {}).get("pagination", {}) or {} + total = pagination.get("total", len(posts)) + header = f"{total} post(s):" + if pagination.get("pages") and pagination["pages"] > 1: + header += f" page {pagination.get('page')} of {pagination['pages']} — use --page to browse" + emit(header + "\n" + "\n".join(lines), + {"total": total, "page": pagination, "posts": posts}) def cmd_create_post(client, args): - p = argparse.ArgumentParser(prog="ghost posts create") + p = argparse.ArgumentParser(prog="ghost create-post") p.add_argument("--title", required=True) p.add_argument("--html", default="") p.add_argument("--status", default="draft", choices=["draft", "published", "scheduled"]) p.add_argument("--slug", default="") + p.add_argument("--published-at", dest="published_at", default="", + help="ISO 8601 timestamp; required when --status scheduled") parsed, _ = p.parse_known_args(args) - if client.dry_run: - return emit(f"[dry-run] Create post: {parsed.title}", {"dry_run": True, **vars(parsed)}) + if parsed.status == "scheduled" and not parsed.published_at: + die("--status scheduled requires --published-at with a future ISO 8601 timestamp.") - data = client.create_post(parsed.title, html=parsed.html, status=parsed.status, slug=parsed.slug) or {} + data = client.create_post(parsed.title, html=parsed.html, status=parsed.status, + slug=parsed.slug, published_at=parsed.published_at) or {} + if data.get("dry_run"): + # Plan and real request share one code path, so the previewed URL, + # method, and JSON envelope are exactly what execution would send. + plan = {"dry_run": True, "method": data.get("method"), + "url": data.get("url"), "json": data.get("json"), **vars(parsed)} + return emit(f"[dry-run] Create post '{parsed.title}' " + f"-> {data.get('method', 'POST').upper()} {data.get('url')}", plan) posts = data.get("posts", []) if posts: post = posts[0] @@ -244,15 +353,121 @@ def cmd_create_post(client, args): emit("Post created but no data returned.", {"status": "created"}) +def cmd_get_post(client, args): + p = argparse.ArgumentParser(prog="ghost get-post") + p.add_argument("id_or_slug_hint", help="Post ID (recommended). Slug lookup: use the API /posts/slug/{slug}/ route.") + parsed, _ = p.parse_known_args(args) + + data = client.get_post(parsed.id_or_slug_hint) or {} + if data.get("dry_run"): + return emit("[dry-run] Get post detail", _plan_payload(data)) + posts = data.get("posts", []) + if not posts: + return emit("Post not found.", {"post": None}) + post = posts[0] + lines = [f"{post.get('title', '?')} [{post.get('status', '?')}] /{post.get('slug', '')}", + f" id: {post.get('id', '?')}", + f" updated_at: {post.get('updated_at', '?')}", + f" url: {post.get('url', '?')}"] + emit("\n".join(lines), {"post": post}) + + +def cmd_update_post(client, args): + p = argparse.ArgumentParser(prog="ghost update-post") + p.add_argument("post_id") + p.add_argument("--title") + p.add_argument("--html") + p.add_argument("--status", choices=["draft", "published", "scheduled"]) + p.add_argument("--published-at", dest="published_at", default="") + p.add_argument("--updated-at", required=True, + help="Latest updated_at of the post (collision guard; re-GET first)") + parsed, _ = p.parse_known_args(args) + + fields: Dict[str, Any] = {"updated_at": parsed.updated_at} + if parsed.title is not None: + fields["title"] = parsed.title + if parsed.html is not None: + fields["html"] = parsed.html + if parsed.status is not None: + fields["status"] = parsed.status + if parsed.published_at: + fields["published_at"] = parsed.published_at + + data = client.update_post(parsed.post_id, **fields) or {} + if data.get("dry_run"): + plan = _plan_payload(data) + plan["post_id"] = parsed.post_id + plan["fields"] = fields + return emit(f"[dry-run] Update post {parsed.post_id}", plan) + posts = data.get("posts", []) + if posts: + post = posts[0] + emit(f"✅ Updated: {post.get('title')} (/{post.get('slug')}) [{post.get('status')}] " + f"new updated_at {post.get('updated_at')}", + {"status": "updated", "post": post}) + else: + emit("Post updated but no data returned.", {"status": "updated"}) + + +def cmd_delete_post(client, args): + p = argparse.ArgumentParser(prog="ghost delete-post") + p.add_argument("post_id") + parsed, _ = p.parse_known_args(args) + + result = client.delete_post(parsed.post_id) + if isinstance(result, dict) and result.get("dry_run"): + return emit("[dry-run] Delete post", _plan_payload(result)) + emit(f"🗑 Deleted post {parsed.post_id}. Deletion is permanent; the Content API stops serving it immediately.", + {"status": "deleted", "id": parsed.post_id}) + + +def cmd_create_page(client, args): + p = argparse.ArgumentParser(prog="ghost create-page") + p.add_argument("--title", required=True) + p.add_argument("--html", default="") + p.add_argument("--status", default="draft", choices=["draft", "published", "scheduled"]) + p.add_argument("--slug", default="") + parsed, _ = p.parse_known_args(args) + + data = client.create_page(parsed.title, html=parsed.html, status=parsed.status, slug=parsed.slug) or {} + if data.get("dry_run"): + return emit(f"[dry-run] Create page '{parsed.title}'", _plan_payload(data)) + pages = data.get("pages", []) + if pages: + page = pages[0] + emit(f"✅ Created page: {page.get('title')} (/{page.get('slug')}) [{page.get('status')}]", + {"status": "created", "page": page}) + else: + emit("Page created but no data returned.", {"status": "created"}) + + +def cmd_create_tag(client, args): + p = argparse.ArgumentParser(prog="ghost create-tag") + p.add_argument("--name", required=True) + p.add_argument("--slug", default="") + p.add_argument("--description", default="") + parsed, _ = p.parse_known_args(args) + + data = client.create_tag(parsed.name, slug=parsed.slug, description=parsed.description) or {} + if data.get("dry_run"): + return emit(f"[dry-run] Create tag '{parsed.name}'", _plan_payload(data)) + tags = data.get("tags", []) + if tags: + tag = tags[0] + emit(f"✅ Created tag: {tag.get('name')} (/{tag.get('slug')})", {"status": "created", "tag": tag}) + else: + emit("Tag created but no data returned.", {"status": "created"}) + + def cmd_pages(client, args): p = argparse.ArgumentParser(prog="ghost pages") p.add_argument("--limit", type=int, default=20) + p.add_argument("--page", type=int) parsed, _ = p.parse_known_args(args) - if client.dry_run: - return emit("[dry-run] List pages", {"dry_run": True}) - - data = client.get_pages(limit=parsed.limit) or {} + data = client.get_pages(limit=parsed.limit, page=parsed.page) or {} + if data.get("dry_run"): + return emit("[dry-run] List pages", _plan_payload(data)) pages = data.get("pages", []) if not pages: return emit("No pages found.", {"pages": []}) @@ -264,12 +479,12 @@ def cmd_pages(client, args): def cmd_tags(client, args): p = argparse.ArgumentParser(prog="ghost tags") p.add_argument("--limit", type=int, default=50) + p.add_argument("--page", type=int) parsed, _ = p.parse_known_args(args) - if client.dry_run: - return emit("[dry-run] List tags", {"dry_run": True}) - - data = client.get_tags(limit=parsed.limit) or {} + data = client.get_tags(limit=parsed.limit, page=parsed.page) or {} + if data.get("dry_run"): + return emit("[dry-run] List tags", _plan_payload(data)) tags = data.get("tags", []) if not tags: return emit("No tags found.", {"tags": []}) @@ -284,9 +499,9 @@ def cmd_tags(client, args): def cmd_site(client, args): - if client.dry_run: - return emit("[dry-run] Get site info", {"dry_run": True}) data = client.get_site() or {} + if data.get("dry_run"): + return emit("[dry-run] Get site info", _plan_payload(data)) site = data.get("site", {}) title = site.get("title", "?") desc = site.get("description", "") @@ -310,15 +525,48 @@ def main(): pp = sub.add_parser("posts", help="List posts") pp.add_argument("--limit", type=int, default=20) pp.add_argument("--status", choices=["published", "draft", "scheduled"]) + pp.add_argument("--page", type=int) + pp.add_argument("--order", default="") + + gp = sub.add_parser("get-post", help="Show one post by ID") + gp.add_argument("id_or_slug_hint") cp = sub.add_parser("create-post", help="Create a post") cp.add_argument("--title", required=True) cp.add_argument("--html", default="") cp.add_argument("--status", default="draft", choices=["draft", "published", "scheduled"]) cp.add_argument("--slug", default="") + cp.add_argument("--published-at", dest="published_at", default="") - sub.add_parser("pages", help="List pages").add_argument("--limit", type=int, default=20) - sub.add_parser("tags", help="List tags").add_argument("--limit", type=int, default=50) + up = sub.add_parser("update-post", help="Update a post (send latest updated_at)") + up.add_argument("post_id") + up.add_argument("--title") + up.add_argument("--html") + up.add_argument("--status", choices=["draft", "published", "scheduled"]) + up.add_argument("--published-at", dest="published_at", default="") + up.add_argument("--updated-at", required=True) + + dp = sub.add_parser("delete-post", help="Delete a post permanently") + dp.add_argument("post_id") + + cpg = sub.add_parser("create-page", help="Create a page") + cpg.add_argument("--title", required=True) + cpg.add_argument("--html", default="") + cpg.add_argument("--status", default="draft", choices=["draft", "published", "scheduled"]) + cpg.add_argument("--slug", default="") + + ct = sub.add_parser("create-tag", help="Create a tag") + ct.add_argument("--name", required=True) + ct.add_argument("--slug", default="") + ct.add_argument("--description", default="") + + pages_p = sub.add_parser("pages", help="List pages") + pages_p.add_argument("--limit", type=int, default=20) + pages_p.add_argument("--page", type=int) + + tags_p = sub.add_parser("tags", help="List tags") + tags_p.add_argument("--limit", type=int, default=50) + tags_p.add_argument("--page", type=int) args = parser.parse_args(filtered_argv[1:]) if not args.command: @@ -328,8 +576,10 @@ def main(): client = GhostClient(dry_run=GLOBAL_FLAGS.get("dry_run", False)) cmd_map = { - "site": cmd_site, "posts": cmd_posts, "create-post": cmd_create_post, - "pages": cmd_pages, "tags": cmd_tags, + "site": cmd_site, "posts": cmd_posts, "get-post": cmd_get_post, + "create-post": cmd_create_post, "update-post": cmd_update_post, + "delete-post": cmd_delete_post, "create-page": cmd_create_page, + "create-tag": cmd_create_tag, "pages": cmd_pages, "tags": cmd_tags, } handler = cmd_map.get(args.command) if not handler: diff --git a/ghost/scripts/test_ghost.py b/ghost/scripts/test_ghost.py new file mode 100644 index 0000000..ef07814 --- /dev/null +++ b/ghost/scripts/test_ghost.py @@ -0,0 +1,446 @@ +"""Offline suite for ghost/scripts/ghost — zero network egress by construction. + +Covers four behavior classes (--help, argument errors, --dry-run, mocked-client +logic) plus JWT known-answer signing checks with fixed inputs, per-skill pipeline +consumability chains exercised end-to-end through subprocesses and jq (pipeline +stages feed each other's outputs verbatim; every stage's exit code is asserted), +and Ghost-specific error signatures (409 update collision, 404 draft read, +INVALID_AUTH_HEADER, 204 No Content deletes). + +All HTTP is short-circuited by --dry-run or replaced with unittest mocks bound at +the cli.requests call site. No sockets are opened. +""" + +import contextlib +import importlib.machinery +import importlib.util +import io +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +SCRIPT = Path(__file__).with_name("ghost") + +# Fixed Admin API key in the official {id}:{secret} shape: 24-hex ObjectID half, +# 64-hex 32-byte secret half. Both halves are SYNTHETIC placeholder patterns, +# not credentials; nothing here authenticates anywhere. +KID = "5f9d4b1c8e2a43d7b6c0a1e9" +SECRET_HEX = "00ff" * 16 +FIXED_KEY = f"{KID}:{SECRET_HEX}" + +# Known-answer fixture computed once with iat frozen at 1700000000: +# header/payload are compact-JSON base64url segments; SIG_B64 is HMAC-SHA256 +# over "
." keyed with bytes.fromhex(SECRET_HEX). +HEADER_B64 = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IjVmOWQ0YjFjOGUyYTQzZDdiNmMwYTFlOSJ9" +PAYLOAD_B64 = "eyJpYXQiOjE3MDAwMDAwMDAsImV4cCI6MTcwMDAwMDMwMCwiYXVkIjoiL2FkbWluLyJ9" +SIG_B64 = "nxiluHMEVbp05Gi4kDYp28CCyOrwWuirtSzZeuWoisg" +TOKEN_TTL = 300 + + +def load_cli(): + loader = importlib.machinery.SourceFileLoader("ghost_cli", str(SCRIPT)) + spec = importlib.util.spec_from_loader("ghost_cli", loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def clean_env(): + env = os.environ.copy() + env.pop("GHOST_URL", None) + env.pop("GHOST_ADMIN_KEY", None) + return env + + +class GhostCliTests(unittest.TestCase): + """Subprocess-level CLI surface: help, arg errors, dry-run plans.""" + + def run_cli(self, *args): + return subprocess.run([str(SCRIPT), *args], text=True, + capture_output=True, env=clean_env()) + + def test_help_lists_all_subcommands(self): + result = self.run_cli("--help") + self.assertEqual(result.returncode, 0) + for noun in ("site", "posts", "pages", "tags", "create-post", + "get-post", "update-post", "delete-post", + "create-page", "create-tag"): + self.assertIn(noun, result.stdout) + + def test_no_subcommand_prints_help_and_fails(self): + result = self.run_cli() + self.assertNotEqual(result.returncode, 0) + + def test_missing_title_is_argument_error(self): + result = self.run_cli("--json", "create-post") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--title", result.stderr) + + def test_invalid_status_choice_rejected_without_crash(self): + result = self.run_cli("--json", "create-post", "--title", "T", "--status", "archived") + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("Traceback", result.stderr) + + def test_update_post_requires_updated_at_flag(self): + result = self.run_cli("--json", "update-post", "abc123") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--updated-at", result.stderr) + + def test_dry_run_site_plan_is_valid_json_without_credentials(self): + result = self.run_cli("--dry-run", "--json", "site") + self.assertEqual(result.returncode, 0) + payload = json.loads(result.stdout) + self.assertTrue(payload["dry_run"]) + + def test_dry_run_human_output_requires_explicit_json_flag(self): + # Without --json the same command stays human-readable; the flag dict + # is per-invocation, so callers must pass --json explicitly each run. + result = self.run_cli("--dry-run", "site") + self.assertEqual(result.returncode, 0) + self.assertIn("[dry-run]", result.stdout) + with self.assertRaises(json.JSONDecodeError): + json.loads(result.stdout) + + def test_create_post_dry_run_plan_includes_url_and_envelope(self): + result = self.run_cli("--dry-run", "--json", "create-post", "--title", "Draft A") + self.assertEqual(result.returncode, 0) + payload = json.loads(result.stdout) + self.assertTrue(payload["dry_run"]) + self.assertIn("/ghost/api/admin/posts", payload["url"]) + self.assertEqual(payload["json"], {"posts": [{"title": "Draft A", "status": "draft"}]}) + + def test_update_post_dry_run_carries_collision_guard_field(self): + result = self.run_cli("--dry-run", "--json", "update-post", "abc123", + "--status", "published", + "--updated-at", "2026-08-26T12:00:00.000Z") + self.assertEqual(result.returncode, 0) + payload = json.loads(result.stdout) + fields = payload["fields"] + self.assertEqual(fields["updated_at"], "2026-08-26T12:00:00.000Z") + self.assertEqual(fields["status"], "published") + + def test_scheduled_post_requires_published_at_even_in_dry_run(self): + result = self.run_cli("--dry-run", "--json", "create-post", + "--title", "Later", "--status", "scheduled") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--published-at", result.stderr) + + +class JwtSigningTests(unittest.TestCase): + """Known-answer JWT checks against fixed inputs (no network, no real keys).""" + + def setUp(self): + self.cli = load_cli() + + def sign_with_fixed_clock(self, key=FIXED_KEY): + original_time = self.cli.time.time + self.cli.time.time = lambda: 1700000000 + try: + client = self.cli.GhostClient(url="https://example.com", key=key) + token = client._jwt_token() + finally: + self.cli.time.time = original_time + return token + + @staticmethod + def decode_segment(segment): + import base64 + padded = segment + "=" * (-len(segment) % 4) + return json.loads(base64.urlsafe_b64decode(padded)) + + def test_jwt_matches_known_answer_token_exactly(self): + token = self.sign_with_fixed_clock() + self.assertEqual(token, f"{HEADER_B64}.{PAYLOAD_B64}.{SIG_B64}") + + def test_header_uses_hs256_kid_and_typ(self): + header = self.decode_segment(self.sign_with_fixed_clock().split(".")[0]) + self.assertEqual(header["alg"], "HS256") + self.assertEqual(header["typ"], "JWT") + self.assertEqual(header["kid"], KID) + + def test_payload_audience_and_five_minute_expiry(self): + payload = self.decode_segment(self.sign_with_fixed_clock().split(".")[1]) + self.assertEqual(payload["aud"], "/admin/") + self.assertEqual(payload["exp"] - payload["iat"], TOKEN_TTL) + self.assertEqual(payload["iat"], 1700000000) + + def test_signature_keys_hex_decoded_secret_not_literal_chars(self): + import base64 as b64 + import hashlib as hl + import hmac as hm + header_b64, payload_b64, sig_b64 = self.sign_with_fixed_clock().split(".") + expected = b64.urlsafe_b64encode( + hm.new(bytes.fromhex(SECRET_HEX), f"{header_b64}.{payload_b64}".encode(), hl.sha256).digest() + ).rstrip(b"=").decode() + self.assertEqual(sig_b64, expected) + literal_hex_signature = b64.urlsafe_b64encode( + hm.new(SECRET_HEX.encode(), f"{header_b64}.{payload_b64}".encode(), hl.sha256).digest() + ).rstrip(b"=").decode() + self.assertNotEqual(sig_b64, literal_hex_signature) + + def test_malformed_secret_half_is_graceful_error_not_traceback(self): + client = self.cli.GhostClient(url="https://example.com", key="5f9d4b1c8e2a43d7b6c0a1e9:not-hex!") + stderr = io.StringIO() + with self.assertRaises(SystemExit): + with contextlib.redirect_stderr(stderr): + client._jwt_token() + self.assertIn("hexadecimal", stderr.getvalue()) + + def test_key_without_colon_names_required_format(self): + client = self.cli.GhostClient(url="https://example.com", key="justonepart") + stderr = io.StringIO() + with self.assertRaises(SystemExit): + with contextlib.redirect_stderr(stderr): + client._jwt_token() + self.assertIn("id:secret", stderr.getvalue()) + + +class AdminApiAudienceTests(unittest.TestCase): + cli = load_cli() + + def test_unversioned_admin_path_uses_root_admin_audience(self): + self.assertEqual(self.cli.admin_api_audience("/ghost/api/admin/posts/"), "/admin/") + self.assertEqual(self.cli.admin_api_audience("/ghost/api/admin/"), "/admin/") + self.assertEqual(self.cli.admin_api_audience("nonsense"), "/admin/") + + def test_legacy_versioned_paths_scope_the_audience(self): + self.assertEqual(self.cli.admin_api_audience("/ghost/api/v3/admin/posts/"), "/v3/admin/") + self.assertEqual(self.cli.admin_api_audience("/ghost/api/v4/admin/"), "/v4/admin/") + + +class PipelineChainTests(unittest.TestCase): + """Per-skill contract: documented multi-step pipelines must execute stage by + stage, with each stage consuming the previous stage's emitted output.""" + + @classmethod + def setUpClass(cls): + cls.tmpdir = tempfile.TemporaryDirectory(prefix="ghost-pipeline-") + + @classmethod + def tearDownClass(cls): + cls.tmpdir.cleanup() + + def run_cli(self, *args, creds=False): + env = clean_env() + if creds: + env["GHOST_URL"] = "https://example.com" + env["GHOST_ADMIN_KEY"] = FIXED_KEY + return subprocess.run([str(SCRIPT), *args], text=True, + capture_output=True, env=env, + cwd=self.tmpdir.name) + + def run_jq(self, *jq_args): + return subprocess.run(["jq", *jq_args], + text=True, capture_output=True, env=clean_env(), + cwd=self.tmpdir.name) + + def write_stage_file(self, name, document): + path = Path(self.tmpdir.name) / name + path.write_text(json.dumps(document)) + return path.name + + def test_draft_then_publish_then_delete_chain_consumability(self): + post_id = "624c2b3fc1a5b7e9d4a0f2aa" + + # Stage 1: mint the draft plan; jq extracts method + URL + title field. + r1 = self.run_cli("--dry-run", "--json", "create-post", "--title", "Chain Post") + self.assertEqual(r1.returncode, 0) + stage1 = self.write_stage_file("stage1.json", json.loads(r1.stdout)) + check = self.run_jq("-r", ".method | select(. == \"post\") // empty", stage1) + self.assertEqual(check.stdout.strip(), "post") + url = self.run_jq("-r", ".url", stage1).stdout.strip() + self.assertIn("/ghost/api/admin/posts", url) + + # Stage 2: publish plan consumes a hand-built id + updated_at guard; + # jq asserts the collision-guard field travels into the request body. + r2 = self.run_cli("--dry-run", "--json", "update-post", post_id, + "--status", "published", + "--updated-at", "2026-08-26T12:00:00.000Z") + self.assertEqual(r2.returncode, 0) + stage2 = self.write_stage_file("stage2.json", json.loads(r2.stdout)) + guarded_at = self.run_jq("-r", ".fields.updated_at", stage2).stdout.strip() + self.assertRegex(guarded_at, r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}") + body_type = self.run_jq("-r", '.fields.status | select(. == "published") // empty', stage2) + self.assertEqual(body_type.stdout.strip(), "published") + + # Stage 3: teardown plan for the same post id consumed from stage 2's + # positional argument (verbatim string, not re-typed). + r3 = self.run_cli("--dry-run", "--json", "delete-post", post_id) + self.assertEqual(r3.returncode, 0) + stage3 = self.write_stage_file("stage3.json", json.loads(r3.stdout)) + self.assertEqual(self.run_jq("-r", ".method", stage3).stdout.strip(), "delete") + + def test_list_to_get_post_chain_type_contract(self): + row = {"title": "Hello World", "slug": "hello-world", "id": "abc123"} + + # Stage 1: a listing document (as ghost posts --json would produce); + # jq extracts .posts[0].id and asserts its JSON type is string. + listing_name = self.write_stage_file( + "listing.json", + {"total": 1, "page": {}, "posts": [row]}) + extracted = self.run_jq("-r", ".posts[0].id", listing_name) + self.assertEqual(extracted.returncode, 0, extracted.stderr) + self.assertEqual(extracted.stdout.strip(), row["id"]) + type_check = self.run_jq("-r", ".posts[0].id | type", listing_name) + self.assertEqual(type_check.stdout.strip(), "string") + slug_check = self.run_jq("-r", ".posts[0].slug | type", listing_name) + self.assertEqual(slug_check.stdout.strip(), "string") + + # Stage 2: get-post plan consumes exactly that id string positionally. + r2 = self.run_cli("--dry-run", "--json", "get-post", extracted.stdout.strip(), creds=True) + self.assertEqual(r2.returncode, 0) + plan = json.loads(r2.stdout) + self.assertTrue(plan["dry_run"]) + self.assertTrue(str(row["id"]) in plan["url"], plan["url"]) + self.assertTrue(plan["url"].startswith("https://")) + + +class MockedClientTests(unittest.TestCase): + """In-process handler logic with requests mocked at the call site.""" + + def load_with_flags(self, json_mode=True): + cli = load_cli() + cli.GLOBAL_FLAGS = {"json": json_mode, "dry_run": False, + "quiet": False, "verbose": False} + return cli + + def test_cmd_posts_parses_envelope_and_pagination_totals(self): + cli = self.load_with_flags() + client = cli.GhostClient(url="https://example.com", key=FIXED_KEY) + client.get_posts = Mock(return_value={ + "posts": [ + {"title": "First", "slug": "first", "status": "draft"}, + {"title": "Second", "slug": "second", "status": "published"}, + ], + "meta": {"pagination": {"page": 2, "limit": 20, "pages": 7, + "total": 124, "next": 3, "prev": 1}}, + }) + captured = [] + + def capture_print(*args): + captured.append(args) + + with patch("builtins.print", capture_print): + cli.cmd_posts(client, ["--limit", "20"]) + client.get_posts.assert_called_once_with(limit=20, status=None, page=None, order=None) + self.assertEqual(len(captured), 1, captured) + emitted = captured[0][0] # emit() prints one argument in json mode + payload = json.loads(emitted) + self.assertEqual(payload["total"], 124) + self.assertEqual(len(payload["posts"]), 2) + self.assertEqual(payload["page"]["pages"], 7) + + def test_client_get_posts_builds_status_filter_param(self): + cli = self.load_with_flags() + client = cli.GhostClient(url="https://example.com", key=FIXED_KEY) + seen = {} + + def fake_get(path, params=None): + seen["path"] = path + seen["params"] = params + return {"posts": [], "meta": {"pagination": {}}} + + client._get = fake_get + data = client.get_posts(limit=50, status="draft") + self.assertEqual(data["posts"], []) + self.assertEqual(seen["path"], "/posts") + self.assertEqual(seen["params"]["filter"], "status:draft") + self.assertEqual(seen["params"]["limit"], 50) + + def test_delete_post_tolerates_204_empty_body(self): + cli = self.load_with_flags(json_mode=False) + ok_empty = Mock(status_code=204) + del ok_empty.json # a real 204 carries no JSON body at all + cli.requests.delete = Mock(return_value=ok_empty) + client = cli.GhostClient(url="https://example.com", key=FIXED_KEY) + captured = [] + + with patch("builtins.print", lambda *a, **k: captured.append(a)): + cli.cmd_delete_post(client, ["abc123"]) + + cli.requests.delete.assert_called_once() + called_url = cli.requests.delete.call_args.args[0] + self.assertEqual(called_url, "https://example.com/ghost/api/admin/posts/abc123") + auth_header = cli.requests.delete.call_args.kwargs["headers"]["Authorization"] + self.assertTrue(auth_header.startswith("Ghost eyJ")) + + def test_update_collision_error_message_advises_reget(self): + cli = self.load_with_flags() + collision = Mock(status_code=409, text="conflict") + collision.json = Mock(return_value={ + "errors": [{"message": "Saving failed! Someone else is editing this post.", + "type": "UpdateCollisionError", "code": "UPDATE_COLLISION"}], + }) + cli.requests.put = Mock(return_value=collision) + client = cli.GhostClient(url="https://example.com", key=FIXED_KEY) + stderr = io.StringIO() + with self.assertRaises(SystemExit): + with contextlib.redirect_stderr(stderr): + client.update_post("abc123", status="published", + updated_at="2026-01-01T00:00:00.000Z") + message = stderr.getvalue() + self.assertIn("409", message) + self.assertIn("Someone else is editing this post", message) + self.assertIn("Re-GET", message) + + def test_draft_read_on_content_api_style_404_routes_to_admin_guidance(self): + cli = self.load_with_flags() + missing = Mock(status_code=404, text="not found") + missing.json = Mock(return_value={ + "errors": [{"message": "Resource not found error.", + "type": "NotFoundError", "code": None}]}) + cli.requests.get = Mock(return_value=missing) + client = cli.GhostClient(url="https://example.com", key=FIXED_KEY) + stderr = io.StringIO() + with self.assertRaises(SystemExit): + with contextlib.redirect_stderr(stderr): + client._get("/posts/does-not-exist") + message = stderr.getvalue() + self.assertIn("404", message) + self.assertIn("Admin API", message) + + def test_auth_header_scheme_mistake_surfaces_ghost_scheme_hint(self): + cli = self.load_with_flags() + bad_scheme = Mock(status_code=401) + bad_scheme.text = ('{"errors":[{"message":"Authorization header format is ' + '"Authorization: Ghost [token]","context":null,' + '"type":"UnauthorizedError","code":"INVALID_AUTH_HEADER"}]}') + bad_scheme.json = Mock(return_value={"errors": [{ + "message": "Authorization header format is \"Authorization: Ghost [token]\"", + "type": "UnauthorizedError", "code": "INVALID_AUTH_HEADER"}]}) + cli.requests.get = Mock(return_value=bad_scheme) + client = cli.GhostClient(url="https://example.com", key=FIXED_KEY) + stderr = io.StringIO() + with self.assertRaises(SystemExit): + with contextlib.redirect_stderr(stderr): + client._get("/posts") + self.assertIn("Ghost [token]", stderr.getvalue()) + + def test_authorization_header_uses_ghost_scheme_and_version_headers(self): + cli = self.load_with_flags() + ok = Mock(status_code=200) + ok.json = Mock(return_value={"posts": []}) + cli.requests.get = Mock(return_value=ok) + client = cli.GhostClient(url="https://example.com", key=FIXED_KEY) + client._get("/posts") + headers = cli.requests.get.call_args.kwargs["headers"] + self.assertTrue(headers["Authorization"].startswith("Ghost ")) + self.assertEqual(headers["Accept-Version"], "v6.0") + + def test_request_paths_target_unversioned_admin_api(self): + cli = self.load_with_flags() + ok = Mock(status_code=200) + ok.json = Mock(return_value={}) + cli.requests.get = Mock(return_value=ok) + client = cli.GhostClient(url="https://example.com/", key=FIXED_KEY) + client._get("/site") + called_url = cli.requests.get.call_args.args[0] + self.assertEqual(called_url, "https://example.com/ghost/api/admin/site") + + +if __name__ == "__main__": + unittest.main() diff --git a/llms.txt b/llms.txt index 75ca5ad..2b7d405 100644 --- a/llms.txt +++ b/llms.txt @@ -55,7 +55,7 @@ - [forward-deployed-engineering](forward-deployed-engineering/SKILL.md): Guide embedded technical engagements from ambiguous stakeholder need through discovery, framing, hypothesis, build, evaluation, deployment, adoption, measurement, and generalization while preserving evidence, decision rights, and field learning. Use when one accountable technical lead must carry continuity across customer or stakeholder discovery, implementation, production fit, adoption, and measurable outcomes. Do not use for a bounded repository change, product investment governance, ongoing reliability or platform ownership, an isolated specialist task, or advisory work that ends before implementation and adoption. - [frontend-engineering](frontend-engineering/SKILL.md): Build and maintain web frontends — component architecture, state management, API integration, responsive layout, client-side performance, and frontend testing patterns. Framework agnostic, focused on web frontend implementation. Do not use for backend service implementation, data engineering, or platform infrastructure work. - [genius-life](genius-life/SKILL.md): Guide a person in cultivating creativity in their own work and life: open conversational sessions on creative blocks, habits, environment, motivation, and resilience, or structured development of a concrete project or fledgling idea through a five-phase practice. Do not use for therapy or clinical support, general life coaching, product or stakeholder discovery, or as a study guide for a book. -- [ghost](ghost/SKILL.md): Manage Ghost CMS content from the terminal — create and list posts, pages, and tags, and fetch site info via the Ghost Admin API (v5/v6). Use when the user asks about ghost, cms, blog, blogging, posts, pages, tags, publishing, or site configuration. +- [ghost](ghost/SKILL.md): Manage Ghost CMS content over the Admin API — browse posts, pages, and tags, draft and publish content, schedule posts, and inspect site info from the terminal. Do not use this skill for Ghost server installation or site administration (installing, nginx, SSL, systemd, updates); those belong to the official npm ghost-cli tooling. - [github-runner](github-runner/SKILL.md): Deploy, manage, and troubleshoot self-hosted GitHub Actions runners. Covers systemd service, Docker containers, Kubernetes (Actions Runner Controller), and the Scale Set Client. Use when setting up a CI runner, debugging registration failures, designing autoscaling, or hardening runner security. - [go-to-market](go-to-market/SKILL.md): Plan and execute go-to-market strategy — positioning and messaging frameworks (April Dunford's positioning, message hierarchy), customer acquisition strategy (paid, organic, PLG, SLG), brand architecture (brand house vs house of brands), growth modeling (CAC/LTV by channel, cohort analysis), market entry strategy (beachhead, land-and-expand), and competitive response (pricing wars, feature races, brand defense). Do not use for sales execution and pipeline management, product strategy, or visual brand identity design. - [grafana](grafana/SKILL.md): Operate, configure, provision, secure, and troubleshoot Grafana OSS, Enterprise, and Cloud, including dashboards, folders, data sources, annotations, alert rules, contact points, notification policies, silences, mute timings, service accounts, RBAC, plugins, APIs, and as-code workflows. Use for Grafana product work and Grafana-side integrations. Do not use for defining SLOs or paging policy, operating Prometheus/Loki/Tempo/InfluxDB backends, generic Docker/Kubernetes/Terraform/reverse-proxy work, plugin development, or authorized security assessments; use the corresponding specialist skill. From 5120238f2e6d0d92cc3fe9e13d2f4eca16dab6da Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sat, 29 Aug 2026 12:40:16 -0400 Subject: [PATCH 29/40] fix(raleigh): classify upstream WAF challenges (#423) Classify Cloudflare managed browser challenges as visible non-failing canary observations while preserving blocking behavior for ordinary authentication failures. --- raleigh/EVIDENCE-LEDGER.md | 33 +++++++++++++ raleigh/references/civic-content-reference.md | 3 ++ raleigh/scripts/canary.py | 46 ++++++++++++++++++- raleigh/tests/test_raleigh.py | 39 ++++++++++++++++ 4 files changed, 119 insertions(+), 2 deletions(-) diff --git a/raleigh/EVIDENCE-LEDGER.md b/raleigh/EVIDENCE-LEDGER.md index 240196c..9860b38 100644 --- a/raleigh/EVIDENCE-LEDGER.md +++ b/raleigh/EVIDENCE-LEDGER.md @@ -191,6 +191,39 @@ The following public service boundaries were exercised successfully on 2026-07-2 - No model-backed eval run, CI run, commit, push, pull request, deployment, release, or merge is claimed. - Roll back the aggregate adapter and CLI wiring if the official site removes JSON:API access or publication links cannot be validated without broadening the trust boundary. +## Current Issue: Upstream Browser Challenge + +### Intent and authority + +- Keep the scheduled canary truthful when Raleigh's Cloudflare edge blocks + non-browser clients, without bypassing the provider's challenge or hiding + genuine contract failures. +- Modify the local Raleigh skill only. No publish, deploy, merge, or provider + configuration authority was used. + +### Evidence and decision + +- Runs `33259202346`, `33211420732`, and `33113598834` each reported HTTP 403 + for both civic probes, while all other probes passed. +- Direct probes returned the Cloudflare `cf-mitigated: challenge` marker and + challenge HTML. The same endpoints returned valid content from a browser-like + local request, establishing an access-policy boundary rather than a Raleigh + schema failure. +- Classify this exact marker as `waf_challenge`, preserve source and target in + the report, and keep the observation non-failing. Other 403 responses remain + `auth_regression` and continue to fail the canary. + +### Verification target and follow-up + +- Deterministic tests cover the marker-specific classification and the + non-failing summary accounting. +- The scheduled workflow remains the delivery-boundary check. A later green + run proves the canary no longer treats this known upstream challenge as a + durable failure; it does not prove Raleigh machine access has been restored. +- Do not add retries, browser automation, or challenge bypasses. Reclassify only + when the provider removes the marker or a new upstream access contract is + verified. + ## Issue 157 Addendum: Restore Live RPD Queries ### Intent and authority diff --git a/raleigh/references/civic-content-reference.md b/raleigh/references/civic-content-reference.md index 49fc5fa..e39298a 100644 --- a/raleigh/references/civic-content-reference.md +++ b/raleigh/references/civic-content-reference.md @@ -40,6 +40,9 @@ on later `--new-only` runs. ## Notes +- Raleigh's site may present a Cloudflare browser challenge to non-browser + clients. The CLI does not attempt to bypass that challenge; the scheduled + canary records it as a visible, non-failing upstream access observation. - The CLI preserves canonical page URLs so users can inspect the source presentation. - Rendered HTML is treated as content, not executable markup. - Publication status is requested server-side with `filter[status]=1`; the CLI diff --git a/raleigh/scripts/canary.py b/raleigh/scripts/canary.py index 7b9d521..5a86802 100644 --- a/raleigh/scripts/canary.py +++ b/raleigh/scripts/canary.py @@ -54,11 +54,18 @@ FAILURE_CLASSES = ( "parser_failure", "empty_but_valid", "restricted_folder", + "waf_challenge", ) def _classify_exception(exc: Exception) -> str: if isinstance(exc, urllib.error.HTTPError): + if ( + exc.code == 403 + and exc.headers + and exc.headers.get("cf-mitigated", "").lower() == "challenge" + ): + return "waf_challenge" if exc.code in (401, 403): return "auth_regression" if exc.code >= 500: @@ -78,6 +85,13 @@ def _classify_exception(exc: Exception) -> str: return "parser_failure" +def _waf_observation(source: str, target: str, err: dict[str, Any] | None) -> dict[str, Any] | None: + """Keep a provider browser challenge visible without treating it as drift.""" + if err and err.get("failure_class") == "waf_challenge": + return {"source": source, "target": target, "status": "pass", **err} + return None + + def _is_transient(failure_class: str) -> bool: return failure_class == "transport_outage" @@ -90,9 +104,12 @@ def _probe_with_retry(fn, *args, **kwargs) -> tuple[Any, None] | tuple[None, dic return result, None except Exception as exc: fc = _classify_exception(exc) + error_text = str(exc) + if fc == "waf_challenge": + error_text = "Cloudflare managed browser challenge (cf-mitigated: challenge)" evidence = { "failure_class": fc, - "error": str(exc), + "error": error_text, "attempt": attempt, } if first_evidence is None: @@ -278,6 +295,9 @@ def probe_civic_jsonapi() -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] index, err = _probe_with_retry(core.json_request, civic.JSONAPI_ROOT) if err: + observation = _waf_observation("civic", "jsonapi-index", err) + if observation: + return [observation] results.append({"source": "civic", "target": "jsonapi-index", "status": "fail", **err}) return results @@ -297,6 +317,9 @@ def probe_civic_rss() -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] data, err = _probe_with_retry(core.raw_request, civic.RSS_FEED) if err: + observation = _waf_observation("civic", "rss-feed", err) + if observation: + return [observation] results.append({"source": "civic", "target": "rss-feed", "status": "fail", **err}) return results @@ -481,6 +504,7 @@ def run_canary() -> dict[str, Any]: transient_failures = 0 empty_valid = 0 restricted = 0 + waf_challenges = 0 for name, probe_fn in ALL_PROBES: try: @@ -505,6 +529,8 @@ def run_canary() -> dict[str, Any]: empty_valid += 1 elif r.get("failure_class") == "restricted_folder": restricted += 1 + elif r.get("failure_class") == "waf_challenge": + waf_challenges += 1 continue fc = r.get("failure_class", "unknown") if _is_transient(fc): @@ -524,6 +550,7 @@ def run_canary() -> dict[str, Any]: "transient_failures": transient_failures, "empty_but_valid": empty_valid, "restricted_folders": restricted, + "waf_challenges": waf_challenges, }, "results": all_results, } @@ -546,6 +573,7 @@ def write_github_summary(report: dict[str, Any]) -> None: lines.append(f"| Transient failures | {s['transient_failures']} |") lines.append(f"| Empty-but-valid | {s['empty_but_valid']} |") lines.append(f"| Restricted folders | {s.get('restricted_folders', 0)} |") + lines.append(f"| WAF challenges | {s.get('waf_challenges', 0)} |") lines.append("") restricted = [r for r in report["results"] if r.get("failure_class") == "restricted_folder"] @@ -558,6 +586,19 @@ def write_github_summary(report: dict[str, Any]) -> None: lines.append(f"| {r.get('source', '?')} | {r.get('target', '?')} | {r.get('error', '')[:120]} |") lines.append("") + challenges = [r for r in report["results"] if r.get("failure_class") == "waf_challenge"] + if challenges: + lines.append("### Upstream WAF challenges (blocked machine access; non-failing)") + lines.append("") + lines.append("| Source | Target | Evidence |") + lines.append("|--------|--------|----------|") + for r in challenges: + lines.append( + f"| {r.get('source', '?')} | {r.get('target', '?')} " + f"| {r.get('error', '')[:120]} |" + ) + lines.append("") + failures = [r for r in report["results"] if r.get("status") == "fail"] if failures: lines.append("### Failures") @@ -592,7 +633,8 @@ def main() -> int: f"{s['durable_failures']} durable failures, " f"{s['transient_failures']} transient failures, " f"{s['empty_but_valid']} empty-but-valid, " - f"{s.get('restricted_folders', 0)} restricted folders" + f"{s.get('restricted_folders', 0)} restricted folders, " + f"{s.get('waf_challenges', 0)} WAF challenges" ) print(f"Report written to {report_path}") diff --git a/raleigh/tests/test_raleigh.py b/raleigh/tests/test_raleigh.py index 6ab62c3..48d837b 100644 --- a/raleigh/tests/test_raleigh.py +++ b/raleigh/tests/test_raleigh.py @@ -2726,6 +2726,45 @@ class PoliceTests(unittest.TestCase): self.assertFalse(report["passed"]) self.assertEqual(report["summary"]["transient_failures"], 1) + def test_canary_classifies_cloudflare_challenge_as_visible_non_failing_observation(self): + headers = Message() + headers["Server"] = "cloudflare" + headers["cf-mitigated"] = "challenge" + error = urllib.error.HTTPError( + "https://raleighnc.gov/jsonapi", 403, "Forbidden", headers, None + ) + with patch("canary.core.json_request", side_effect=error): + results = canary_lib.probe_civic_jsonapi() + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["status"], "pass") + self.assertEqual(results[0]["failure_class"], "waf_challenge") + self.assertIn("Cloudflare", results[0]["error"]) + + def test_canary_keeps_plain_forbidden_as_auth_failure(self): + headers = Message() + error = urllib.error.HTTPError( + "https://raleighnc.gov/jsonapi", 403, "Forbidden", headers, None + ) + with patch("canary.core.json_request", side_effect=error): + results = canary_lib.probe_civic_jsonapi() + self.assertEqual(results[0]["status"], "fail") + self.assertEqual(results[0]["failure_class"], "auth_regression") + + def test_canary_summary_counts_waf_challenges_without_failing(self): + observations = [{ + "source": "civic", + "target": "jsonapi-index", + "status": "pass", + "failure_class": "waf_challenge", + "error": "Cloudflare managed challenge", + "attempt": 1, + }] + with patch.object(canary_lib, "ALL_PROBES", [("civic", lambda: observations)]): + report = canary_lib.run_canary() + self.assertTrue(report["passed"]) + self.assertEqual(report["summary"]["waf_challenges"], 1) + self.assertEqual(report["summary"]["durable_failures"], 0) + def test_canary_imagery_probe_reports_restricted_folders_as_non_failing(self): with patch("canary.imagery.list_services", return_value=( [{"name": "Orthos2025", "type": "ImageServer"}], From 22f1d52456cca73edd56fccc44a9ca4dfa8645fa Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sat, 29 Aug 2026 13:12:42 -0400 Subject: [PATCH 30/40] fix(raleigh): keep WAF outages blocking (#424) Keep Cloudflare WAF challenges distinct while preserving a blocking canary signal for unavailable civic adapters. --- raleigh/EVIDENCE-LEDGER.md | 10 ++-- raleigh/references/civic-content-reference.md | 2 +- raleigh/scripts/canary.py | 26 +++++----- raleigh/tests/test_raleigh.py | 52 ++++++++++++++++--- 4 files changed, 65 insertions(+), 25 deletions(-) diff --git a/raleigh/EVIDENCE-LEDGER.md b/raleigh/EVIDENCE-LEDGER.md index 9860b38..b02b93e 100644 --- a/raleigh/EVIDENCE-LEDGER.md +++ b/raleigh/EVIDENCE-LEDGER.md @@ -210,16 +210,16 @@ The following public service boundaries were exercised successfully on 2026-07-2 local request, establishing an access-policy boundary rather than a Raleigh schema failure. - Classify this exact marker as `waf_challenge`, preserve source and target in - the report, and keep the observation non-failing. Other 403 responses remain - `auth_regression` and continue to fail the canary. + the report, and keep it as a blocking availability failure. Other 403 + responses remain `auth_regression` and continue to fail the canary. ### Verification target and follow-up - Deterministic tests cover the marker-specific classification and the - non-failing summary accounting. + blocking summary accounting. - The scheduled workflow remains the delivery-boundary check. A later green - run proves the canary no longer treats this known upstream challenge as a - durable failure; it does not prove Raleigh machine access has been restored. + run requires Raleigh machine access to be restored; a challenged civic + endpoint remains a canary failure. - Do not add retries, browser automation, or challenge bypasses. Reclassify only when the provider removes the marker or a new upstream access contract is verified. diff --git a/raleigh/references/civic-content-reference.md b/raleigh/references/civic-content-reference.md index e39298a..917e315 100644 --- a/raleigh/references/civic-content-reference.md +++ b/raleigh/references/civic-content-reference.md @@ -42,7 +42,7 @@ on later `--new-only` runs. - Raleigh's site may present a Cloudflare browser challenge to non-browser clients. The CLI does not attempt to bypass that challenge; the scheduled - canary records it as a visible, non-failing upstream access observation. + canary records it as a visible upstream availability failure. - The CLI preserves canonical page URLs so users can inspect the source presentation. - Rendered HTML is treated as content, not executable markup. - Publication status is requested server-side with `filter[status]=1`; the CLI diff --git a/raleigh/scripts/canary.py b/raleigh/scripts/canary.py index 5a86802..8edf4a0 100644 --- a/raleigh/scripts/canary.py +++ b/raleigh/scripts/canary.py @@ -13,7 +13,7 @@ the report. Exit codes: 0 all probes passed (or only empty-but-valid / restricted observations) - 1 one or more durable contract failures detected + 1 one or more durable contract or availability failures detected 2 script-level error (bad arguments, import failure, etc.) """ @@ -85,10 +85,10 @@ def _classify_exception(exc: Exception) -> str: return "parser_failure" -def _waf_observation(source: str, target: str, err: dict[str, Any] | None) -> dict[str, Any] | None: - """Keep a provider browser challenge visible without treating it as drift.""" +def _waf_failure(source: str, target: str, err: dict[str, Any] | None) -> dict[str, Any] | None: + """Keep provider browser challenges visible as availability failures.""" if err and err.get("failure_class") == "waf_challenge": - return {"source": source, "target": target, "status": "pass", **err} + return {"source": source, "target": target, "status": "fail", **err} return None @@ -295,9 +295,9 @@ def probe_civic_jsonapi() -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] index, err = _probe_with_retry(core.json_request, civic.JSONAPI_ROOT) if err: - observation = _waf_observation("civic", "jsonapi-index", err) - if observation: - return [observation] + failure = _waf_failure("civic", "jsonapi-index", err) + if failure: + return [failure] results.append({"source": "civic", "target": "jsonapi-index", "status": "fail", **err}) return results @@ -317,9 +317,9 @@ def probe_civic_rss() -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] data, err = _probe_with_retry(core.raw_request, civic.RSS_FEED) if err: - observation = _waf_observation("civic", "rss-feed", err) - if observation: - return [observation] + failure = _waf_failure("civic", "rss-feed", err) + if failure: + return [failure] results.append({"source": "civic", "target": "rss-feed", "status": "fail", **err}) return results @@ -524,13 +524,13 @@ def run_canary() -> dict[str, Any]: for r in all_results: if r.get("target") == "summary": continue + if r.get("failure_class") == "waf_challenge": + waf_challenges += 1 if r.get("status") != "fail": if r.get("failure_class") == "empty_but_valid": empty_valid += 1 elif r.get("failure_class") == "restricted_folder": restricted += 1 - elif r.get("failure_class") == "waf_challenge": - waf_challenges += 1 continue fc = r.get("failure_class", "unknown") if _is_transient(fc): @@ -588,7 +588,7 @@ def write_github_summary(report: dict[str, Any]) -> None: challenges = [r for r in report["results"] if r.get("failure_class") == "waf_challenge"] if challenges: - lines.append("### Upstream WAF challenges (blocked machine access; non-failing)") + lines.append("### Upstream WAF challenges (blocked machine access; failing)") lines.append("") lines.append("| Source | Target | Evidence |") lines.append("|--------|--------|----------|") diff --git a/raleigh/tests/test_raleigh.py b/raleigh/tests/test_raleigh.py index 48d837b..4ab8e6a 100644 --- a/raleigh/tests/test_raleigh.py +++ b/raleigh/tests/test_raleigh.py @@ -2726,7 +2726,7 @@ class PoliceTests(unittest.TestCase): self.assertFalse(report["passed"]) self.assertEqual(report["summary"]["transient_failures"], 1) - def test_canary_classifies_cloudflare_challenge_as_visible_non_failing_observation(self): + def test_canary_classifies_cloudflare_challenge_as_visible_availability_failure(self): headers = Message() headers["Server"] = "cloudflare" headers["cf-mitigated"] = "challenge" @@ -2736,10 +2736,50 @@ class PoliceTests(unittest.TestCase): with patch("canary.core.json_request", side_effect=error): results = canary_lib.probe_civic_jsonapi() self.assertEqual(len(results), 1) - self.assertEqual(results[0]["status"], "pass") + self.assertEqual(results[0]["status"], "fail") self.assertEqual(results[0]["failure_class"], "waf_challenge") self.assertIn("Cloudflare", results[0]["error"]) + def test_canary_classifies_rss_cloudflare_challenge_as_availability_failure(self): + headers = Message() + headers["cf-mitigated"] = "challenge" + error = urllib.error.HTTPError( + "https://raleighnc.gov/rss.xml", 403, "Forbidden", headers, None + ) + with patch("canary.core.raw_request", side_effect=error): + results = canary_lib.probe_civic_rss() + self.assertEqual(results[0]["status"], "fail") + self.assertEqual(results[0]["failure_class"], "waf_challenge") + + def test_canary_summary_renders_waf_challenges_as_failures(self): + report = { + "passed": False, + "summary": { + "total_results": 1, + "durable_failures": 1, + "transient_failures": 0, + "empty_but_valid": 0, + "restricted_folders": 0, + "waf_challenges": 1, + }, + "results": [{ + "source": "civic", + "target": "jsonapi-index", + "status": "fail", + "failure_class": "waf_challenge", + "error": "Cloudflare managed challenge", + }], + } + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as summary: + summary_path = summary.name + self.addCleanup(os.unlink, summary_path) + with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": summary_path}): + canary_lib.write_github_summary(report) + contents = pathlib.Path(summary_path).read_text() + self.assertIn("WAF challenges | 1", contents) + self.assertIn("blocked machine access; failing", contents) + self.assertIn("| civic | jsonapi-index | waf_challenge |", contents) + def test_canary_keeps_plain_forbidden_as_auth_failure(self): headers = Message() error = urllib.error.HTTPError( @@ -2750,20 +2790,20 @@ class PoliceTests(unittest.TestCase): self.assertEqual(results[0]["status"], "fail") self.assertEqual(results[0]["failure_class"], "auth_regression") - def test_canary_summary_counts_waf_challenges_without_failing(self): + def test_canary_summary_counts_waf_challenges_as_failures(self): observations = [{ "source": "civic", "target": "jsonapi-index", - "status": "pass", + "status": "fail", "failure_class": "waf_challenge", "error": "Cloudflare managed challenge", "attempt": 1, }] with patch.object(canary_lib, "ALL_PROBES", [("civic", lambda: observations)]): report = canary_lib.run_canary() - self.assertTrue(report["passed"]) + self.assertFalse(report["passed"]) self.assertEqual(report["summary"]["waf_challenges"], 1) - self.assertEqual(report["summary"]["durable_failures"], 0) + self.assertEqual(report["summary"]["durable_failures"], 1) def test_canary_imagery_probe_reports_restricted_folders_as_non_failing(self): with patch("canary.imagery.list_services", return_value=( From 2140d0d58d6545f84c9e9a932b9ec4a6ba511b3a Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sat, 29 Aug 2026 17:37:35 -0400 Subject: [PATCH 31/40] docs(jellyfin): thicken media-server skill against current API research Full lastfm-model rebuild of the jellyfin skill against the 12.0-era OpenAPI spec, core-dev authorization guidance, and server source: - Document the researched auth sequence end to end: complete pre-token Authorization: MediaBrowser Client/Device/DeviceId/Version header required by POST /Users/AuthenticateByName (400 "Error processing request." without it), AccessToken returned, then Token= on the same header (legacy X-Emby-Token deprecated, disableable since 10.11, targeted for removal at 12.0). - Extend scripts/jellyfin: new `login` subcommand demonstrating the pre-token header and printing session exports (password via stdin/prompt/env only), seasons/episodes TV navigation, next-up --series-id, browse --user-id (userId is required on non-API-key auth per the ItemsController guard), modern Token= header transport with X-Emby-Token fallback, 503 Retry-After handling, search Id/deprecated-ItemId fallback. - Add 5 cited reference files (auth/sessions, endpoint catalog, user-scoping matrix, gotchas field guide, worked recipes) plus quick-connect; all cite api.jellyfin.org and live-verified sources. - Upgrade relocated scripts/test_jellyfin_cli.py to the double-runner standard: 24 tests (was 8) covering help, argument errors, dry-run, mocked login header sequence, TV navigation, search-id fallback, and jq-executed pipeline-consumability chains; zero egress proven via proxy-trap rerun. - Add evals/evals.json (6 cases incl. emby-install-not-for-jellyfin negative probe); rewrite SKILL.md (224 lines) and README; sync root README blurb and skill-triggers row; regenerate marketplace.json and llms.txt (description-embedding artifacts). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- jellyfin/README.md | 59 ++- jellyfin/SKILL.md | 226 ++++++---- jellyfin/evals/evals.json | 68 +++ jellyfin/references/auth-and-sessions.md | 175 ++++++++ jellyfin/references/endpoint-catalog.md | 191 +++++++++ jellyfin/references/gotchas-field-guide.md | 130 ++++++ jellyfin/references/quick-connect.md | 60 +++ .../references/user-scoping-and-errors.md | 89 ++++ jellyfin/references/worked-recipes.md | 156 +++++++ jellyfin/scripts/jellyfin | 309 +++++++++++++- jellyfin/scripts/test_jellyfin_cli.py | 390 +++++++++++++++++- 11 files changed, 1736 insertions(+), 117 deletions(-) create mode 100644 jellyfin/evals/evals.json create mode 100644 jellyfin/references/auth-and-sessions.md create mode 100644 jellyfin/references/endpoint-catalog.md create mode 100644 jellyfin/references/gotchas-field-guide.md create mode 100644 jellyfin/references/quick-connect.md create mode 100644 jellyfin/references/user-scoping-and-errors.md create mode 100644 jellyfin/references/worked-recipes.md diff --git a/jellyfin/README.md b/jellyfin/README.md index 36ff5ee..b66f963 100644 --- a/jellyfin/README.md +++ b/jellyfin/README.md @@ -1,44 +1,69 @@ # Jellyfin Media Server from the Terminal -Query your Jellyfin media library — recently added movies and episodes, search and inspect items, browse library contents, see next-up episodes, and check server stats. +Query your Jellyfin media library — recently added movies and episodes, search and inspect +items, walk series, seasons, and episodes, browse library contents, see next-up episodes, +log in as a user, and check server stats. ## Why Install This Skill -When your agent loads this skill, it can **navigate your home media server** without opening a browser. That means: +When your agent loads this skill, it can **navigate your home media server** without +opening a browser. That means: -- **See what's new** — recently added movies and TV episodes +- **See what's new** — recently added movies and TV episodes, filtered server-side - **Search your library** — find any movie, show, or episode by keyword -- **Navigate your library** — inspect search results, browse collections, and page through items -- **See what is next** — find the next unwatched episodes for a user -- **Check server details** — server name, version, operating system, user count +- **Navigate series** — walk a show's seasons and episodes, and see what's next unwatched +- **Browse collections** — list your libraries and page through everything in them +- **Authenticate properly** — log in as a user (or use Quick Connect) without fumbling + Jellyfin's unusual `MediaBrowser` authorization header, which trips up most scripts +- **Check server details** — server name, version, operating system, user count, counts + +Every command is read-only (plus a `login` helper), and `--dry-run` previews any request +without touching the network. ## What You Get -| Directory | Purpose | -|-----------|---------| -| `SKILL.md` | Complete command reference with setup and examples | -| `scripts/jellyfin` | CLI tool for Jellyfin API operations | +| Path | Purpose | +|------|---------| +| `SKILL.md` | Complete command reference with setup, gotchas, and recipes | +| `scripts/jellyfin` | CLI for Jellyfin API operations (`--json`, `--dry-run`) | +| `scripts/test_jellyfin_cli.py` | Offline test suite (all HTTP mocked) | +| `references/auth-and-sessions.md` | The MediaBrowser header scheme, login flow, token channels, deprecation timeline | +| `references/endpoint-catalog.md` | Endpoint-by-endpoint parameter and response-shape catalog | +| `references/user-scoping-and-errors.md` | Which calls need a user id, and why queries 400/404 without one | +| `references/gotchas-field-guide.md` | Wire-level failure signatures and version differences | +| `references/worked-recipes.md` | Multi-step curl/jq and CLI workflows | +| `references/quick-connect.md` | Passwordless Quick Connect login | +| `evals/evals.json` | Behavioral eval cases including negative triggers | ## Quick Start ```bash scripts/jellyfin --help export JELLYFIN_URL="http://your-server:8096" -export JELLYFIN_API_KEY="your-api-key" -export JELLYFIN_USER_ID="your-jellyfin-user-id" # required by recent, next-up, and item +export JELLYFIN_API_KEY="your-api-key" # Dashboard → API Keys +export JELLYFIN_USER_ID="your-jellyfin-user-id" # required by user-scoped commands ``` -API key from Dashboard → API Keys in the Jellyfin admin panel. +```bash +scripts/jellyfin search --query "dune" --type Movie --json +scripts/jellyfin recent --movies --limit 5 +``` + +No API key yet? Log in as a user instead — the script sends the pre-token +`Authorization: MediaBrowser Client=..., Device=..., DeviceId=..., Version=...` header +that `POST /Users/AuthenticateByName` requires and prints the values to export: ```bash -scripts/jellyfin search --query "dune" --type Movie -scripts/jellyfin next-up --limit 5 +scripts/jellyfin login --username alice --prompt ``` ## Triggers -Load this when asking about Jellyfin, media server content, recently added movies or TV, or browsing your home media library. +Load this when asking about Jellyfin, media server content, recently added movies or TV, +next-up episodes, browsing your home media library, or Jellyfin API authentication. ## Requirements -Python 3.8+ with `requests` library. Jellyfin server with API key; `recent`, `next-up`, and `item` also require a Jellyfin user ID. +Python 3.8+ with `requests`. A running Jellyfin server (10.8+ behaviors assumed). +Authentication: an API key (Dashboard → API Keys), a user access token via `login`, or +Quick Connect. User-scoped commands also need a Jellyfin user id. diff --git a/jellyfin/SKILL.md b/jellyfin/SKILL.md index 4ab9bbc..188b6a3 100644 --- a/jellyfin/SKILL.md +++ b/jellyfin/SKILL.md @@ -1,138 +1,224 @@ --- name: jellyfin description: Query your Jellyfin media server from the terminal — recently added media, - search, item details, next-up episodes, library browsing, server info, and stats. Use - when the user asks about Jellyfin, media server, movies, TV shows, next episodes, or - their media library. + search, item details, series navigation, next-up episodes, library browsing, server + info, and user login. Use when the user asks about Jellyfin, media servers, movies, TV + shows, next episodes, or their media library. Do not use this skill for server + installation, library management, playback control, or Emby/Plex servers. license: MIT -compatibility: Requires JELLYFIN_URL (default http://localhost:8096) and JELLYFIN_API_KEY - env vars; `recent`, `next-up`, and `item` also require JELLYFIN_USER_ID or --user-id. - Python 3.8+ and the `requests` library. Generate an API key at Dashboard → API Keys in the Jellyfin admin panel. +compatibility: Requires Python 3.8+ and `requests`. Authenticate with JELLYFIN_API_KEY + (Dashboard → API Keys), a user access token from `login`, or Quick Connect; user-scoped + commands (`recent`, `next-up`, `item`, `seasons`, `episodes`) also need JELLYFIN_USER_ID + or --user-id. metadata: tags: jellyfin, media-server, movies, tv, episodes, recently-added, library, home-media, api-client - sources: https://jellyfin.org/docs/general/clients/api, https://jellyfin.org/downloads + sources: https://api.jellyfin.org/, https://gist.github.com/nielsvanvelzen/ea047d9028f676185832e51ffaf12a6f --- # jellyfin — Jellyfin Media Server from the Terminal -Query recently added movies and TV episodes, search and inspect media, browse libraries, see next-up episodes, check server info, and view library statistics — all from your Jellyfin server's REST API. +Query recently added movies and TV episodes, search and inspect media, walk series → +seasons → episodes, browse libraries, see next-up episodes, log in as a user, and check +server stats — all from your Jellyfin server's REST API. Every command is read-only +except `login`. ## Setup -1. Make sure your Jellyfin server is running and accessible. -2. Generate an API key in the Jellyfin Dashboard → **API Keys** → `+` to create a new key. +1. Make sure your Jellyfin server is running and accessible (default `http://localhost:8096`). +2. Pick an authentication route: + - **API key** — Dashboard → **API Keys** → `+`. Administrator-level, no user identity: + every user-scoped command then needs an explicit user id. + - **User token** — run `scripts/jellyfin login --username NAME --prompt` once; it + prints the values to export. 3. Set these environment variables: ```bash export JELLYFIN_URL="http://your-server:8096" # include protocol and port -export JELLYFIN_API_KEY="your-api-key-here" -export JELLYFIN_USER_ID="your-jellyfin-user-id" # required by recent, next-up, and item +export JELLYFIN_API_KEY="your-api-key-here" # or JELLYFIN_TOKEN after `login` +export JELLYFIN_USER_ID="your-jellyfin-user-id" # required by recent, next-up, item, seasons, episodes ``` -Run the bundled CLI as `scripts/jellyfin`. `--help` and `--dry-run` work without credentials. +Run the bundled CLI as `scripts/jellyfin`. `--help` and `--dry-run` work without +credentials. + +### How authentication works + +Jellyfin wants a `MediaBrowser`-scheme `Authorization` header on every call. The login +endpoint requires its `Client=..., Device=..., DeviceId=..., Version=...` quartet **before +any token exists** — the server rejects `POST /Users/AuthenticateByName` with +`400 Error processing request.` otherwise. Afterwards the access token (or API key) rides +the same header as `Token="..."`; the legacy `X-Emby-Token` header means the same thing +and is scheduled for removal from Jellyfin 12.0. The bundled CLI sends the modern form. +See [references/auth-and-sessions.md](references/auth-and-sessions.md). ## Essential Commands +### Authentication — get a session + +```bash +scripts/jellyfin login --username alice --prompt # prints JELLYFIN_* exports +echo "pw" | scripts/jellyfin login --username alice --password-stdin +scripts/jellyfin login --username alice --dry-run --json # preview the pre-token header +``` + +`login` demonstrates the full researched sequence: complete pre-token MediaBrowser header +→ `POST /Users/AuthenticateByName` → capture `User.Id` + `AccessToken` → print the +post-token header for reuse. It never echoes the password. + ### info — Server information ```bash -scripts/jellyfin info # server name, version, OS, user count -scripts/jellyfin info --json # machine-readable -scripts/jellyfin --dry-run info # preview API requests +scripts/jellyfin info # server name, version, OS, user count +scripts/jellyfin info --json ``` -Shows: server name, version, operating system, number of users. - ### recent — Recently added media ```bash -scripts/jellyfin recent # last 10 items added -scripts/jellyfin recent --limit 20 # more results -scripts/jellyfin recent --movies # only recently added movies -scripts/jellyfin recent --episodes # only recently added episodes -scripts/jellyfin recent --user-id USER_ID # override JELLYFIN_USER_ID -scripts/jellyfin recent --movies --limit 5 # top 5 recently added movies -scripts/jellyfin recent --json # machine-readable +scripts/jellyfin recent # last 10 items added for JELLYFIN_USER_ID +scripts/jellyfin recent --movies --limit 5 # server-side includeItemTypes filter +scripts/jellyfin recent --episodes --limit 20 --json +scripts/jellyfin recent --user-id USER_ID # override the env var ``` -Uses Jellyfin's current `/Items/Latest` endpoint. `--movies` and `--episodes` send `includeItemTypes` to the server, so the requested limit applies to the selected media type. Shows: name, type (Movie/Episode), production year, series name (for episodes), date added. +Hits `/Items/Latest` with `userId`; the response is a **bare JSON array** (no `Items` +wrapper), and `groupItems` merges episodes by series, so treat it as "what's new". ### search — Search your media library ```bash -scripts/jellyfin search --query "dune" # search everything -scripts/jellyfin search --query "dune" --type Movie # movies only -scripts/jellyfin search --query "star trek" --type Series,Episode -scripts/jellyfin search --query "inception" --limit 5 # top 5 results -scripts/jellyfin search --query "dune" --json # machine-readable +scripts/jellyfin search --query "dune" # everything +scripts/jellyfin search --query "dune" --type Movie # comma-separated types +scripts/jellyfin search --query "star trek" --type Series,Episode --limit 5 --json ``` -The `--type` flag accepts a comma-separated list of item types (e.g. `Movie,Series,Episode`). +Search hits `/Search/Hints`; results carry `id` (with a deprecated `ItemId` twin on old +servers — the CLI already prefers the modern field). -### Navigation — Inspect media and browse libraries +### Navigation — inspect items and walk series ```bash -scripts/jellyfin search --query "dune" --type Movie # find an item ID -scripts/jellyfin item --id ITEM_ID # inspect that item -scripts/jellyfin libraries # find a library ID -scripts/jellyfin browse --library-id LIBRARY_ID --type Movie --limit 20 -scripts/jellyfin browse --library-id LIBRARY_ID --start-index 20 -scripts/jellyfin next-up --limit 10 # next episodes for JELLYFIN_USER_ID -scripts/jellyfin next-up --user-id USER_ID --json +scripts/jellyfin search --query "dune" --type Movie --json # find an item ID +scripts/jellyfin item --id ITEM_ID # full metadata (needs user) +scripts/jellyfin seasons --series-id SERIES_ID # list seasons +scripts/jellyfin episodes --series-id SERIES_ID --season-id SEASON_ID +scripts/jellyfin next-up --limit 10 # next unwatched episodes +scripts/jellyfin next-up --series-id SERIES_ID --user-id USER_ID ``` -Use `search -> item` to look up a result's metadata, and `libraries -> browse` to page through a collection. `next-up` returns the next unwatched episodes for the selected user. `item` and `next-up` require `JELLYFIN_USER_ID` or `--user-id`; all three commands are read-only. +`item`, `seasons`, `episodes`, and `next-up` are user-scoped: they require +`JELLYFIN_USER_ID` or `--user-id` and fail before any network call without one. -### libraries — List media libraries +### libraries — browse a collection ```bash -scripts/jellyfin libraries # all configured libraries -scripts/jellyfin libraries --json # machine-readable +scripts/jellyfin libraries # library IDs and types +scripts/jellyfin browse --library-id LIBRARY_ID --type Movie --limit 50 +scripts/jellyfin browse --library-id LIBRARY_ID --start-index 50 # paginate +scripts/jellyfin browse --library-id LIBRARY_ID --user-id USER_ID # userId sent explicitly ``` -Shows: library name, collection type (movies, tvshows, music, etc.), library ID. +`libraries` reads `/Library/MediaFolders`, which is **admin-only** — non-admin tokens get +403 and should use `/UserViews` (see references). `browse` pages `/Items` with +`startIndex`/`limit` and passes `userId` when provided, since servers using non-API-key +auth reject unscoped queries with `400 userId is required`. ### stats — Library statistics ```bash -scripts/jellyfin stats # movie, series, episode, song counts -scripts/jellyfin stats --json # machine-readable +scripts/jellyfin stats # movie, series, episode, song counts (/Items/Counts) ``` -Shows: total count of movies, series, episodes, and songs in the library. +## Pipeline recipes -## Global Flags - -These flags work anywhere in the command — before or after the subcommand: +### Find a series, then its next unwatched episode ```bash -scripts/jellyfin --json recent --limit 5 # JSON output -scripts/jellyfin recent --limit 5 --json # same result, after subcommand -scripts/jellyfin --dry-run search --query "dune" # preview request without API call +scripts/jellyfin search --query "breaking bad" --type Series --json | jq -r '.results[0].id' +scripts/jellyfin next-up --series-id "$SERIES_ID" --user-id "$JELLYFIN_USER_ID" --json | jq -r '.items[0].name' ``` -| Flag | Effect | -|------|--------| -| `--json` | Output machine-readable JSON instead of human-readable text | -| `--dry-run` | Show each request path and parameters without executing it | +### Page through a whole library + +```bash +scripts/jellyfin browse --library-id "$LIB_ID" --limit 100 --start-index 0 --json | jq -c '.items' +# loop: advance --start-index by the returned count until .total_record_count is reached +``` + +### Log in and persist a session + +```bash +scripts/jellyfin login --username alice --prompt --json | jq -r '"\(.user_id) \(.access_token)"' +``` + +## JSON and jq + +Put `--json` before or after the subcommand. Output keys are stable snake_case: `items` +(with `id`, `name`, `type`, `year`, `series`, `season_number`, `episode_number`), +`results`, `libraries`, `total_record_count`, `start_index`. `--dry-run` emits the exact +`path` + `params` (or `authorization_header` for `login`) that would be sent, so jq can +verify a chain before running it live. Use `jq -r '.items[] | [.name, .year] | @tsv'` for +tabular handoff. ## Known Gotchas -- **JELLYFIN_URL must include protocol and port** — Both are required, e.g. `http://192.168.1.100:8096`. A bare hostname or IP without `http://` and `:8096` will fail. The default is `http://localhost:8096`. -- **User-scoped commands require an explicit user** — Set `JELLYFIN_USER_ID` or pass `--user-id USER_ID` to `recent`, `next-up`, or `item`. The CLI never selects an administrator automatically. A real request without either value fails before network access; dry-run previews the request with a null user ID. -- **Recent type filtering is server-side** — `--movies` and `--episodes` become the `/Items/Latest` `includeItemTypes` parameter before `limit`; no local filtering is applied. -- **Search type values** — The `--type` flag for `search` uses Jellyfin item type names (e.g. `Movie`, `Series`, `Episode`, `MusicArtist`, `MusicAlbum`). Multiple types are comma-separated without spaces. -- **API key location** — Generate the key in the Jellyfin Dashboard under **Dashboard → API Keys**. The key is sent as the `X-Emby-Token` header. -- **Lazy auth** — `--help` and `--dry-run` work even when `JELLYFIN_URL` and `JELLYFIN_API_KEY` are not set. Dry-run reports request paths and parameters but never sends credentials or makes a network call. -- **No pagination** — Every command returns a single page of results. The CLI does not auto-paginate beyond the first response. Use `--limit` to control result size. +- **JELLYFIN_URL must include protocol and port** — e.g. `http://192.168.1.100:8096`. +- **User-scoped commands require an explicit user** — `recent`, `next-up`, `item`, + `seasons`, `episodes` refuse to run without `JELLYFIN_USER_ID`/`--user-id`. The CLI + never picks an administrator for you. A missing userId on user-token requests makes the + server answer `400 userId is required`. +- **API keys have no user** — `/Users/Me` answers `400 Token is not owned by a user.` to + API keys by design; per-user queries need an explicit user id (see + [references/user-scoping-and-errors.md](references/user-scoping-and-errors.md)). +- **The login 400 vs 401 trap** — missing/partial MediaBrowser header → `400` with plain + text `Error processing request.`; wrong credentials → `401`. Same endpoint, different + failures. +- **Response shapes differ per endpoint** — `/Items` and `/Shows/*` wrap results in + `{Items, TotalRecordCount, StartIndex}`; `/Items/Latest` returns a bare array; search + uses a `SearchHints` key. Generic clients must branch (the CLI already does). +- **Recent type filtering is server-side** — `--movies`/`--episodes` become + `includeItemTypes` before `limit`; no local filtering. +- **NextUp needs userId on every server version** — omitting it crashed servers ≤10.8 and + silently scopes to the session user on ≥10.9. The CLI always sends it. +- **`libraries` is admin-only** — `/Library/MediaFolders` requires an administrator token; + non-admin tokens get 403. +- **Legacy auth is going away** — `X-Emby-Token`, `X-MediaBrowser-Token`, and the + `api_key` query parameter are deprecated; admins can already disable them (10.11+), and + removal targets 12.0. Prefer the modern Authorization header the CLI sends. +- **Lazy auth** — `--help` and `--dry-run` work without credentials; dry-run never touches + the network. -## References +## When to use -- [scripts/jellyfin](scripts/jellyfin) — The bundled read-only CLI binary with `--json`, `--dry-run`, and lazy authentication. -- [Jellyfin API Docs](https://jellyfin.org/docs/general/clients/api) — Official API documentation. -- [Jellyfin Downloads](https://jellyfin.org/downloads) — Server download and setup guide. +Use this skill for read-only interaction with a running Jellyfin server: discovery of +what's new, searching and inspecting items, walking series and seasons, next-up planning, +library inventories, and obtaining a user session via `login` or Quick Connect. ## When not to use -Do not use this skill for playback control or library management (starting streams, editing item metadata, creating users) — every bundled command is read-only; route remote-control automation to Jellyfin's official clients, and Plex or Kodi servers expose their own separate APIs. +Do not use this skill for server installation or administration (installing Jellyfin or +Emby, editing libraries, managing users) — every bundled command is read-only except +`login`. It does not target Plex or Kodi (different APIs — use their own tools), and it is +not a playback remote: route streaming or remote-control automation to Jellyfin's official +clients. + +## Reference Files + +| File | Use it for | +| ---- | ---------- | +| [references/auth-and-sessions.md](references/auth-and-sessions.md) | MediaBrowser header scheme, login flow, token channels, legacy deprecation, error signatures | +| [references/endpoint-catalog.md](references/endpoint-catalog.md) | Every read endpoint's parameters, response shapes, image URLs, pagination loop | +| [references/user-scoping-and-errors.md](references/user-scoping-and-errors.md) | The userId requirement matrix, API-key identity quirks, 400-vs-404 diagnosis | +| [references/gotchas-field-guide.md](references/gotchas-field-guide.md) | Wire-level failure signatures, version-drift ledger, mock shapes | +| [references/worked-recipes.md](references/worked-recipes.md) | Multi-step curl/jq and CLI recipes: login → latest, libraries → browse, search → seasons → episodes | +| [references/quick-connect.md](references/quick-connect.md) | Passwordless Quick Connect login flow | + +## Available Scripts and Prerequisites + +- `scripts/jellyfin` — the bundled Python CLI (`--json`, `--dry-run`, lazy auth). + Imports only the standard library and `requests`. +- `scripts/test_jellyfin_cli.py` — offline test suite (pytest + unittest compatible); + all HTTP behavior is mocked, zero network egress. +- Requires Python 3.8+ and `requests`. A running Jellyfin server (10.8+ assumed; tested + behaviors anchored to the 12.0-era OpenAPI spec). No service is started by this skill. diff --git a/jellyfin/evals/evals.json b/jellyfin/evals/evals.json new file mode 100644 index 0000000..093a85a --- /dev/null +++ b/jellyfin/evals/evals.json @@ -0,0 +1,68 @@ +{ + "schema_version": 1, + "skill_name": "jellyfin", + "evals": [ + { + "id": "recent-movies-json", + "prompt": "Show me the five movies most recently added to my Jellyfin server, as JSON.", + "expected_output": "Run scripts/jellyfin recent --movies --limit 5 --json with JELLYFIN_URL, JELLYFIN_API_KEY, and JELLYFIN_USER_ID configured; the /Items/Latest response is a bare JSON array, not an {Items: [...]} wrapper.", + "assertions": [ + "invokes scripts/jellyfin recent with --movies and --limit 5", + "uses --json for machine-readable output", + "does not wrap the result in an Items key because /Items/Latest returns a bare array" + ] + }, + { + "id": "search-to-episodes-pipeline", + "prompt": "Find the series Breaking Bad on my Jellyfin server and then list its episodes.", + "expected_output": "Chain scripts/jellyfin search --query 'breaking bad' --type Series --json to get a result id, then scripts/jellyfin seasons --series-id --user-id , then scripts/jellyfin episodes --series-id --season-id --user-id , consuming each stage's id output in the next command.", + "assertions": [ + "starts with a search using --type Series", + "extracts the id field from search results before the next stage", + "walks seasons then episodes with --series-id and --user-id", + "passes the user id explicitly on every user-scoped command" + ] + }, + { + "id": "authenticatebyname-pretoken-header", + "prompt": "My script calls POST /Users/AuthenticateByName on Jellyfin with just the username and password JSON and gets HTTP 400 'Error processing request.' even though the credentials are correct. Why?", + "expected_output": "The login endpoint requires a complete Authorization header in the MediaBrowser scheme - Client=\"...\", Device=\"...\", DeviceId=\"...\", Version=\"...\" - BEFORE any access token exists; the server uses those values to create the session and rejects the request with 400 when the header is missing. The returned AccessToken is then sent as Token=\"...\" in the same Authorization header on subsequent calls (X-Emby-Token is the deprecated legacy equivalent).", + "assertions": [ + "explains the pre-token MediaBrowser Client/Device/DeviceId/Version header requirement", + "diagnoses the 400 Error processing request. body as the missing-header signature", + "distinguishes wrong-credential 401 from missing-header 400", + "notes the AccessToken is sent via Token= or legacy X-Emby-Token afterwards" + ] + }, + { + "id": "user-id-scoping-diagnosis", + "prompt": "Queries against my Jellyfin server work with curl on /System/Info but GET /Items returns 400 saying 'userId is required', and /Users/Me returns 400 'Token is not owned by a user.' What is wrong?", + "expected_output": "The token is a dashboard API key, which authenticates as an administrator but has NO user identity. User-scoped endpoints need an explicit userId query parameter, and /Users/Me intentionally rejects API keys. Find a user id via GET /Users and pass it (or use a user access token from login).", + "assertions": [ + "identifies API-key authentication as the cause", + "explains userId must be passed explicitly on /Items and other user-scoped endpoints", + "resolves it by listing /Users or logging in for a user token" + ] + }, + { + "id": "latest-shape-and-pagination-gotcha", + "prompt": "Parse Jellyfin recently-added output in a generic client: which response shapes differ across endpoints, and how should paging loop over /Items?", + "expected_output": "/Items, /Shows/*, and /Search/Hints return a wrapper object {Items, TotalRecordCount, StartIndex}; /Items/Latest returns a bare array of items; /Items/{id} returns a single object. Page /Items with startIndex plus limit and stop on an empty page or when startIndex reaches TotalRecordCount, since counts can go stale mid-scan.", + "assertions": [ + "distinguishes wrapper-object, bare-array, and single-object response shapes", + "names /Items/Latest as the bare-array exception", + "describes startIndex/limit paging with an empty-page guard" + ] + }, + { + "id": "emby-install-not-for-jellyfin", + "prompt": "Help me install an Emby server on my NAS and migrate my Plex library into it.", + "expected_output": "This must not trigger the jellyfin skill: server installation, Emby/Plex administration, and media-ripper workflows are outside its read-only Jellyfin query scope. Use OS package tooling for the install and the target server's own migration tooling.", + "assertions": [ + "must not trigger jellyfin for Emby or Plex server installation", + "refuses server administration and library-migration work", + "points to OS tooling instead of the read-only Jellyfin query CLI" + ] + } + ] +} diff --git a/jellyfin/references/auth-and-sessions.md b/jellyfin/references/auth-and-sessions.md new file mode 100644 index 0000000..9060abb --- /dev/null +++ b/jellyfin/references/auth-and-sessions.md @@ -0,0 +1,175 @@ +# Jellyfin authentication and sessions + +How tokens are minted, how they travel, and what each failure looks like on the wire. +Every behavioral claim here traces to the canonical OpenAPI spec served from api.jellyfin.org, +the Jellyfin server source code, or a document authored by a Jellyfin core developer (Sources footer). + +## The two token postures + +Jellyfin has two credential families plus anonymous access: + +| Posture | Obtained via | Lifetime | Identity | Typical use | +| --- | --- | --- | --- | --- | +| User access token | `POST /Users/AuthenticateByName` (or Quick Connect) | Session-based, valid until logout or revocation | Bound to one user (and one device id) | Acting as a person; playstate, per-user views | +| API key | Dashboard → API Keys (admin panel) | Persistent until revoked | No user identity; administrator-level privileges | Server automation, read-only browsing CLIs | + +Anonymous access (no header at all) works only for endpoints that opt in, notably +`GET /System/Info/Public`, `GET /Users/Public`, and `POST /QuickConnect/Initiate`. +`GET /System/Info/Public` is the recommended pre-auth probe: it returns `ServerName` and +`Version` without credentials, which is how a client learns the server version before +adapting its behavior. + +An API key is not a lesser credential: it bypasses user identity entirely and gets +administrator role. It also means every user-scoped concept (per-user views, playstate, +"my" recent items) has no one to attach to unless you pass an explicit `userId` parameter. + +## The MediaBrowser authorization scheme (required BEFORE any token exists) + +Jellyfin's `Authorization` header uses a custom scheme named `MediaBrowser` with +comma-separated, order-insensitive `Key="value"` parameters: + +``` +Authorization: MediaBrowser Client="my-cli", Device="terminal", DeviceId="unique-device-id", Version="1.0.0" +``` + +Parameter keys (case-sensitive, alphanumeric, unknown keys ignored by the server): + +| Key | Meaning | +| --- | --- | +| `Client` | Name of the client application | +| `Device` | Human-readable device name | +| `DeviceId` | Client-generated unique device identifier | +| `Version` | Client application version | +| `Token` | Access token or API key — present only AFTER you have one | + +Values must be wrapped in double quotes and should be URL-encoded; the official TypeScript +SDK wraps every value in `encodeURIComponent`. The server URL-decodes values after parsing. + +**The login chicken-and-egg.** `POST /Users/AuthenticateByName` must be sent with a +COMPLETE `Client`/`Device`/`DeviceId`/`Version` header — before any token exists. The server +uses those four strings as the new session's identity: the controller builds an +`AuthenticationRequest` from the parsed header values, and `SessionManager` hard-fails with +`ArgumentException.ThrowIfNullOrEmpty` on any missing `App`, `DeviceId`, `DeviceName`, or +`AppVersion`. The exception middleware maps `ArgumentException` to HTTP 400, so the classic +CLI failure mode — sending no header, or only `Content-Type` — looks like this on the wire: + +``` +HTTP/1.1 400 Bad Request +Content-Type: text/plain + +Error processing request. +``` + +(Non-development servers replace the real exception text with the literal string +`Error processing request.`.) A bare-token `Authorization: ` without the MediaBrowser +scheme fails the same class of parse and was observed as 401 in issue #12990; the +correctly-wrapped header succeeded. DeviceId hygiene: the server permits a single access +token per `DeviceId`, and re-logging-in the same `(DeviceId, user)` pair silently revokes +that pair's previous token — mix a per-profile discriminator (e.g. hashed username) into the +DeviceId when one machine drives several accounts. + +## Exact login flow + +Request (note: full MediaBrowser header, NO Token segment yet): + +``` +curl -X POST "http://localhost:8096/Users/AuthenticateByName" \ + -H "Content-Type: application/json" \ + -H 'Authorization: MediaBrowser Client="my-cli", Device="terminal", DeviceId="dev-1", Version="1.0.0"' \ + -d '{"Username": "alice", "Pw": "secret"}' +``` + +Body schema `AuthenticateUserByName`: `Username` (string) and `Pw` (PLAIN TEXT password). +There is an older `Password` sha1-hash slot in some legacy documentation — do not use it; +send plaintext in `Pw`. + +Response 200 (`AuthenticationResult`; properties shown PascalCase, the server default): + +```json +{ + "User": { "Name": "alice", "Id": "6eec632a-ff0d-4d09-aad0-bf9e90b14bc6", "HasPassword": true }, + "SessionInfo": { "Id": "a1b2c3d4e5f6", "UserId": "6eec632a-ff0d-4d09-aad0-bf9e90b14bc6", + "UserName": "alice", "Client": "my-cli", "DeviceId": "dev-1", + "DeviceName": "terminal", "ApplicationVersion": "1.0.0" }, + "AccessToken": "", + "ServerId": "abc123def456" +} +``` + +Capture `User.Id` (this is the user id every user-scoped endpoint wants) and `AccessToken`. +The spec marks the operation's only documented non-200 as 503 (server starting); credential +failures arrive via exception mapping instead: wrong username/password → 401 +("Invalid username or password entered." in server logs), disabled user or device/session +policy rejection → 403, missing/partial MediaBrowser header → 400 as above. + +After login, `GET /Users/Me` with the token is the cleanest identity re-check; it returns +the authenticated user's `UserDto`. With an API key instead of a user token, `/Users/Me` +answers 400 with the JSON body `Token is not owned by a user.` — API keys are userless. + +## Sending the token afterwards + +| Channel | Form | Status | +| --- | --- | --- | +| `Authorization` header, full scheme | `Authorization: MediaBrowser Client="c", Device="d", DeviceId="i", Version="v", Token=""` | Recommended | +| `Authorization` header, token-only | `Authorization: MediaBrowser Token=""` | Valid (API-key example in official docs) | +| Query parameter | `?ApiKey=` | Discouraged (leak risk in logs); never combine with the header | +| `X-Emby-Token` header | `X-Emby-Token: ` | Deprecated (legacy) | +| `X-MediaBrowser-Token` header | `X-MediaBrowser-Token: ` | Deprecated (legacy) | +| `X-Emby-Authorization` header | full MediaBrowser scheme on a legacy header name | Deprecated (legacy) | + +The bundled `scripts/jellyfin` sends the modern form: +`Authorization: MediaBrowser Client="...", Device="...", DeviceId="...", Version="...", Token=""`. +It is wire-equivalent to the legacy `X-Emby-Token` header on every server that supports +legacy auth, and it keeps working when legacy channels are switched off. + +Never send two different tokens in one request — server precedence across channels is +unspecified at the contract level and the value used becomes uncertain. + +### Legacy kill-switch and deprecation timeline + +- All legacy channels above are gated by server config `EnableLegacyAuthorization` + (`system.xml`), default **true** through 10.11.x. Setting it to `false` (introduced in + 10.11) makes `X-Emby-Token`, `X-MediaBrowser-Token`, `api_key`, and + `X-Emby-Authorization` stop resolving — a client that "worked yesterday" and now gets + uniform 401s has almost certainly met this toggle. +- Maintainers have targeted disabling the deprecated options starting with the 12.0 + release. Speak the modern `Authorization` header natively; keep `X-Emby-Token` only as a + compat fallback for old servers. + +## Error signatures worth mocking + +| Scenario | Status | Body | +| --- | --- | --- | +| AuthenticateByName without (full) MediaBrowser header | 400 | text/plain `Error processing request.` | +| AuthenticateByName with wrong credentials | 401 | text/plain `Error processing request.` | +| Disabled user / device or session policy reject | 403 | text/plain | +| Token matches nothing (garbage token on a secured read) | 403 | `Invalid token.` (SecurityException mapping; some doc renders simplify this to 401 — trust the middleware mapping) | +| Secured read with no token at all | 401 | challenge | +| API key on `GET /Users/Me` | 400 | JSON ProblemDetails containing `Token is not owned by a user.` | +| Server starting / restarting | 503 | HTML/text body with `Retry-After: ` and `Message: ` headers | + +The 503 can appear on ANY endpoint during startup; retry loops should honor `Retry-After`. + +## Quick Connect (passwordless alternative) + +For shared or headless setups where you do not want to handle a password: + +1. `POST /QuickConnect/Initiate` (no auth) → 200 `QuickConnectResult` with `Secret`, + `Code`, `Authenticated:false`. A 401 here means Quick Connect is disabled on that server. +2. Poll the quick-connect state endpoint (about every 5 seconds) until + `Authenticated` flips to `true`, while the user approves the `Code` in their client. +3. `POST /Users/AuthenticateWithQuickConnect` with body `{"Secret": ""}` → + 200 `AuthenticationResult` — same capture and follow-up as the password flow. + +## Sources + +- https://api.jellyfin.org/ — official Jellyfin API reference (ReDoc), version 12.0.0 stable +- https://api.jellyfin.org/openapi/jellyfin-openapi-stable.json — canonical OpenAPI spec (AuthenticateByName schema, 503 blocks, /Users/Me 400) +- https://gist.github.com/nielsvanvelzen/ea047d9028f676185832e51ffaf12a6f — "Jellyfin API Authorization" by a Jellyfin core developer (MediaBrowser scheme, legacy table, kill-switch steps, removal-to-12.0 quote) +- https://mintlify.wiki/jellyfin/jellyfin/api/authentication/overview — official server docs, authentication overview (login curl examples, API key vs user token table, logout) +- https://jmshrv.com/posts/jellyfin-api/ — community API overview by the Jellyfin for Jellyfin/Roku author +- https://typescript-sdk.jellyfin.org/ — official TypeScript SDK (getAuthorizationHeader construction, login-then-update flow) +- https://kotlin-sdk.jellyfin.org/guide/authentication.html — official Kotlin SDK authentication guide (401-on-bad-credentials, Quick Connect cadence) +- https://github.com/jellyfin/jellyfin/issues/12990 — wire-level reproduction of missing/bare-token header failures +- https://github.com/jellyfin/jellyfin — server source: `AuthorizationContext.cs`, `SessionManager.cs`, `UserController.cs`, `ExceptionMiddleware.cs`, `AuthService.cs`, `CustomAuthenticationHandler.cs`, `ServerConfiguration.cs` +- https://github.com/jellyfin/jellyfin-apiclient-python — reference client implementation diff --git a/jellyfin/references/endpoint-catalog.md b/jellyfin/references/endpoint-catalog.md new file mode 100644 index 0000000..149ebbc --- /dev/null +++ b/jellyfin/references/endpoint-catalog.md @@ -0,0 +1,191 @@ +# Jellyfin endpoint catalog for browsing and search + +Read-only endpoints a browsing CLI needs. Parameter names are verbatim from the canonical +stable OpenAPI spec (api.jellyfin.org). Casing rule resolved there: **query parameters are +camelCase** (`sortBy`, `sortOrder`, `startIndex`, `includeItemTypes`, `parentId`, +`searchTerm`), while **JSON payload properties are PascalCase in the default profile** +(`Items`, `TotalRecordCount`, `Name`, `Id`, `AccessToken`). Some servers emit camelCase +properties depending on the response profile; treat casing as a compatibility minefield and +normalize client-side. + +## Auth posture per endpoint + +| Endpoint | Method | Auth | +| --- | --- | --- | +| `/System/Info/Public` | GET | none — pre-auth version probe | +| `/Users/Public` | GET | none — login-screen user list | +| `/System/Info` | GET | any valid token (API key or user token) | +| `/Users` | GET | any valid token; params `isHidden`, `isDisabled` | +| `/Users/Me` | GET | user token; 400 "Token is not owned by a user." with an API key | +| `/Users/AuthenticateByName` | POST | none (but see auth reference: MediaBrowser header mandatory) | +| `/UserViews?userId=` | GET | any token — the per-user library list | +| `/Library/MediaFolders` | GET | **admin-only** (RequiresElevation policy) | +| `/Items` | GET | token; `userId` required unless API-key auth | +| `/Items/Latest` | GET | token; `userId` param | +| `/Items/{itemId}` | GET | token; `userId` optional | +| `/Items/Counts` | GET | token; `userId` optional | +| `/Search/Hints` | GET | token; `searchTerm` required | +| `/Shows/{seriesId}/Seasons` | GET | token; `userId` param | +| `/Shows/{seriesId}/Episodes` | GET | token; `userId` param | +| `/Shows/NextUp` | GET | token; `userId` param | + +`GET /Users/{userId}/Items` still routes (legacy twin of `/Items?userId=`); controllers mark +the `/Users/{userId}/Views` route `[Obsolete]`. Prefer `/Items` and `/UserViews` on any +recent server. + +## /System/Info and /System/Info/Public + +`/System/Info/Public` (no auth) returns `PublicSystemInfo`: `ServerName`, `Id`, `Version`, +`ProductName`. Use it as the cheap pre-flight: learn the version before choosing between +behaviors that differ across server releases. `/System/Info` (token) adds `OperatingSystem`, +`HasUpdateAvailable`, and more; the bundled `info` command calls `/System/Info` plus +`/Users` to count users. + +## /UserViews — the user's libraries + +`GET /UserViews?userId=` returns a `BaseItemDtoQueryResult` of the libraries (views) +that user can see: `Items[]` with `Id`, `Name`, `CollectionType` (`movies`, `tvshows`, +`music`, ...), `TotalRecordCount`. This is the correct "list my libraries" endpoint for any +token; `/Library/MediaFolders` lists raw library folders but requires an administrator +token (403 Forbidden otherwise) and is not user-scoped. + +## /Items — the workhorse query + +`GET /Items` declares ~88 query parameters in the stable spec. The core browsing set: + +| Param | Meaning | +| --- | --- | +| `userId` | **Required unless authenticating with an API key.** Missing on a user-token request → 400 with body `userId is required`. | +| `parentId` | Localize the query to one folder/view; omit for the root | +| `recursive` | Recurse into subfolders (use with `parentId` to enumerate a whole view) | +| `includeItemTypes` | Comma-delimited item types (see enum below) | +| `excludeItemTypes` | Comma-delimited inverse filter | +| `sortBy` | Comma-delimited sort keys: `SortName`, `DateCreated`, `PremiereDate`, `CommunityRating`, `Random`, `ProductionYear`, `ParentIndexNumber`, `IndexNumber`, ... | +| `sortOrder` | `Ascending` or `Descending` (comma-delimited to match multi-key sorts) | +| `startIndex`, `limit` | Paging window | +| `fields` | Comma-delimited extra fields to populate (see below) | +| `filters` | `IsUnplayed`, `IsPlayed`, `IsFavorite`, `IsResumable`, ... | +| `searchTerm` | Term filter inside `/Items` | +| `isPlayed`, `isFavorite` | Boolean filters | +| `genres`, `years`, `studios`, `artists`, `person`, `tags` | Facet filters | +| `enableTotalRecordCount` | Default true; server may skip computing `TotalRecordCount` when false | + +Item type enum (`BaseItemKind`, the values you actually use): `Movie`, `Series`, `Season`, +`Episode`, `BoxSet`, `MusicAlbum`, `MusicArtist`, `Audio`, `Photo`, `PhotoAlbum`, `Book`, +`AudioBook`, `Playlist`, `Trailer`, `Channel`, `Folder`, `UserView`, `Genre`, `Studio`, +`Person`, `Year`. `fields` enum members include `Overview`, `Genres`, `People`, `Path`, +`MediaSources`, `MediaStreams`, `ProviderIds`, `Tags`, `DateCreated`, `ChildCount`, +`RecursiveItemCount`, `PrimaryImageAspectRatio`, `SortName`, `OriginalTitle`, `Etag`. + +Response 200 is a `BaseItemDtoQueryResult` OBJECT — always this wrapper: + +```json +{ "Items": [ { "Name": "Arrival", "Id": "72c5b8e6-...", "Type": "Movie" } ], + "TotalRecordCount": 137, "StartIndex": 0 } +``` + +Error contract: `400` (text body `userId is required`) when `userId` is absent on +non-API-key auth; `404` when a supplied-but-nonexistent `userId` fails lookup (the user +lookup happens before the missing-userId guard, so invalid id ≠ absent id); `401`/`403` +per the auth matrix. + +## /Items/Latest — recently added (returns a bare ARRAY) + +`GET /Items/Latest` params: `userId`, `parentId`, `fields`, `includeItemTypes`, `isPlayed`, +`enableImages`, `imageTypeLimit`, `enableImageTypes`, `enableUserData`, `limit` +(**default 20**), `groupItems` (**default true** — groups episodes by series and movies by +edition). + +**Shape trap: the 200 response is a bare JSON ARRAY of `BaseItemDto` — NOT a +`BaseItemDtoQueryResult` wrapper.** There is no `Items` key, no `TotalRecordCount`, no +`StartIndex`. Clients that unwrap `.Items` unconditionally break here (that is exactly the +shape branch the bundled CLI handles in `cmd_recent`). + +Because grouping can merge an entire series into one entry, and the item ids in the array +are per-entry, treat `groupItems=true` results as "what's new", not "how many". Filter by +type server-side with `includeItemTypes` (e.g. `Movie,Series,Episode`); the requested +`limit` then applies to the selected types. + +## /Search/Hints — fast fuzzy search + +`GET /Search/Hints` requires `searchTerm`; optional `startIndex`, `limit`, `userId` +("search within a user's library or omit to search all"), `includeItemTypes`, +`excludeItemTypes`, `mediaTypes` (`Unknown,Video,Audio,Photo,Book`), `parentId`, and +boolean includes (`includePeople`, `includeMedia`, `includeGenres`, `includeStudios`, +`includeArtists`, all default true). + +Response 200: + +```json +{ "SearchHints": [ { "Id": "e60d4fa5-...", "ItemId": "e60d4fa5-...", "Name": "Breaking Bad", + "Type": "Series", "ProductionYear": 2008, "MatchedTerm": "break", + "Series": "..." } ], + "TotalRecordCount": 3 } +``` + +SearchHint carries BOTH `Id` and `ItemId` with the same value — `ItemId` is marked +deprecated in the spec; read `Id` and fall back to `ItemId` on old servers. Hints also +surface people/genres/studios as pseudo-results when those includes are on, which `/Items` +does not do; hints honor fewer filters and no `sortBy`. + +## TV navigation family + +| Endpoint | Params | Response | +| --- | --- | --- | +| `/Shows/{seriesId}/Seasons` | `seriesId` (path, required), `userId`, `fields`, `isSpecialSeason`, `isMissing` | `BaseItemDtoQueryResult` of `Season` items (`IndexNumber` 0 = specials convention) | +| `/Shows/{seriesId}/Episodes` | above plus `season` (int) or `seasonId` (Guid), `startItemId`, `startIndex`, `limit`, `sortBy` (SCALAR here, unlike /Items' comma array) | `BaseItemDtoQueryResult` of `Episode` items | +| `/Shows/NextUp` | `userId`, `startIndex`, `limit`, `fields`, `seriesId`, `parentId`, `nextUpDateCutoff` (ISO date-time), `enableTotalRecordCount`, `enableResumable` (default true), `enableRewatching` (default false) | `BaseItemDtoQueryResult` of `Episode` items | + +Prefer `seasonId` (GUID) over numeric `season` — season numbers shift when specials are +inserted. Season-less `/Shows/{seriesId}/Episodes` returns every episode across seasons in +order. `enableRewatching` defaults to false: a fully-watched series never reappears in +NextUp unless opted in. + +## /Items/{itemId} and /Items/Counts + +`GET /Items/{itemId}?userId=` returns one `BaseItemDto` (~155 properties: `Name`, `Id`, +`Type`, `SeriesId`, `SeasonId`, `SeriesName`, `IndexNumber`, `ParentIndexNumber`, +`RunTimeTicks`, `ProductionYear`, `Overview`, `Genres`, `MediaSources`, `ImageTags`, +`BackdropImageTags`, `UserData`, `ProviderIds`, ...). `userId` is optional — supply it to +get that user's `UserData` (playstate, favorites) embedded. Missing item → 404. + +`GET /Items/Counts?userId=` returns the `ItemCounts` object: `MovieCount`, +`SeriesCount`, `EpisodeCount`, `SongCount`, `AlbumCount`, `ArtistCount`, `TrailerCount`, +`BoxSetCount`, `BookCount`, `MusicVideoCount`, `ProgramCount`, `ItemCount`. + +## Images (read) + +`GET /Items/{itemId}/Images/{imageType}` serves image bytes for `Primary`, `Logo`, `Thumb`, +`Backdrop`, `Banner`, and more. Tunables: `maxWidth`/`maxHeight`, `quality`, `fillWidth`/ +`fillHeight`, `tag` (supply the `ImageTags` value from the DTO to get long-lived cacheable +URLs), `format`. A 404 is the documented response when the item simply has no such image — +treat it as normal fallback flow, not an error. Backdrops are indexed: +`BackdropImageTags[i]` pairs with `/Items/{itemId}/Images/Backdrop/{i}`. + +## Pagination pattern (that actually works) + +`/Items`, `/Shows/*`, and `/Search/Hints` page with `startIndex` + `limit` and report +`TotalRecordCount`. Loop with BOTH guards — counts can go stale mid-scan on a live server: + +```python +start, page = 0, 100 +while True: + r = get_items(user_id, parent_id, start_index=start, limit=page) + items = r.get("Items", []) + if not items: + break # short/empty page = done, even if count disagrees + yield from items + start += len(items) + if r.get("TotalRecordCount") and start >= r["TotalRecordCount"]: + break +``` + +`/Items/Latest` has no pagination at all — it is a single array capped by `limit`. + +## Sources + +- https://api.jellyfin.org/ — official Jellyfin API reference (ReDoc), version 12.0.0 stable +- https://api.jellyfin.org/openapi/jellyfin-openapi-stable.json — canonical OpenAPI spec (all parameter tables, enums, response schemas, 400/401/403/404/503 blocks) +- https://github.com/jellyfin/jellyfin — server source confirming behavior: `ItemsController.cs` (userId 400/404 ordering, legacy /Users/{userId}/Items), `UserViewsController.cs` (obsolete Views route), `LibraryController.cs` (MediaFolders elevation), `TvShowsController.cs` (NextUp/Seasons/Episodes), `UserLibraryController.cs` (GET /Items/{itemId}) +- https://mintlify.wiki/jellyfin/jellyfin/api/authentication/overview — official server docs (logout, /Users/Me semantics) +- https://jmshrv.com/posts/jellyfin-api/ — community walkthrough of the items/search/images surface diff --git a/jellyfin/references/gotchas-field-guide.md b/jellyfin/references/gotchas-field-guide.md new file mode 100644 index 0000000..1286351 --- /dev/null +++ b/jellyfin/references/gotchas-field-guide.md @@ -0,0 +1,130 @@ +# Jellyfin gotchas field guide + +Version-sensitive, wire-level failure signatures observed in server source, the OpenAPI +spec, and tracked issues. Diagnostic order: auth posture → user scoping → response shape → +version drift. + +## Response-shape asymmetry (the biggest interop trap) + +Three families, three shapes — a generic client must branch: + +| Endpoint | 200 shape | +| --- | --- | +| `GET /Items`, `/Shows/*`, `/Search/Hints`, `/UserViews` | Wrapper object: `{ Items: [...], TotalRecordCount, StartIndex }` (search: `SearchHints` key) | +| `GET /Items/Latest` | **Bare ARRAY** of `BaseItemDto` — no `Items` key, no `TotalRecordCount`, no `StartIndex` | +| `GET /Items/{itemId}`, `AuthenticateByName` | Single object | + +`/Items/Latest` defaults: `limit` 20, `groupItems` true (episodes merge into series rows — +so it answers "what's new", not "how many"). + +## Property-casing minefield + +Every JSON-producing operation documents three profiles: `application/json`, +`application/json; profile="CamelCase"`, `application/json; profile="PascalCase"`. +Observed defaults vary between server eras and clients; SDKs read PascalCase off raw +payloads while sample captures show camelCase. Normalize defensively: read both `Name` +and `name`, both `AccessToken` and `accessToken`, rather than trusting one casing. Query +PARAMETERS are always camelCase (`sortBy`, `startIndex`) regardless of profile. + +## Error signatures on the wire + +| Symptom | Actual cause | +| --- | --- | +| 400 text/plain `Error processing request.` on login | Missing/partial `Authorization: MediaBrowser Client=..., Device=..., DeviceId=..., Version=...` header — required BEFORE any token exists (ArgumentException mapping) | +| 401 on login | Wrong username/password ("Invalid username or password entered." in server logs) | +| 403 on login | Disabled user, device-access policy, or `MaxActiveSessions` cap | +| 401 + log `AuthenticationScheme: "CustomAuthentication" was challenged.` | Secured read with no/insufficient token | +| 403 with valid-format token | Token matches nothing (`Invalid token.` via SecurityException) or permission denied — note this is 403, not 401; some doc renders simplify it to 401 | +| 400 `Token is not owned by a user.` (JSON) | API key on `/Users/Me` — API keys are userless | +| 400 `userId is required` (plain string body) | `GET /Items` without `userId` on non-API-key auth | +| 404 on a user-scoped query | Supplied `userId` does not exist (user lookup precedes the missing-param guard) | +| 503 + `Retry-After` + `Message` headers | Server starting/restarting — can hit ANY endpoint; honor `Retry-After` | +| "Worked yesterday", now uniform 401s | Admin set `EnableLegacyAuthorization=false` (possible since 10.11): all `X-Emby-*`/`api_key` channels stopped resolving | + +No rate limiting exists in the spec — Jellyfin is self-hosted. A 429 comes from a reverse +proxy, not Jellyfin. + +## Auth-channel pitfalls + +- `X-Emby-Token`, `X-MediaBrowser-Token`, `api_key` query param, and the + `X-Emby-Authorization` header are deprecated legacy channels; maintainers target + disabling them from 12.0. Prefer `Authorization: MediaBrowser ... Token="..."`. +- One access token per `(DeviceId, user)` pair: re-logging-in the same pair revokes the + pair's previous token. Multi-profile CLIs must vary the DeviceId per profile or they will + keep logging each other out. +- Never send two token channels in one request; which one wins is not contractual. + +## User-scoping pitfalls + +- API key = administrator + no user. Everything per-user (views, latest, next-up, played + state) needs an explicit `userId` parameter, and `UserData` fields stay empty otherwise. +- Recent servers fall back to the token's user when `userId` is omitted on some endpoints — + which makes omissions work on your server and fail on someone else's API-key deployment. + Always send it. + +## TV navigation quirks + +- **NextUp `userId` version split:** ≤10.8 crashes with `ArgumentException: Guid can't be + empty` when omitted; ≥10.9 falls back to the session user. Pass `userId` unconditionally. +- **NextUp `limit` limits returned items, not series scanned.** The 2024 attempt to make + `limit` prune the scan made items vanish from NextUp days later and was reverted — keep + page sizes modest, expect long-tail behavior differences between 10.9.x and 10.10.x. +- `enableRewatching` defaults false: a fully-watched series never reappears in NextUp. +- Prefer `seasonId` (GUID) over numeric `season` on `/Shows/{seriesId}/Episodes` — season + numbers shift when specials get inserted. Season `IndexNumber` 0 is the specials + convention. +- `sortBy` on `/Shows/{seriesId}/Episodes` is SCALAR, unlike the comma-delimited array form + `/Items` accepts. + +## Field-selection and caching + +- `BaseItemDto` declares ~155 properties but only requested `fields` populate extras; + `Overview`, `Path`, `MediaSources`, `ProviderIds`, `ChildCount` are null unless asked for. + `fields=DateCreated` is what makes "recently added" sorting meaningful client-side. +- `ImageTags` values are cache keys for image routes; passing `tag=` yields long-lived + cacheable URLs. `Etag` changes on metadata edits — treat both as opaque. +- Image routes document **404 as the normal "no such image" response** — fall back to the + next image type (Primary → Thumb → Parent* tags) instead of erroring. + +## Version-drift ledger (what to gate on /System/Info/Public) + +| Behavior | 10.7–10.8 | 10.9–10.10 | 10.11 / 12 | +| --- | --- | --- | --- | +| Legacy auth channels | on | on, default true | toggle exists (`EnableLegacyAuthorization`); removal targeted at 12.0 | +| NextUp userId omission | crash (500) | token-user fallback | same | +| `/Items` userId 400 vs 404 ordering | unverified | confirmed | confirmed | +| NextUp extras | `disableFirstEpisode`, `nextUpDateCutoff`, `enableRewatching` | adds `enableResumable` (default true) | same | +| `api.jellyfin.org` spec label | — | — | publishes "12.0.0" branding | + +Pre-flight `GET /System/Info/Public` (no auth) gives the `Version` to branch on. + +## Mock-test wire shapes (exact) + +Success paths: +1. `POST /Users/AuthenticateByName` with complete MediaBrowser header + `{"Username","Pw"}` + → 200 `AuthenticationResult` (`User.Id` hyphenated lowercase UUID; `AccessToken` string). +2. `GET /Items?userId=...&parentId=...&recursive=true&includeItemTypes=Movie&sortBy=SortName&sortOrder=Ascending&startIndex=0&limit=50` + → 200 `{Items: [...], TotalRecordCount: N, StartIndex: 0}`. +3. `GET /Search/Hints?searchTerm=break&limit=20` + → `{SearchHints: [{Id, ItemId (deprecated twin), Name, Type, MatchedTerm, ...}], TotalRecordCount}`. +4. `GET /Items/Latest?userId=...&limit=20&includeItemTypes=Movie,Series` + → 200 bare `[{...BaseItemDto}]`. + +Failure paths (assert status AND content-type): +5. Login without header → 400 text/plain `Error processing request.` +6. Login wrong password → 401 text/plain. +7. Reads: no token → 401 challenge; garbage token → 403. +8. API key on `/Users/Me` → 400 JSON containing `Token is not owned by a user.` +9. `/Items` absent userId (non-API-key) → 400 body literally `userId is required`. +10. `/Items` nonexistent userId → 404 `Error processing request.` +11. Any endpoint during startup → 503 with `Retry-After` and `Message` headers. + +## Sources + +- https://api.jellyfin.org/openapi/jellyfin-openapi-stable.json — canonical OpenAPI spec (response profiles, 503 blocks, defaults, schemas) +- https://github.com/jellyfin/jellyfin — server source: `ExceptionMiddleware.cs` (status mapping, body suppression), `SessionManager.cs` (session caps, token rotation), `AuthorizationContext.cs` (legacy gates), `ServerConfiguration.cs` (legacy flag), `ItemsController.cs` (400/404 ordering) +- https://github.com/jellyfin/jellyfin/issues/12990 — wire-level header-failure reproduction and challenge log line +- https://api.github.com/repos/jellyfin/jellyfin/pulls/9321 — NextUp userId omission crash evidence (also pulls/11956, pulls/12414, issue #12367 for the limit saga) +- https://gist.github.com/nielsvanvelzen/ea047d9028f676185832e51ffaf12a6f — core-developer authorization guide (legacy table, disable steps, removal timeline) +- https://kotlin-sdk.jellyfin.org/guide/authentication.html — 401-on-bad-credentials, Quick Connect cadence +- https://mintlify.wiki/jellyfin/jellyfin/api/authentication/overview — official docs error table (contrast case) diff --git a/jellyfin/references/quick-connect.md b/jellyfin/references/quick-connect.md new file mode 100644 index 0000000..fd3894f --- /dev/null +++ b/jellyfin/references/quick-connect.md @@ -0,0 +1,60 @@ +# Jellyfin quick connect + +Quick Connect is Jellyfin's passwordless login: the server displays a short code, the user +approves it on a device where they are already signed in, and your client polls until the +approval lands. Use it for headless or shared setups where you do not want to handle a +password — or when the user account has no password at all (send an empty-string `Pw` for +those in the final exchange). + +## When Quick Connect is the right flow + +- The CLI runs where you cannot (or should not) type a password: CI, SSH sessions, cron. +- You do not want the script to ever see the user's password. +- The server has Quick Connect enabled — otherwise `POST /QuickConnect/Initiate` answers + **401** with "Quick connect is not active on this server" (that 401 is the disable + signal, not an auth failure). + +## The flow + +``` +1. POST /QuickConnect/Initiate # no authentication required + → 200 { "Secret": "", "Code": "123456", "Authenticated": false } + → 401 = feature disabled on this server + +2. Show the Code to the user; on another signed-in client they approve it + (Dashboard or the client prompt). + +3. Poll every ~5 seconds: + GET /QuickConnect/Connect?secret={Secret} # quick-connect state + → QuickConnectResult with Authenticated flipping to true when approved + +4. POST /Users/AuthenticateWithQuickConnect # body: {"Secret": ""} + → 200 AuthenticationResult # SAME capture as password login +``` + +`AuthenticationResult` is identical to the password flow's: capture `User.Id` as +`USER_ID` and `AccessToken` as `TOKEN`, then send the standard +`Authorization: MediaBrowser ... Token="..."` header on every subsequent call. The same +session rules apply — one token per `(DeviceId, user)` pair, re-login revokes the pair's +previous token — so still send a complete `Client`/`Device`/`DeviceId`/`Version` header +with the `AuthenticateWithQuickConnect` call. + +Error paths: `400 "Missing token"` on step 4 when `Secret` is absent; the 401-on-initiate +disable case above; polling forever if the user never approves — bound your loop. + +## Which login flow should my client use? + +| Situation | Flow | +| --- | --- | +| Scripting with a persistent admin credential | API key from Dashboard → API Keys (`Authorization: MediaBrowser Token=""`) | +| Interactive one-user session | `POST /Users/AuthenticateByName` with the full pre-token MediaBrowser header | +| Headless / passwordless / shared device | Quick Connect (this reference) | +| Server with legacy auth disabled and a very old client | Nothing helps — upgrade the client to speak the modern header | + +## Sources + +- https://api.jellyfin.org/openapi/jellyfin-openapi-stable.json — canonical OpenAPI spec (QuickConnect operations: Enabled, Initiate, Connect state, AuthenticateWithQuickConnect; AuthenticationResult schema) +- https://kotlin-sdk.jellyfin.org/guide/authentication.html — official Kotlin SDK authentication guide (Quick Connect cadence ~5s poll, disabled-server 401, empty-password note) +- https://mintlify.wiki/jellyfin/jellyfin/api/authentication/overview — official server docs (token usage after login, session lifetime) +- https://jellyfin.org/docs/general/server/quick-connect — Quick Connect feature overview +- https://gist.github.com/nielsvanvelzen/ea047d9028f676185832e51ffaf12a6f — core-developer authorization guide (device identity and token pairing rules) diff --git a/jellyfin/references/user-scoping-and-errors.md b/jellyfin/references/user-scoping-and-errors.md new file mode 100644 index 0000000..4281b1b --- /dev/null +++ b/jellyfin/references/user-scoping-and-errors.md @@ -0,0 +1,89 @@ +# Jellyfin user scoping — the userId matrix + +The single most common Jellyfin integration failure after authentication: a request that +authenticates fine but 404s or 400s because **which user** the query is about was never +stated. Jellyfin's data model is per-user at the library level; most read endpoints need +you to say whose library you are looking at. + +## Why user id is everywhere + +A Jellyfin library is not one global catalog. Views, playstate, favorites, resume points, +and parental-control visibility all attach to a user. The authentication layer may or may +not imply a user: + +- **User access token** (from `AuthenticateByName`): implies one user. Recent servers fall + back to that user when `userId` is omitted on some endpoints. +- **API key** (from Dashboard → API Keys): implies NO user. `AuthorizationInfo.User` stays + null and the request gets administrator role. Every user-scoped concept must be named + explicitly with a `userId` parameter — including `/UserViews`, which is meaningless + without one. + +The bundled CLI always sends `userId` explicitly on user-scoped commands regardless of +which credential type it holds. That is the cross-version-safe baseline. + +## The userId requirement matrix + +| Endpoint | User token, userId omitted | API key | +| --- | --- | --- | +| `GET /Items` | **400** — body `userId is required` (servers enforce `if (!isApiKey && user is null) return BadRequest("userId is required")`) | Optional — omitted means an unrestricted, userless view; pass it anyway for `UserData` in DTOs | +| `GET /Items/{itemId}` | Defaults to the token's user; `userId` supplies whose `UserData` embeds | Pass explicitly for user data | +| `GET /UserViews` | Parameter accepted; pass it | REQUIRED to be meaningful | +| `GET /Shows/{seriesId}/Seasons` / `Episodes` | ≥10.9 falls back to token user; ≤10.8 crashes on omission — always pass | REQUIRED | +| `GET /Shows/NextUp` | Same version split as above — always pass | REQUIRED | +| `GET /Items/Latest` | `userId` scopes "recently added for whom"; always pass | REQUIRED for meaningful results | +| `GET /Search/Hints` | Optional — "omit to search all" | Optional | +| `GET /Users/Me` | Works (returns token's user) | **400** `Token is not owned by a user.` | +| `GET /System/Info`, `/System/Info/Public`, `/Users` | No user concept | Fine | + +Note the deliberate trap in `/Users/Me`: it is the natural "who am I" endpoint for a user +token and returns exactly the id you need — but with an API key it is a 400, by design, +because an API key is nobody. + +## 400 vs 404: absent vs invalid userId on /Items + +Two distinct failure signatures, enforced in this order by the items controller +(verified at master and v10.10.7): + +1. The user lookup runs first: a supplied-but-nonexistent `userId` throws + `ResourceNotFoundException` → mapped to **404** (`Error processing request.` body). +2. Then the guard: an ABSENT `userId` on non-API-key auth returns **400** with the literal + string body `userId is required` (not the middleware's generic text). + +So: `400 userId is required` = you forgot the parameter; `404` = the parameter names a user +that does not exist. Mock both distinctly. + +## Finding a user id without logging in as one + +1. **`GET /Users`** (any valid token): array of `UserDto` with `Name` and `Id`. The + administrator-flavored listing — API keys see everyone. +2. **`GET /Users/Public`** (no auth): only users flagged visible on login screens. +3. **After `AuthenticateByName`**: the response's `User.Id` is the documented primary. +4. **With a user token**: `GET /Users/Me`. + +```bash +# Discover user ids with an API key +curl -s -H 'Authorization: MediaBrowser Token="YOUR_API_KEY"' \ + "http://localhost:8096/Users" | jq -r '.[] | [.Name, .Id] | @tsv' +# → alice 6eec632a-ff0d-4d09-aad0-bf9e90b14bc6 +``` + +## Scoping errors look like 404s + +The confusion this reference exists for: `GET /Items` (or `/Shows/NextUp`) called without a +`userId` under a context where one is required does not answer "you forgot the user" on +every endpoint and version — older servers crash (NextUp ≤10.8: 500 from an empty-Guid +`ArgumentException`), and user-token fallbacks silently change results. Symptoms cluster as +"endpoint exists but returns 400/404/empty" even though the token is perfectly valid. The +fix is uniform: resolve the user id once, pass it explicitly on every user-scoped call. + +The bundled CLI mirrors that baseline: `recent`, `next-up`, and `item` require +`JELLYFIN_USER_ID` or `--user-id` before any network call, refuse to guess an +administrator, and dry-run previews show the `userId` that would have been sent. + +## Sources + +- https://api.jellyfin.org/ — official Jellyfin API reference (ReDoc), version 12.0.0 stable +- https://api.jellyfin.org/openapi/jellyfin-openapi-stable.json — canonical OpenAPI spec (`/Users/Me` 400 "Token is not owned by a user.", `userId` parameter descriptions across /Items, /Items/Latest, /Shows/*, /Search/Hints) +- https://github.com/jellyfin/jellyfin — server source: `ItemsController.cs` (userId-required guard and 400/404 ordering), `RequestHelpers.cs` (GetUserId token fallback), `TvShowsController.cs` (NextUp userId version history), `AuthorizationContext.cs` (API key ⇒ User null + admin role) +- https://mintlify.wiki/jellyfin/jellyfin/api/authentication/overview — official server docs (API key vs user token semantics) +- https://gist.github.com/nielsvanvelzen/ea047d9028f676185832e51ffaf12a6f — core-developer authorization guide (API-key identity behavior) diff --git a/jellyfin/references/worked-recipes.md b/jellyfin/references/worked-recipes.md new file mode 100644 index 0000000..55b09e2 --- /dev/null +++ b/jellyfin/references/worked-recipes.md @@ -0,0 +1,156 @@ +# Jellyfin worked recipes + +Multi-step workflows against a real server. Every stage consumes the previous stage's +output; field names and types match the endpoint catalog. Constants: `BASE` = +`http://:8096` (official default port). Wire examples use PascalCase properties; your +client should normalize casing (see the gotchas guide). + +## Recipe 1 — Log in, capture identity, list what's new for that user + +The full pre-token → token → user-scoped-read sequence: + +``` +1. (probe) GET /System/Info/Public # no auth → ServerName, Version +2. POST /Users/AuthenticateByName + Authorization: MediaBrowser Client="my-cli", Device="terminal", + DeviceId="dev-1", Version="1.0.0" # COMPLETE header, no Token yet + {"Username": "alice", "Pw": "secret"} + → AuthenticationResult +3. USER_ID = .User.Id TOKEN = .AccessToken # hyphenated UUID; both strings +4. Subsequent calls: + Authorization: MediaBrowser Client="my-cli", Device="terminal", + DeviceId="dev-1", Version="1.0.0", Token="{TOKEN}" +5. GET /Items/Latest?userId={USER_ID}&limit=20&includeItemTypes=Movie,Series,Episode + → 200 BARE ARRAY of BaseItemDto (NOT a wrapper) # shape branch here +``` + +One-liner with curl + jq: + +```bash +AUTH='Authorization: MediaBrowser Client="my-cli", Device="terminal", DeviceId="dev-1", Version="1.0.0"' +res=$(curl -s -X POST -H "$AUTH" -H 'Content-Type: application/json' \ + -d '{"Username":"alice","Pw":"secret"}' "$BASE/Users/AuthenticateByName") +user_id=$(jq -r '.User.Id' <<<"$res") +token=$(jq -r '.AccessToken' <<<"$res") +curl -s -H "Authorization: MediaBrowser Token=\"$token\"" \ + "$BASE/Items/Latest?userId=$user_id&limit=20" | jq -r '.[] | .Name' +``` + +Failure branches: step 2 without the full MediaBrowser header → 400 `Error processing +request.`; wrong password → 401; server restarting → 503 + `Retry-After` (retry). + +With the bundled CLI, steps 2–5 collapse to env vars: export `JELLYFIN_URL`, +`JELLYFIN_API_KEY` (or `JELLYFIN_TOKEN`), `JELLYFIN_USER_ID`; then +`scripts/jellyfin recent --json`. The login path itself is available as +`scripts/jellyfin login --username alice` (reads `JELLYFIN_PASSWORD` interactively or via +`--password`/`--password-stdin`), which prints the captured `user_id`/`access_token` for +exporting. + +## Recipe 2 — Libraries → paged browse of one collection + +``` +1. GET /UserViews?userId={USER_ID} # token-auth + → {Items: [{Id, Name, CollectionType: "movies"|"tvshows"|..., ...}], TotalRecordCount} +2. VIEW_ID = .Items[] | select(.Name == "Movies") | .Id +3. Page loop (both guards — counts can go stale mid-scan): + start = 0; PAGE = 100 + loop: + GET /Items?userId={USER_ID}&parentId={VIEW_ID}&recursive=true + &includeItemTypes=Movie&sortBy=SortName&sortOrder=Ascending + &startIndex={start}&limit={PAGE} + → {Items: [...], TotalRecordCount: N, StartIndex: start} + emit .Items; start += len(Items) + stop when len(Items) == 0 OR start >= TotalRecordCount +``` + +Why `/UserViews` and not `/Library/MediaFolders`: MediaFolders is admin-elevated +(RequiresElevation) — a non-admin token gets 403. UserViews is the per-user library list +for any token. + +```bash +scripts/jellyfin libraries --json | jq -r '.libraries[] | [.name, .id] | @tsv' +scripts/jellyfin browse --library-id "$VIEW_ID" --type Movie --limit 100 --start-index 0 --json \ + | jq -r '.items[] | [.name, .year] | @tsv' +``` + +## Recipe 3 — Search → seasons → episodes walk + +``` +1. FIND SERIES: + GET /Search/Hints?searchTerm=breaking&includeItemTypes=Series&limit=20&userId={USER_ID} + → {SearchHints: [{Id (read .Id; .ItemId is the deprecated twin), Name, Type, ...}]} + Fuller DTOs alternative: + GET /Items?userId=..&recursive=true&includeItemTypes=Series&searchTerm=..&fields=Overview +2. SEASONS: + GET /Shows/{SERIES_ID}/Seasons?userId={USER_ID}&fields=Overview + → {Items: [{Id, Name, IndexNumber (0 = specials), Type: "Season"}]} +3. EPISODES per season (prefer seasonId over numeric season — numbers shift): + GET /Shows/{seriesId}/Episodes?userId={USER_ID}&seasonId={SEASON_ID}&sortBy=AiredEpisodeOrder + → {Items: [{Name, IndexNumber, ParentIndexNumber, UserData.PlayedPercentage, RunTimeTicks}]} +4. Whole-series flat option (skip seasons): + GET /Shows/{seriesId}/Episodes?userId={USER_ID}&startIndex=0&limit=100 +5. NEXT UP for one series: + GET /Shows/NextUp?userId={USER_ID}&seriesId={SERIES_ID}&limit=10 +``` + +```bash +scripts/jellyfin search --query "breaking bad" --type Series --json | jq -r '.results[0].id' +scripts/jellyfin item --id "$SERIES_ID" --user-id "$USER_ID" --json +scripts/jellyfin next-up --user-id "$USER_ID" --limit 10 --json | jq -r '.items[] | .series' +``` + +## Recipe 4 — server health → stats → next-up evening plan + +``` +1. GET /System/Info/Public # no auth: is the server up? which version? +2. GET /System/Info # token: OperatingSystem, Version (full) +3. GET /Items/Counts?userId={USER_ID} # MovieCount, SeriesCount, EpisodeCount, SongCount +4. GET /Shows/NextUp?userId={USER_ID}&limit=5 # always pass userId (NextUp ≤10.8 crashes without) + → {Items: [Episode...], TotalRecordCount} +``` + +```bash +scripts/jellyfin info --json +scripts/jellyfin stats --json | jq -r '.movies, .episodes' +scripts/jellyfin next-up --limit 5 --json +``` + +## Recipe 5 — find an item id → full details → image URL + +``` +1. scripts/jellyfin search --query "dune" --type Movie --json → .results[0].id +2. GET /Items/{ITEM_ID}?userId={USER_ID} + → BaseItemDto: Overview, Genres, CommunityRating, OfficialRating, RunTimeTicks, + ProductionYear, ImageTags.Primary, BackdropImageTags[], ProviderIds +3. Image URL: + {BASE}/Items/{ITEM_ID}/Images/Primary?maxWidth=300&tag={ImageTags.Primary} + # 404 here means "no such image" — fall back to Thumb/Backdrop, not an error +``` + +`RunTimeTicks` are 100-nanosecond ticks (divide by 600,000,000 for minutes). + +## Cross-version-safe baseline (derive your own recipes from these rules) + +1. Speak modern auth (`Authorization: MediaBrowser ...`); treat `X-Emby-Token` as a compat + fallback for old servers only. +2. Send a complete pre-token header everywhere — zero cost, avoids the 400-on-login trap. +3. Always send explicit `userId` on `/Items*`, `/UserViews`, `/Shows/*`, `/Items/Latest`. +4. Resolve identity once: `USER_ID = AuthenticationResult.User.Id` (fallback `/Users/Me` + with a user token; NEVER with an API key). +5. Honor 503 + `Retry-After` on any endpoint; never retry 400-class. +6. Probe `GET /System/Info/Public` first; branch on semver (NextUp userId <10.9; legacy + header availability ≥10.11 config). +7. Branch on shape: wrapper object vs bare array (`/Items/Latest`) vs `SearchHints` key. +8. DeviceId per profile — one token per `(DeviceId, user)` pair; re-login revokes the pair's + previous token. + +## Sources + +Endpoint semantics, pagination, and auth sequencing inherit citations from the auth, +endpoint, and gotchas references (all fetched this session): +https://api.jellyfin.org/ · +https://api.jellyfin.org/openapi/jellyfin-openapi-stable.json · +https://gist.github.com/nielsvanvelzen/ea047d9028f676185832e51ffaf12a6f · +https://mintlify.wiki/jellyfin/jellyfin/api/authentication/overview · +https://kotlin-sdk.jellyfin.org/guide/authentication.html · +https://github.com/jellyfin/jellyfin (ItemsController.cs, TvShowsController.cs, UserViewsController.cs, LibraryController.cs, SessionManager.cs) diff --git a/jellyfin/scripts/jellyfin b/jellyfin/scripts/jellyfin index 662da94..66a1a54 100755 --- a/jellyfin/scripts/jellyfin +++ b/jellyfin/scripts/jellyfin @@ -2,12 +2,18 @@ """jellyfin — Jellyfin media server from the terminal. Query recently added media, search your library, browse by collection, -and check server status. Requires JELLYFIN_URL and JELLYFIN_API_KEY. +walk series seasons and episodes, and check server status. Authenticates +with an API key (JELLYFIN_API_KEY), a user access token (JELLYFIN_TOKEN), +or a named login (`login` subcommand). Requires JELLYFIN_URL for a +non-default server. """ import argparse +import getpass +import hashlib import json import os +import socket import sys import warnings from typing import Any, Dict @@ -17,13 +23,45 @@ warnings.simplefilter("ignore") import requests DEFAULT_SERVER = "http://localhost:8096" +DEFAULT_PORT = "8096" ENV_URL = os.getenv("JELLYFIN_URL", DEFAULT_SERVER) ENV_KEY = os.getenv("JELLYFIN_API_KEY", "") +ENV_TOKEN = os.getenv("JELLYFIN_TOKEN", "") ENV_USER_ID = os.getenv("JELLYFIN_USER_ID", "") +ENV_DEVICE_ID = os.getenv("JELLYFIN_DEVICE_ID", "") +ENV_PASSWORD = os.getenv("JELLYFIN_PASSWORD", "") + +CLIENT_NAME = "jellyfin-cli" +CLIENT_VERSION = "1.0.0" GLOBAL_FLAGS: Dict[str, Any] = {"json": False, "dry_run": False} +def default_device_id() -> str: + """Stable per-machine device identifier (Jellyfin allows one token per device id).""" + seed = f"{socket.gethostname()}:{getattr(os, 'getuid', lambda: 0)()}" + return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:16] + + +def build_authorization_header(device_id, token=""): + """Build the Jellyfin MediaBrowser Authorization header. + + The Client/Device/DeviceId/Version quartet is REQUIRED by the server on + POST /Users/AuthenticateByName even though no token exists yet — the login + call itself must be sent with this pre-token header. After login the + access token (or an API key) rides the same header in a Token= parameter. + """ + parts = [ + f'Client="{CLIENT_NAME}"', + f'Device="{socket.gethostname()}"', + f'DeviceId="{device_id}"', + f'Version="{CLIENT_VERSION}"', + ] + if token: + parts.append(f'Token="{token}"') + return "MediaBrowser " + ", ".join(parts) + + def die(msg, exit_code=1): print(f"Error: {msg}", file=sys.stderr) sys.exit(exit_code) @@ -57,27 +95,49 @@ def _preparse_global_flags(argv): class JellyfinClient: - """Jellyfin API client (v10.8+ compatible).""" + """Jellyfin API client (10.8+ compatible, modern Authorization header).""" - def __init__(self, url="", key="", dry_run=False): + def __init__(self, url="", key="", token="", device_id="", dry_run=False): self.url = (url or ENV_URL).rstrip("/") self.key = key or ENV_KEY + self.token = token or ENV_TOKEN + self.device_id = device_id or ENV_DEVICE_ID or default_device_id() self.dry_run = dry_run + if not self.key and self.token: + # User access tokens are only meaningful together with their user id; + # user-scoped commands resolve that separately via --user-id. + pass + + def _headers(self, with_token=True): + headers = { + "Accept": "application/json", + "Authorization": build_authorization_header( + self.device_id, token=(self.key or self.token) if with_token else "") + } + if with_token and self.key: + headers["X-Emby-Token"] = self.key + return headers def _get(self, path, params=None): url = f"{self.url}{path}" if self.dry_run: return {"dry_run": True, "url": url, "params": params} - if not self.key: - die("JELLYFIN_API_KEY not set. Generate one in Dashboard → API Keys.") + if not (self.key or self.token): + die("JELLYFIN_API_KEY (or JELLYFIN_TOKEN) not set. " + "Generate an API key in Dashboard → API Keys, or run `login`.") try: - resp = requests.get(url, params=params, - headers={"X-Emby-Token": self.key, "Accept": "application/json"}, - timeout=30) + resp = requests.get(url, params=params, headers=self._headers(), timeout=30) except requests.ConnectionError as e: die(f"Cannot connect to {self.url}: {e}") + if resp.status_code == 503: + retry_after = resp.headers.get("Retry-After", "?") + message = resp.headers.get("Message", "server starting or unavailable") + die(f"Server unavailable (503): {message}. Retry after {retry_after}s.") if resp.status_code == 401: - die("Auth failed (401). Check JELLYFIN_API_KEY.") + die("Auth failed (401). Check JELLYFIN_API_KEY/JELLYFIN_TOKEN; on servers with " + "legacy auth disabled only the Authorization: MediaBrowser header works.") + if resp.status_code == 400 and "userId is required" in resp.text: + die("Server requires a userId on this request. Pass --user-id or set JELLYFIN_USER_ID.") if resp.status_code >= 400: try: detail = resp.json() @@ -86,12 +146,40 @@ class JellyfinClient: die(f"API error ({resp.status_code}): {detail}") return resp.json() + def _post(self, path, payload=None, send_auth_header=True): + url = f"{self.url}{path}" + if self.dry_run: + return {"dry_run": True, "url": url, "payload": payload} + try: + resp = requests.post(url, json=payload, headers=self._headers(send_auth_header), + timeout=30) + except requests.ConnectionError as e: + die(f"Cannot connect to {self.url}: {e}") + if resp.status_code >= 400: + try: + detail = resp.json() + except Exception: + detail = resp.text[:200] + die(f"API error ({resp.status_code}): {detail}") + return resp.json() + + def authenticate_by_name(self, username, password): + """POST /Users/AuthenticateByName — requires the pre-token MediaBrowser header.""" + return self._post("/Users/AuthenticateByName", + payload={"Username": username, "Pw": password}) + + def get_public_info(self): + return self._get("/System/Info/Public") + def get_info(self): return self._get("/System/Info") def get_users(self): return self._get("/Users") + def get_user_views(self, user_id): + return self._get("/UserViews", params={"userId": user_id}) + def get_recent(self, user_id, limit=10, include_types=None): params = {"userId": user_id, "fields": "DateCreated"} if include_types: @@ -99,9 +187,11 @@ class JellyfinClient: params["limit"] = limit return self._get("/Items/Latest", params=params) - def get_next_up(self, user_id, limit=10): - return self._get(f"/Shows/NextUp", - params={"userId": user_id, "limit": limit}) + def get_next_up(self, user_id, limit=10, series_id=None): + params = {"userId": user_id, "limit": limit} + if series_id: + params["seriesId"] = series_id + return self._get("/Shows/NextUp", params=params) def search(self, query, limit=20, include_types=None): params = {"searchTerm": query, "limit": limit, "recursive": True} @@ -113,10 +203,12 @@ class JellyfinClient: return self._get("/Library/MediaFolders") def get_items(self, parent_id, types=None, limit=50, sort_by="SortName", - sort_order="Ascending", start_index=0): + sort_order="Ascending", start_index=0, user_id=None): params = {"parentId": parent_id, "limit": limit, "sortBy": sort_by, "sortOrder": sort_order, "startIndex": start_index, "recursive": True} + if user_id: + params["userId"] = user_id if types: params["includeItemTypes"] = ",".join(types) return self._get("/Items", params=params) @@ -129,7 +221,7 @@ class JellyfinClient: def get_episodes(self, series_id, season_id, user_id): return self._get(f"/Shows/{series_id}/Episodes", - params={"seasonId": season_id, "userId": user_id}) + params={"userId": user_id, "seasonId": season_id}) def get_stats(self): return self._get("/Items/Counts") @@ -160,6 +252,153 @@ def normalize_item(item): return {key: value for key, value in fields.items() if value is not None and value != ""} +def cmd_login(client, args): + p = argparse.ArgumentParser(prog="jellyfin login") + p.add_argument("--server", help="JELLYFIN_URL override, e.g. http://host:8096") + p.add_argument("--username", "-u", required=True) + password_group = p.add_mutually_exclusive_group() + password_group.add_argument("--password", help="Account password (prefer --password-stdin)") + password_group.add_argument("--password-stdin", action="store_true", + help="Read the password from stdin") + password_group.add_argument("--prompt", action="store_true", + help="Prompt for the password interactively") + p.add_argument("--device-id", help="Device id to bind the session to " + "(default: derived from hostname)") + parsed, _ = p.parse_known_args(args) + + if parsed.password_stdin: + password = sys.stdin.readline().rstrip("\n") + elif parsed.prompt: + password = getpass.getpass(f"Jellyfin password for {parsed.username}: ") + else: + password = parsed.password or ENV_PASSWORD + + target_url = (parsed.server or client.url).rstrip("/") + device_id = parsed.device_id or client.device_id + auth_header = build_authorization_header(device_id) # pre-token: no Token segment + path = "/Users/AuthenticateByName" + payload = {"Username": parsed.username, "Pw": password} + + if client.dry_run or GLOBAL_FLAGS.get("dry_run", False): + return emit("[dry-run] POST /Users/AuthenticateByName " + f"Authorization: {auth_header} " + f"payload: {json.dumps({'Username': parsed.username, 'Pw': '***'})}", { + "dry_run": True, + "path": path, + "server": target_url, + "username": parsed.username, + "authorization_header": auth_header, + "pre_token_header": True, + "notes": "POST /Users/AuthenticateByName requires this MediaBrowser header " + "BEFORE any access token exists; the returned AccessToken is then " + "sent via Token= in subsequent Authorization headers.", + }) + + if password is None: + die("login needs --password, --password-stdin, --prompt, or JELLYFIN_PASSWORD.") + if not password and not ENV_PASSWORD: + # Empty passwords are valid for passwordless accounts only when intentional. + pass + + login_client = JellyfinClient(url=target_url, device_id=device_id) + try: + resp = requests.post( + f"{target_url}{path}", + json=payload, + headers={"Content-Type": "application/json", + "Authorization": auth_header, + "Accept": "application/json"}, + timeout=30, + ) + except requests.ConnectionError as e: + die(f"Cannot connect to {target_url}: {e}") + if resp.status_code == 400: + die("Login rejected (400). The server requires a complete " + 'Authorization: MediaBrowser Client=..., Device=..., DeviceId=..., Version=... ' + "header — the client sends one; a 400 with 'Error processing request.' usually " + "means the header did not reach the server (proxy stripping) or the username " + "does not exist.") + if resp.status_code == 401: + die("Login failed (401): invalid username or password.") + if resp.status_code == 403: + die("Login rejected (403): user disabled, device not allowed, or session cap reached.") + if resp.status_code >= 400: + die(f"Login failed ({resp.status_code}): {resp.text[:200]}") + + result = resp.json() + user = result.get("User", {}) + session = { + "server": target_url, + "user": user.get("Name"), + "user_id": user.get("Id"), + "access_token": result.get("AccessToken"), + "device_id": device_id, + "authorization_header": build_authorization_header( + device_id, token=result.get("AccessToken") or ""), + } + emit( + f"Logged in as {session['user']} (user id {session['user_id']})\n" + f" export JELLYFIN_URL=\"{target_url}\"\n" + f" export JELLYFIN_TOKEN=\"{session['access_token']}\"\n" + f" export JELLYFIN_USER_ID=\"{session['user_id']}\"", + session, + ) + + +def cmd_seasons(client, args): + p = argparse.ArgumentParser(prog="jellyfin seasons") + p.add_argument("--series-id", required=True) + p.add_argument("--user-id", default=ENV_USER_ID) + parsed, _ = p.parse_known_args(args) + + params = {"userId": parsed.user_id or None} + path = f"/Shows/{parsed.series_id}/Seasons" + if client.dry_run: + return emit(f"[dry-run] GET {path} " + json.dumps(params), { + "dry_run": True, "path": path, "params": params, + }) + if not parsed.user_id: + die("seasons requires --user-id or JELLYFIN_USER_ID.") + + data = client.get_seasons(parsed.series_id, parsed.user_id) or {} + items = [normalize_item(item) for item in data.get("Items", [])] + lines = [f" {item.get('name', '?')} (season {item.get('season_number', '?')})" + for item in items] + emit("Seasons:\n" + "\n".join(lines) if lines else "No seasons found.", { + "items": items, "total_record_count": data.get("TotalRecordCount", 0), + }) + + +def cmd_episodes(client, args): + p = argparse.ArgumentParser(prog="jellyfin episodes") + p.add_argument("--series-id", required=True) + p.add_argument("--season-id") + p.add_argument("--user-id", default=ENV_USER_ID) + p.add_argument("--limit", type=int, default=50) + p.add_argument("--start-index", type=int, default=0) + parsed, _ = p.parse_known_args(args) + + params = {"userId": parsed.user_id or None, "limit": parsed.limit, + "startIndex": parsed.start_index} + if parsed.season_id: + params["seasonId"] = parsed.season_id + path = f"/Shows/{parsed.series_id}/Episodes" + if client.dry_run: + return emit(f"[dry-run] GET {path} " + json.dumps(params), { + "dry_run": True, "path": path, "params": params, + }) + if not parsed.user_id: + die("episodes requires --user-id or JELLYFIN_USER_ID.") + + data = client.get_episodes(parsed.series_id, parsed.season_id, parsed.user_id) or {} + items = [normalize_item(item) for item in data.get("Items", [])] + lines = [f" E{item.get('episode_number', '?'):>3} {item.get('name', '?')}" + for item in items] + emit("Episodes:\n" + "\n".join(lines) if lines else "No episodes found.", { + "items": items, "total_record_count": data.get("TotalRecordCount", 0), + }) + + def cmd_info(client, args): if client.dry_run: return emit("[dry-run] GET /System/Info; GET /Users", { @@ -249,7 +488,8 @@ def cmd_search(client, args): series = h.get("Series", "") series_str = f" [{series}]" if series else "" lines.append(f" {name:45}{series_str} ({year}) [{itype}]") - out.append({"name": name, "type": itype, "year": year, "series": series, "id": h.get("ItemId")}) + item_id = h.get("Id") or h.get("ItemId") # ItemId is the deprecated twin on old servers + out.append({"name": name, "type": itype, "year": year, "series": series, "id": item_id}) emit(f"{len(hints)} result(s):\n" + "\n".join(lines), {"results": out}) @@ -257,9 +497,12 @@ def cmd_next_up(client, args): p = argparse.ArgumentParser(prog="jellyfin next-up") p.add_argument("--user-id", default=ENV_USER_ID) p.add_argument("--limit", type=int, default=10) + p.add_argument("--series-id") parsed, _ = p.parse_known_args(args) params = {"userId": parsed.user_id or None, "limit": parsed.limit} + if parsed.series_id: + params["seriesId"] = parsed.series_id if client.dry_run: return emit("[dry-run] GET /Shows/NextUp " + json.dumps(params), { "dry_run": True, "path": "/Shows/NextUp", "params": params, @@ -267,7 +510,8 @@ def cmd_next_up(client, args): if not parsed.user_id: die("next-up requires --user-id or JELLYFIN_USER_ID.") - data = client.get_next_up(parsed.user_id, limit=parsed.limit) or {} + data = client.get_next_up(parsed.user_id, limit=parsed.limit, + series_id=parsed.series_id) or {} items = [normalize_item(item) for item in data.get("Items", [])] lines = [f" {item.get('name', '?')} [{item.get('type', '?')}]" for item in items] emit("Next up:\n" + "\n".join(lines) if lines else "No next-up episodes.", { @@ -300,6 +544,8 @@ def cmd_browse(client, args): p.add_argument("--type") p.add_argument("--limit", type=int, default=50) p.add_argument("--start-index", type=int, default=0) + p.add_argument("--user-id", default=ENV_USER_ID, + help="User id (sent when present; servers using non-API-key auth require it)") parsed, _ = p.parse_known_args(args) types = parsed.type.split(",") if parsed.type else None @@ -307,6 +553,8 @@ def cmd_browse(client, args): "parentId": parsed.library_id, "limit": parsed.limit, "sortBy": "SortName", "sortOrder": "Ascending", "startIndex": parsed.start_index, "recursive": True, } + if parsed.user_id: + params["userId"] = parsed.user_id if types: params["includeItemTypes"] = ",".join(types) if client.dry_run: @@ -315,7 +563,8 @@ def cmd_browse(client, args): }) data = client.get_items(parsed.library_id, types=types, limit=parsed.limit, - start_index=parsed.start_index) or {} + start_index=parsed.start_index, + user_id=parsed.user_id or None) or {} items = [normalize_item(item) for item in data.get("Items", [])] lines = [f" {item.get('name', '?')} [{item.get('type', '?')}]" for item in items] emit("Browse results:\n" + "\n".join(lines) if lines else "No items found.", { @@ -370,6 +619,14 @@ def main(): parser.add_argument("--json", action="store_true", help="Output machine-readable JSON") parser.add_argument("--dry-run", action="store_true", help="Preview API requests without network access") sub = parser.add_subparsers(dest="command") + lg = sub.add_parser("login", help="Authenticate a user by name", description="Log in to Jellyfin with a username and password, demonstrating the pre-token MediaBrowser Authorization header, and print the session values to export.", epilog="Example: jellyfin login --username alice --prompt") + lg.add_argument("--server", help="JELLYFIN_URL override, e.g. http://host:8096") + lg.add_argument("--username", "-u", required=True, help="Jellyfin username") + lg_password_group = lg.add_mutually_exclusive_group() + lg_password_group.add_argument("--password", help="Account password (prefer --password-stdin)") + lg_password_group.add_argument("--password-stdin", action="store_true", help="Read the password from stdin") + lg_password_group.add_argument("--prompt", action="store_true", help="Prompt for the password interactively") + lg.add_argument("--device-id", help="Device id to bind the session to (default: derived from hostname)") sub.add_parser("info", help="Server info", description="Show Jellyfin server details.", epilog="Example: jellyfin info") re = sub.add_parser("recent", help="Recently added", description="Show recently added movies or episodes.", epilog="Example: jellyfin recent --movies --limit 5") re.add_argument("--limit", type=int, default=10, help="Maximum items to return (default: 10)") @@ -384,14 +641,25 @@ def main(): nu = sub.add_parser("next-up", help="Next unwatched episodes", description="Show the next unwatched episodes for a Jellyfin user.", epilog="Example: jellyfin next-up --user-id USER_ID --limit 5") nu.add_argument("--user-id", help="Jellyfin user ID (default: JELLYFIN_USER_ID)") nu.add_argument("--limit", type=int, default=10, help="Maximum episodes to return (default: 10)") + nu.add_argument("--series-id", help="Limit next-up to one series") it = sub.add_parser("item", help="Show item details", description="Show metadata for one Jellyfin library item.", epilog="Example: jellyfin item --id ITEM_ID --user-id USER_ID") it.add_argument("--id", required=True, help="Jellyfin item ID") it.add_argument("--user-id", help="Jellyfin user ID (default: JELLYFIN_USER_ID)") + sn = sub.add_parser("seasons", help="List seasons of a series", description="List the seasons of one Jellyfin series.", epilog="Example: jellyfin seasons --series-id SERIES_ID --user-id USER_ID") + sn.add_argument("--series-id", required=True, help="Jellyfin series ID") + sn.add_argument("--user-id", help="Jellyfin user ID (default: JELLYFIN_USER_ID)") + ep = sub.add_parser("episodes", help="List episodes of a series or season", description="List episodes for a Jellyfin series, optionally scoped to one season.", epilog="Example: jellyfin episodes --series-id SERIES_ID --season-id SEASON_ID --user-id USER_ID") + ep.add_argument("--series-id", required=True, help="Jellyfin series ID") + ep.add_argument("--season-id", help="Jellyfin season ID (omit for every episode of the series)") + ep.add_argument("--user-id", help="Jellyfin user ID (default: JELLYFIN_USER_ID)") + ep.add_argument("--limit", type=int, default=50, help="Maximum episodes to return (default: 50)") + ep.add_argument("--start-index", type=int, default=0, help="Zero-based result offset (default: 0)") br = sub.add_parser("browse", help="Browse a library", description="List items in a Jellyfin media library.", epilog="Example: jellyfin browse --library-id LIBRARY_ID --type Movie --limit 20") br.add_argument("--library-id", required=True, help="Jellyfin library ID") br.add_argument("--type", help="Comma-separated item types, such as Movie,Series") br.add_argument("--limit", type=int, default=50, help="Maximum items to return (default: 50)") br.add_argument("--start-index", type=int, default=0, help="Zero-based result offset (default: 0)") + br.add_argument("--user-id", help="User id sent on the query (servers using non-API-key auth require it)") sub.add_parser("libraries", help="List libraries", description="List configured media libraries.", epilog="Example: jellyfin libraries") sub.add_parser("stats", help="Library statistics", description="Show media library item counts.", epilog="Example: jellyfin stats") @@ -403,9 +671,10 @@ def main(): client = JellyfinClient(dry_run=GLOBAL_FLAGS.get("dry_run", False)) cmd_map = { - "info": cmd_info, "recent": cmd_recent, "search": cmd_search, - "next-up": cmd_next_up, "item": cmd_item, "browse": cmd_browse, - "libraries": cmd_libraries, "stats": cmd_stats, + "login": cmd_login, "info": cmd_info, "recent": cmd_recent, "search": cmd_search, + "next-up": cmd_next_up, "item": cmd_item, "seasons": cmd_seasons, + "episodes": cmd_episodes, "browse": cmd_browse, "libraries": cmd_libraries, + "stats": cmd_stats, } handler = cmd_map.get(args.command) if not handler: diff --git a/jellyfin/scripts/test_jellyfin_cli.py b/jellyfin/scripts/test_jellyfin_cli.py index fd6fcca..fadab56 100644 --- a/jellyfin/scripts/test_jellyfin_cli.py +++ b/jellyfin/scripts/test_jellyfin_cli.py @@ -3,9 +3,12 @@ import importlib.machinery import importlib.util import io import json +import os import pathlib import subprocess +import tempfile import unittest +from unittest.mock import Mock, patch SCRIPT = pathlib.Path(__file__).resolve().parent / "jellyfin" @@ -15,6 +18,27 @@ jellyfin_cli = importlib.util.module_from_spec(SPEC) LOADER.exec_module(jellyfin_cli) +def clean_env(): + env = os.environ.copy() + for var in ("JELLYFIN_URL", "JELLYFIN_API_KEY", "JELLYFIN_TOKEN", + "JELLYFIN_USER_ID", "JELLYFIN_DEVICE_ID", "JELLYFIN_PASSWORD"): + env.pop(var, None) + return env + + +class FakeResponse: + def __init__(self, status_code=200, json_body=None, text="", headers=None): + self.status_code = status_code + self._json = json_body + self.text = text or (json.dumps(json_body) if json_body is not None else "") + self.headers = headers or {} + + def json(self): + if self._json is None: + raise ValueError("no json") + return self._json + + class FakeClient: def __init__(self, libraries=None, dry_run=False): self.dry_run = dry_run @@ -35,9 +59,11 @@ class NavigationFakeClient: self.next_up_calls = [] self.item_calls = [] self.items_calls = [] + self.seasons_calls = [] + self.episodes_calls = [] - def get_next_up(self, user_id, limit=10): - self.next_up_calls.append((user_id, limit)) + def get_next_up(self, user_id, limit=10, series_id=None): + self.next_up_calls.append((user_id, limit, series_id)) return { "Items": [{"Name": "The Signal", "Type": "Episode", "Id": "episode-1", "SeriesName": "Voyagers", "IndexNumber": 4}], @@ -52,8 +78,9 @@ class NavigationFakeClient: "Overview": "A message arrives."} def get_items(self, parent_id, types=None, limit=50, sort_by="SortName", - sort_order="Ascending", start_index=0): - self.items_calls.append((parent_id, types, limit, sort_by, sort_order, start_index)) + sort_order="Ascending", start_index=0, user_id=None): + self.items_calls.append((parent_id, types, limit, sort_by, sort_order, + start_index, user_id)) return { "Items": [{"Name": "Arrival", "Type": "Movie", "Id": "movie-1", "ProductionYear": 2016}], @@ -61,6 +88,18 @@ class NavigationFakeClient: "TotalRecordCount": 1, } + def get_seasons(self, series_id, user_id): + self.seasons_calls.append((series_id, user_id)) + return {"Items": [{"Name": "Season 1", "Type": "Season", "Id": "season-1", + "ParentIndexNumber": 1}], + "TotalRecordCount": 1} + + def get_episodes(self, series_id, season_id, user_id): + self.episodes_calls.append((series_id, season_id, user_id)) + return {"Items": [{"Name": "The Signal", "Type": "Episode", "Id": "episode-1", + "ParentIndexNumber": 1, "IndexNumber": 4}], + "TotalRecordCount": 1} + class JellyfinCliTests(unittest.TestCase): def setUp(self): @@ -126,18 +165,31 @@ class JellyfinCliTests(unittest.TestCase): client = jellyfin_cli.JellyfinClient() client._get = lambda path, params=None: calls.append((path, params)) or {} - client.get_next_up("user-1", limit=3) + client.get_next_up("user-1", limit=3, series_id="series-1") client.get_item("item-1", "user-1") - client.get_items("library-1", types=["Movie", "Series"], limit=4, start_index=2) + client.get_items("library-1", types=["Movie", "Series"], limit=4, start_index=2, + user_id="user-1") self.assertEqual(calls, [ - ("/Shows/NextUp", {"userId": "user-1", "limit": 3}), + ("/Shows/NextUp", {"userId": "user-1", "limit": 3, "seriesId": "series-1"}), ("/Items/item-1", {"userId": "user-1"}), ("/Items", {"parentId": "library-1", "limit": 4, "sortBy": "SortName", "sortOrder": "Ascending", "startIndex": 2, "recursive": True, + "userId": "user-1", "includeItemTypes": "Movie,Series"}), ]) + def test_authorization_header_builds_media_browser_scheme(self): + header = jellyfin_cli.build_authorization_header("dev-42") + self.assertTrue(header.startswith("MediaBrowser ")) + self.assertIn('Client="jellyfin-cli"', header) + self.assertIn('DeviceId="dev-42"', header) + for required in ("Client=", "Device=", "DeviceId=", "Version="): + self.assertIn(required, header) + self.assertNotIn("Token=", header) # pre-token form carries no token segment + with_token = jellyfin_cli.build_authorization_header("dev-42", token="tok-1") + self.assertIn('Token="tok-1"', with_token) + def test_next_up_item_and_browse_parse_results_as_json(self): client = NavigationFakeClient() @@ -149,7 +201,7 @@ class JellyfinCliTests(unittest.TestCase): "series": "Voyagers", "episode_number": 4}], "total_record_count": 9, }) - self.assertEqual(client.next_up_calls, [("user-1", 3)]) + self.assertEqual(client.next_up_calls, [("user-1", 3, None)]) output = io.StringIO() with contextlib.redirect_stdout(output): @@ -163,13 +215,15 @@ class JellyfinCliTests(unittest.TestCase): output = io.StringIO() with contextlib.redirect_stdout(output): jellyfin_cli.cmd_browse(client, ["--library-id", "library-1", "--type", "Movie", - "--limit", "4", "--start-index", "2"]) + "--limit", "4", "--start-index", "2", + "--user-id", "user-1"]) self.assertEqual(json.loads(output.getvalue()), { "items": [{"id": "movie-1", "name": "Arrival", "type": "Movie", "year": 2016}], "start_index": 2, "total_record_count": 1, }) - self.assertEqual(client.items_calls, [("library-1", ["Movie"], 4, "SortName", "Ascending", 2)]) + self.assertEqual(client.items_calls, + [("library-1", ["Movie"], 4, "SortName", "Ascending", 2, "user-1")]) def test_user_scoped_navigation_requires_user_before_network(self): jellyfin_cli.ENV_USER_ID = "" @@ -186,6 +240,7 @@ class JellyfinCliTests(unittest.TestCase): def test_navigation_dry_runs_do_not_call_network_and_emit_requests(self): cases = ( (jellyfin_cli.cmd_next_up, ["--limit", "3"], {"path": "/Shows/NextUp", "params": {"userId": None, "limit": 3}}), + (jellyfin_cli.cmd_next_up, ["--limit", "3", "--series-id", "s1"], {"path": "/Shows/NextUp", "params": {"userId": None, "limit": 3, "seriesId": "s1"}}), (jellyfin_cli.cmd_item, ["--id", "item-1"], {"path": "/Items/item-1", "params": {"userId": None}}), (jellyfin_cli.cmd_browse, ["--library-id", "library-1", "--type", "Movie,Series", "--limit", "4", "--start-index", "2"], {"path": "/Items", "params": {"parentId": "library-1", "limit": 4, "sortBy": "SortName", "sortOrder": "Ascending", "startIndex": 2, "recursive": True, "includeItemTypes": "Movie,Series"}}), ) @@ -217,5 +272,320 @@ class JellyfinCliTests(unittest.TestCase): self.assertIn("Example:", help_result.stdout) +class LoginAuthSequenceTests(unittest.TestCase): + """The login path must demonstrate the researched auth sequence: + complete pre-token MediaBrowser header on POST /Users/AuthenticateByName, + AccessToken returned, Token= header for everything after.""" + + def run_cli(self, *args): + return subprocess.run([str(SCRIPT), *args], text=True, capture_output=True, + env=clean_env(), cwd=tempfile.gettempdir()) + + def test_help_lists_login_and_every_subcommand(self): + result = self.run_cli("--help") + self.assertEqual(result.returncode, 0) + for noun in ("login", "info", "recent", "search", "next-up", "item", + "seasons", "episodes", "browse", "libraries", "stats"): + self.assertIn(noun, result.stdout) + + def test_login_requires_username(self): + result = self.run_cli("login") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--username", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + def test_login_dry_run_previews_pre_token_header_without_network(self): + result = self.run_cli("--dry-run", "--json", "login", "--username", "alice") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertTrue(payload["dry_run"]) + self.assertEqual(payload["path"], "/Users/AuthenticateByName") + header = payload["authorization_header"] + self.assertIn("MediaBrowser", header) + for part in ("Client=", "Device=", "DeviceId=", "Version="): + self.assertIn(part, header) + self.assertNotIn("Token=", header) # pre-token: no token exists yet + self.assertTrue(payload["pre_token_header"]) + + def test_login_strips_trailing_slash_from_server_override(self): + result = self.run_cli("--dry-run", "--json", "login", "--username", "bob", + "--server", "http://box.local:8096/") + payload = json.loads(result.stdout) + self.assertEqual(payload["server"], "http://box.local:8096") + + def test_mocked_login_sends_media_browser_header_and_returns_session(self): + cli = jellyfin_cli + cli.GLOBAL_FLAGS = {"json": True, "dry_run": False} + captured = {} + response = FakeResponse(200, json_body={ + "User": {"Name": "alice", "Id": "6eec632a-ff0d-4d09-aad0-bf9e90b14bc6"}, + "SessionInfo": {"UserId": "6eec632a-ff0d-4d09-aad0-bf9e90b14bc6"}, + "AccessToken": "at-1234", + "ServerId": "srv-1", + }) + with patch.object(cli.requests, "post") as poster: + poster.return_value = response + output = io.StringIO() + with contextlib.redirect_stdout(output): + cli.cmd_login(cli.JellyfinClient(url="http://s:8096", device_id="dev-9"), + ["--username", "alice", "--password", "pw"]) + captured["call"] = poster.call_args + cli.GLOBAL_FLAGS = {"json": False, "dry_run": False} + + args, kwargs = captured["call"] + self.assertTrue(args[0].endswith("/Users/AuthenticateByName")) + header = kwargs["headers"]["Authorization"] + self.assertIn("MediaBrowser", header) + self.assertIn('DeviceId="dev-9"', header) + self.assertNotIn("Token=", header) # pre-token header on the login call itself + self.assertEqual(kwargs["json"], {"Username": "alice", "Pw": "pw"}) + session = json.loads(output.getvalue()) + self.assertEqual(session["user_id"], "6eec632a-ff0d-4d09-aad0-bf9e90b14bc6") + self.assertEqual(session["access_token"], "at-1234") + self.assertIn('Token="at-1234"', session["authorization_header"]) + + def test_mocked_login_error_paths_do_not_crash(self): + cli = jellyfin_cli + cli.GLOBAL_FLAGS = {"json": False, "dry_run": False} + original_post = cli.requests.post + for status, expected_fragment in ((400, "400"), (401, "401"), (403, "403")): + with self.subTest(status=status): + cli.requests.post = Mock(return_value=FakeResponse(status, text="Error processing request.")) + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr), self.assertRaises(SystemExit): + cli.cmd_login(cli.JellyfinClient(url="http://s:8096", device_id="d"), + ["--username", "alice", "--password", "bad"]) + self.assertIn(expected_fragment, stderr.getvalue()) + cli.requests.post = original_post + cli.GLOBAL_FLAGS = {"json": False, "dry_run": False} + + def test_reads_require_a_credential_before_network(self): + cli = jellyfin_cli + client = cli.JellyfinClient() + client.key = "" + client.token = "" + error = io.StringIO() + with contextlib.redirect_stderr(error), self.assertRaises(SystemExit): + client._get("/System/Info") + self.assertIn("JELLYFIN_API_KEY", error.getvalue()) + + def test_client_headers_use_modern_authorization_scheme(self): + cli = jellyfin_cli + client = cli.JellyfinClient(key="k-1", device_id="dev-1") + headers = client._headers() + self.assertIn('Token="k-1"', headers["Authorization"]) + self.assertTrue(headers["Authorization"].startswith("MediaBrowser ")) + token_client = cli.JellyfinClient(token="t-1", device_id="dev-1") + self.assertIn('Token="t-1"', token_client._headers()["Authorization"]) + + +class TvNavigationCommandTests(unittest.TestCase): + """seasons/episodes commands close the search → seasons → episodes walk.""" + + def setUp(self): + self.flags = jellyfin_cli.GLOBAL_FLAGS + self.env_user_id = jellyfin_cli.ENV_USER_ID + jellyfin_cli.GLOBAL_FLAGS = {"json": True, "dry_run": False} + jellyfin_cli.ENV_USER_ID = "" + + def tearDown(self): + jellyfin_cli.GLOBAL_FLAGS = self.flags + jellyfin_cli.ENV_USER_ID = self.env_user_id + + def test_seasons_and_episodes_emit_items_and_consume_ids(self): + client = NavigationFakeClient() + output = io.StringIO() + with contextlib.redirect_stdout(output): + jellyfin_cli.cmd_seasons(client, ["--series-id", "series-1", + "--user-id", "user-1"]) + seasons = json.loads(output.getvalue()) + self.assertEqual(seasons["items"][0]["name"], "Season 1") + self.assertEqual(seasons["items"][0]["season_number"], 1) + self.assertEqual(client.seasons_calls, [("series-1", "user-1")]) + + output = io.StringIO() + with contextlib.redirect_stdout(output): + jellyfin_cli.cmd_episodes(client, ["--series-id", "series-1", + "--season-id", "season-1", + "--user-id", "user-1"]) + episodes = json.loads(output.getvalue()) + self.assertEqual(episodes["items"][0]["episode_number"], 4) + self.assertEqual(client.episodes_calls, [("series-1", "season-1", "user-1")]) + + def test_seasons_and_episodes_require_user_before_network(self): + for handler, arguments in ( + (jellyfin_cli.cmd_seasons, ["--series-id", "series-1"]), + (jellyfin_cli.cmd_episodes, ["--series-id", "series-1"]), + ): + client = NavigationFakeClient() + with self.subTest(handler=handler.__name__), \ + contextlib.redirect_stderr(io.StringIO()), \ + self.assertRaises(SystemExit): + handler(client, arguments) + self.assertEqual(client.seasons_calls, []) + self.assertEqual(client.episodes_calls, []) + + def test_seasons_and_episodes_dry_run_previews_requests(self): + cases = ( + (jellyfin_cli.cmd_seasons, ["--series-id", "s1"], + {"path": "/Shows/s1/Seasons", "params": {"userId": None}}), + (jellyfin_cli.cmd_episodes, ["--series-id", "s1", "--season-id", "se1", + "--limit", "7"], + {"path": "/Shows/s1/Episodes", + "params": {"userId": None, "limit": 7, "startIndex": 0, + "seasonId": "se1"}}), + ) + for handler, arguments, request in cases: + client = NavigationFakeClient(dry_run=True) + output = io.StringIO() + with self.subTest(handler=handler.__name__), \ + contextlib.redirect_stdout(output): + handler(client, arguments) + self.assertEqual(json.loads(output.getvalue()), + {"dry_run": True, **request}) + self.assertEqual(client.seasons_calls, []) + self.assertEqual(client.episodes_calls, []) + + +class SearchHintIdFallbackTests(unittest.TestCase): + """SearchHint carries both Id and (deprecated) ItemId; newer servers omit ItemId.""" + + def test_prefers_current_id_falls_back_to_deprecated_item_id(self): + jellyfin_cli.GLOBAL_FLAGS = {"json": True, "dry_run": False} + try: + client = Mock() + client.dry_run = False + client.search = Mock(return_value={ + "SearchHints": [ + {"Id": "new-id", "ItemId": "legacy-id", "Name": "Both", "Type": "Series"}, + {"Id": "only-new", "Name": "Modern", "Type": "Movie"}, + {"ItemId": "legacy-only", "Name": "Old server", "Type": "Movie"}, + ], + "TotalRecordCount": 3, + }) + output = io.StringIO() + with contextlib.redirect_stdout(output): + jellyfin_cli.cmd_search(client, ["--query", "dune"]) + results = json.loads(output.getvalue())["results"] + self.assertEqual([r["id"] for r in results], + ["new-id", "only-new", "legacy-only"]) + finally: + jellyfin_cli.GLOBAL_FLAGS = {"json": False, "dry_run": False} + + +class SearchRequiresQueryTests(unittest.TestCase): + def test_missing_query_is_argument_error(self): + result = subprocess.run([str(SCRIPT), "search"], capture_output=True, + text=True, env=clean_env()) + self.assertNotEqual(result.returncode, 0) + self.assertIn("--query", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + +class PipelineChainTests(unittest.TestCase): + """Documented multi-step pipelines must execute stage by stage, each stage + consuming the previous stage's emitted output (field names AND types).""" + + @classmethod + def setUpClass(cls): + cls.tmpdir = tempfile.TemporaryDirectory(prefix="jellyfin-pipeline-") + + @classmethod + def tearDownClass(cls): + cls.tmpdir.cleanup() + + def run_cli(self, *args): + return subprocess.run([str(SCRIPT), "--dry-run", "--json", *args], + text=True, capture_output=True, env=clean_env(), + cwd=self.tmpdir.name) + + def run_jq(self, *jq_args): + return subprocess.run(["jq", *jq_args], text=True, capture_output=True, + env=clean_env(), cwd=self.tmpdir.name) + + def write_stage_file(self, name, document): + path = pathlib.Path(self.tmpdir.name) / name + path.write_text(json.dumps(document)) + return str(path) + + def test_search_then_item_chain_consumability(self): + # Stage 1: search plan; jq extracts the query field (string) the next + # stage would resolve into an item id. + r1 = self.run_cli("search", "--query", "dune", "--type", "Movie") + self.assertEqual(r1.returncode, 0, r1.stderr) + stage1 = self.write_stage_file("stage1.json", json.loads(r1.stdout)) + term = self.run_jq("-r", ".params.searchTerm", stage1).stdout.strip() + self.assertEqual(term, "dune") + types_check = self.run_jq("-r", ".params.includeItemTypes | type", stage1) + self.assertEqual(types_check.stdout.strip(), "string") + + # Stage 2: item detail plan consumes a hand-built id from stage 1's + # contract (results[].id is a string); jq proves the id travels into + # the request path. + r2 = self.run_cli("item", "--id", "movie-123", "--user-id", "user-9") + self.assertEqual(r2.returncode, 0, r2.stderr) + stage2 = self.write_stage_file("stage2.json", json.loads(r2.stdout)) + item_path = self.run_jq("-r", ".path", stage2).stdout.strip() + self.assertEqual(item_path, "/Items/movie-123") + user_id = self.run_jq("-r", ".params.userId", stage2).stdout.strip() + self.assertEqual(user_id, "user-9") + + # Stage 3: next-up plan consumes the same user id and proves it is a + # JSON string parameter on /Shows/NextUp. + r3 = self.run_cli("next-up", "--user-id", "user-9", "--limit", "5") + self.assertEqual(r3.returncode, 0, r3.stderr) + stage3 = self.write_stage_file("stage3.json", json.loads(r3.stdout)) + self.assertEqual(self.run_jq("-r", ".params.userId", stage3).stdout.strip(), + "user-9") + self.assertEqual(self.run_jq("-r", ".path", stage3).stdout.strip(), + "/Shows/NextUp") + + def test_libraries_then_browse_chain_consumability(self): + # Stage 1: libraries plan; jq type-checks the id field browse consumes. + r1 = self.run_cli("libraries") + self.assertEqual(r1.returncode, 0, r1.stderr) + stage1 = self.write_stage_file("stage1.json", json.loads(r1.stdout)) + self.assertEqual(self.run_jq("-r", ".path", stage1).stdout.strip(), + "/Library/MediaFolders") + + # Stage 2: browse plan consumes a library id and paginates by + # startIndex (number type asserted via jq). + r2 = self.run_cli("browse", "--library-id", "lib-77", "--start-index", "100") + self.assertEqual(r2.returncode, 0, r2.stderr) + stage2 = self.write_stage_file("stage2.json", json.loads(r2.stdout)) + parent = self.run_jq("-r", ".params.parentId", stage2).stdout.strip() + self.assertEqual(parent, "lib-77") + start_index_type = self.run_jq("-r", ".params.startIndex | type", stage2) + self.assertEqual(start_index_type.stdout.strip(), "number") + + # Stage 3: seasons plan consumes a series id discovered by browsing. + r3 = self.run_cli("seasons", "--series-id", "series-2", "--user-id", "user-1") + self.assertEqual(r3.returncode, 0, r3.stderr) + stage3 = self.write_stage_file("stage3.json", json.loads(r3.stdout)) + self.assertEqual(self.run_jq("-r", ".path", stage3).stdout.strip(), + "/Shows/series-2/Seasons") + + def test_login_to_recent_chain_previews_token_handoff(self): + # Stage 1: login plan emits the pre-token header the server requires. + r1 = self.run_cli("login", "--username", "alice") + self.assertEqual(r1.returncode, 0, r1.stderr) + stage1 = self.write_stage_file("stage1.json", json.loads(r1.stdout)) + header = self.run_jq("-r", ".authorization_header", stage1).stdout.strip() + self.assertIn("MediaBrowser", header) + self.assertNotIn("Token=", header) + + # Stage 2: user-scoped read plan; the user id login would capture is a + # string parameter on /Items/Latest (shape: bare array per research). + r2 = self.run_cli("recent", "--user-id", "6eec632a", "--limit", "20") + self.assertEqual(r2.returncode, 0, r2.stderr) + stage2 = self.write_stage_file("stage2.json", json.loads(r2.stdout)) + self.assertEqual(self.run_jq("-r", ".path", stage2).stdout.strip(), + "/Items/Latest") + user_id_type = self.run_jq("-r", ".params.userId | type", stage2) + self.assertEqual(user_id_type.stdout.strip(), "string") + self.assertEqual(self.run_jq("-r", ".params.fields", stage2).stdout.strip(), + "DateCreated") + + if __name__ == "__main__": unittest.main() From bf1bba6fc62c347a1fb9813077f16b1d8ba715db Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sat, 29 Aug 2026 17:38:18 -0400 Subject: [PATCH 32/40] chore(catalog): sync jellyfin blurb and regenerated catalogs - Root README blurb and references/skill-triggers.md trigger row now match the thickened jellyfin description (manual-sync requirement). - Regenerate .claude-plugin/marketplace.json and llms.txt via --write: both embed skill descriptions, so the rewrite staled them; check modes exit 0 again. Codex artifact unaffected (no descriptions). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/marketplace.json | 2 +- README.md | 2 +- llms.txt | 2 +- references/skill-triggers.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index b0a4d98..9c55d4e 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -570,7 +570,7 @@ "./jellyfin" ], "strict": false, - "description": "Query your Jellyfin media server from the terminal — recently added media, search, item details, next-up episodes, library browsing, server info, and stats. Use when the user asks about Jellyfin, media server, movies, TV shows, next episodes, or their media library." + "description": "Query your Jellyfin media server from the terminal — recently added media, search, item details, series navigation, next-up episodes, library browsing, server info, and user login. Use when the user asks about Jellyfin, media servers, movies, TV shows, next episodes, or their media library. Do not use this skill for server installation, library management, playback control, or Emby/Plex servers." }, { "name": "jira", diff --git a/README.md b/README.md index 219be84..4ebf185 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ Convert operational incident and near-miss evidence into verified, owned improve ### [jellyfin](jellyfin/SKILL.md) -Jellyfin media server from the terminal. Check server info, browse recently added and library contents, search and inspect media, see next-up episodes, and view statistics. +Jellyfin media server from the terminal. Log in as a user or use an API key, check server info, browse recently added and library contents, search and inspect media, walk series/seasons/episodes, see next-up episodes, and view statistics — with MediaBrowser auth, user-id scoping, and response-shape quirks documented. ### [jira](jira/SKILL.md) diff --git a/llms.txt b/llms.txt index 2b7d405..7982fad 100644 --- a/llms.txt +++ b/llms.txt @@ -64,7 +64,7 @@ - [hugo-theme](hugo-theme/SKILL.md): Build, customize, and debug advanced Hugo CMS themes — template architecture, asset pipeline (CSS/JS/image processing), shortcodes and render hooks, page bundles, cover images, Hugo Modules, performance, SEO, and CI/CD. Use when working on a Hugo theme or site template layer. - [implementation-planning](implementation-planning/SKILL.md): Plan the implementation of an approved requirement or specification: produce an executable, dependency-aware delivery plan covering work breakdown, dependency mapping, critical path, ownership, parallelism and sequencing, rollout strategy, rollback and recovery paths, and verification against the original requirement. Supports cross-team, cross-repository, migration, and staged-rollout scenarios. Do not use for pre-approval discovery or needs-finding, authoring a specification from scratch, coding or implementation, the neckbeard issue-to-PR delivery flow itself, or any work whose prerequisite decision has not been approved — planning unapproved work is an explicit stop condition. - [incident-learning](incident-learning/SKILL.md): Convert operational incident and near-miss evidence into durable product, engineering, test, evaluation, and governance improvements with verified closure. Separate observed facts from causal hypotheses and unresolved uncertainty; map follow-up work across code, tests, skills, operations, product, and governance; track ownership, verification, and closure for every finding. Do not use to assign blame or produce a generic postmortem template; do not close learning because tickets were created — require evidence the intended change occurred. -- [jellyfin](jellyfin/SKILL.md): Query your Jellyfin media server from the terminal — recently added media, search, item details, next-up episodes, library browsing, server info, and stats. Use when the user asks about Jellyfin, media server, movies, TV shows, next episodes, or their media library. +- [jellyfin](jellyfin/SKILL.md): Query your Jellyfin media server from the terminal — recently added media, search, item details, series navigation, next-up episodes, library browsing, server info, and user login. Use when the user asks about Jellyfin, media servers, movies, TV shows, next episodes, or their media library. Do not use this skill for server installation, library management, playback control, or Emby/Plex servers. - [jira](jira/SKILL.md): Interact with Atlassian Jira from the terminal: search issues with JQL, view details, create issues, add comments, count matches with fast approximate-count, list projects, discover valid transitions, and change status. Includes a full JQL language reference (functions, operators, history predicates, date expressions, saved filters, performance tuning) plus REST auth/pagination guidance. Use when the user mentions Jira, a ticket key (e.g. PROJ-123), asks about issues, bugs, tasks, projects, or sprint work, or needs to write, debug, or optimize JQL queries. Do not use for GitHub or GitLab issue tracking, Jira site administration, or generic ticketing systems. - [kanban-guru](kanban-guru/SKILL.md): A virtual Kanban expert who can diagnose flow problems, design board configurations, set up multi-portfolio operating models, calibrate WIP limits, establish service level expectations, and guide Scrum-to-Kanban transitions. Load this when your team is struggling with throughput, cycle times are unpredictable, multiple stakeholders compete for the same engineers, or you're wondering if Kanban is right for you. - [kubernetes](kubernetes/SKILL.md): Operate, troubleshoot, secure, upgrade, and automate Kubernetes clusters and workloads safely across upstream Kubernetes, k3s, RKE2, MicroK8s, k0s, Talos, OpenShift/OKD, kind, Minikube, Rancher-managed clusters, EKS, AKS, and GKE. Use when a task involves kubectl, Kubernetes APIs, Pods, Deployments, StatefulSets, Services, Ingress or Gateway API, CRDs, RBAC, NetworkPolicy, storage, scheduling, autoscaling, cluster lifecycle, or the bundled agent-first k8s-cli. diff --git a/references/skill-triggers.md b/references/skill-triggers.md index c7a17f1..a455567 100644 --- a/references/skill-triggers.md +++ b/references/skill-triggers.md @@ -19,7 +19,7 @@ Each skill's `description` field is the canonical routing contract. This conveni | "Grafana", "Grafana dashboard", "Grafana panel", "Grafana variable", "Grafana data source", "Grafana alerting", "contact point", "notification policy", "mute timing", "Grafana provisioning", "dashboard as code", "Grafana API", "Grafana service account", "Grafana RBAC", "Grafana plugin", "Grafana troubleshooting", "duplicate dashboard UID" | [grafana](../grafana/SKILL.md) | | "hugo theme", "hugo cms", "accessible theme", "wcag theme", "theme design", "theme accessibility", "theme UX", "design tokens", "css theme", "theme contrast", "responsive theme", "hugo template", "hugo pipes", "hugo module", "hugo shortcode", "render hook", "tailwindcss hugo", "hugo i18n", "hugo seo", "hugo output format", "hugo site", "hugo static site" | [hugo-theme](../hugo-theme/SKILL.md) | | "Ghost", "Ghost CMS", "ghost blog", "create a post on my blog", "blog publishing", "GHOST_ADMIN_KEY" | [ghost](../ghost/SKILL.md) | -| "Jellyfin", "Jellyfin media server", "recently added movies", "recently added episodes", "media library", "JELLYFIN_API_KEY" | [jellyfin](../jellyfin/SKILL.md) | +| "Jellyfin", "Jellyfin media server", "recently added movies", "recently added episodes", "media library", "JELLYFIN_API_KEY", "Jellyfin API authentication" | [jellyfin](../jellyfin/SKILL.md) | | "Jira", "Atlassian Jira", "JQL", "ticket PROJ-123", "sprint work", "JIRA_API_TOKEN" | [jira](../jira/SKILL.md) | | "Open Library", "openlibrary", "book search", "ISBN lookup", "author records", "work details", "book editions", "book ratings", "cover image" | [openlibrary](../openlibrary/SKILL.md) | | "weather", "forecast", "temperature", "is it raining", "Tempest" | [tempest](../tempest/SKILL.md) | From 83e07b9ac2ad4f6cd9876f05a668c4db01d81003 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sat, 29 Aug 2026 18:43:00 -0400 Subject: [PATCH 33/40] docs(peertube): thicken federated video skill against current API research Research-driven rebuild of the peertube skill (docs.joinpeertube.org REST reference 8.1.0 + SepiaSearch + server source + live anonymous probes): - SKILL.md rewritten to the lastfm model: intent-grouped commands, pipeline recipes, jq guidance, researched gotchas, When-to-use/When-not-to-use, reference routing table. New negative boundary in the description (YouTube/Vimeo uploads, video editing, server administration). - scripts/peertube-cli -> scripts/peertube, rewritten and extended: offset (start/count) pagination replaces the nonexistent page param, comments fixed to the hyphenated /comment-threads route, server command now composes /config/about + /server/stats (canonical paths), search gains --search-target with searchTarget=local default and help text stating its instance-local scope, new video/comments/channel/account/ my-videos/logout commands, --server hoisted before or after the subcommand, OAuth2 password grant hardened for 2FA (x-peertube-otp) and the production client_secret masking behavior, per-instance owner-only token file with refresh-before-expiry and revocation. - references/: auth-and-tokens, search-and-discovery, endpoint-catalog, gotchas-field-guide, worked-recipes - all cited to official docs with Sources footers (URLs verified live at authoring time). - scripts/test_peertube.py: 54 offline tests (help, argument errors, dry-run plans, mocked OAuth2 persistence/refresh/revocation, handler contracts, documented pipeline chains) passing pytest strict-markers, unittest discovery, and the proxy-trap zero-egress rerun; one env-guarded anonymous live probe (PEERTUBE_LIVE_TESTS=1). - evals/evals.json: six schema-v1 cases incl. SepiaSearch-scope and masked-secret cases plus a should-not-trigger YouTube negative probe. - README refreshed for humans; root README blurb and skill-triggers row synced; marketplace.json/llms.txt regenerated (codex artifacts unchanged); test-results/ gitignored (pytest runner artifact). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/marketplace.json | 2 +- .gitignore | 1 + README.md | 2 +- llms.txt | 2 +- peertube/README.md | 73 +- peertube/SKILL.md | 264 ++++-- peertube/evals/evals.json | 72 ++ peertube/references/auth-and-tokens.md | 152 ++++ peertube/references/endpoint-catalog.md | 131 +++ peertube/references/gotchas-field-guide.md | 125 +++ peertube/references/search-and-discovery.md | 146 +++ peertube/references/worked-recipes.md | 151 +++ peertube/scripts/peertube | 821 +++++++++++++++++ peertube/scripts/peertube-cli | 239 ----- peertube/scripts/test_peertube.py | 961 ++++++++++++++++++++ references/skill-triggers.md | 1 + 16 files changed, 2827 insertions(+), 316 deletions(-) create mode 100644 peertube/evals/evals.json create mode 100644 peertube/references/auth-and-tokens.md create mode 100644 peertube/references/endpoint-catalog.md create mode 100644 peertube/references/gotchas-field-guide.md create mode 100644 peertube/references/search-and-discovery.md create mode 100644 peertube/references/worked-recipes.md create mode 100755 peertube/scripts/peertube delete mode 100755 peertube/scripts/peertube-cli create mode 100644 peertube/scripts/test_peertube.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 9c55d4e..91266f9 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -822,7 +822,7 @@ "./peertube" ], "strict": false, - "description": "Browse PeerTube federated video from the terminal: view videos and channels, search across instances, check server stats, and manage your account. Uses OAuth2 authentication with token persistence. Use when the user mentions PeerTube, federated video, decentralized video platforms, or browsing/uploading to a PeerTube instance." + "description": "Browse PeerTube federated video from the terminal — instance stats, latest videos, video detail, comment threads, channels, accounts, instance-local search, and OAuth2 login with per-instance token persistence. Set PEERTUBE_SERVER to any instance; point it at sepiasearch.org for fediverse-wide search. Use when the user mentions PeerTube, federated video, SepiaSearch, or browsing a specific PeerTube instance. Do not use this skill for YouTube/Vimeo uploads, video editing, or installing and administering a PeerTube server." }, { "name": "platform-engineering", diff --git a/.gitignore b/.gitignore index 48253f4..e0f00bd 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ cashew-brain/ # ─── Python ─────────────────────────────────────────────────── __pycache__/ +test-results/ *.py[cod] *.egg-info/ *.egg diff --git a/README.md b/README.md index 4ebf185..274b871 100644 --- a/README.md +++ b/README.md @@ -377,7 +377,7 @@ Build and maintain owner-approved Primary, Alternate, Contingency, and Emergency ### [peertube](peertube/SKILL.md) -PeerTube federated video platform from the terminal. Browse videos and channels, search across instances, view server info. OAuth2 login with token persistence. Set PEERTUBE_SERVER to point at any instance. +PeerTube federated video from the terminal. Browse videos, channels, and comment threads on any instance, search instance-local or the whole fediverse via SepiaSearch, check server stats, and log in with OAuth2 — with per-instance token persistence and pagination/search-scope quirks documented. ### [platform-engineering](platform-engineering/SKILL.md) diff --git a/llms.txt b/llms.txt index 7982fad..0f1a4bf 100644 --- a/llms.txt +++ b/llms.txt @@ -92,7 +92,7 @@ - [operational-design](operational-design/SKILL.md): Design and improve operational processes and organizational scaling — process design, operational metrics, compliance and audit, vendor management, and team topology. Covers value stream mapping, BPMN, bottleneck analysis, scaling from 10 to 100 to 1000 people, KPI design, balanced scorecard, SOC 2, ISO 27001, GDPR readiness, RFP processes, SLA design, vendor scorecards, team topologies, Conway's Law, and Dunbar's Number. Do not use for engineering delivery, financial modeling, or technology evaluation. - [org-design](org-design/SKILL.md): CHRO methodology — organizational design (team topologies, span of control, reporting structures), talent strategy (make-vs-buy, skill taxonomies, succession planning), compensation frameworks (market benchmarking, equity design, leveling), culture architecture (values codification, rituals, psychological safety), organizational health metrics (eNPS, retention risk, engagement surveys), DEI strategy (inclusive design, equitable systems, belonging). - [pace-plan](pace-plan/SKILL.md): Build, coordinate, operate, troubleshoot, exercise, and improve an authorized Primary, Alternate, Contingency, and Emergency communications plan. Use for resilient emergency-communications paths and their ownership, triggers, check-ins, tests, and corrective actions. Do not use for generic incident status messaging, frequency or channel planning, radio programming, or unauthorized transmission and activation. -- [peertube](peertube/SKILL.md): Browse PeerTube federated video from the terminal: view videos and channels, search across instances, check server stats, and manage your account. Uses OAuth2 authentication with token persistence. Use when the user mentions PeerTube, federated video, decentralized video platforms, or browsing/uploading to a PeerTube instance. +- [peertube](peertube/SKILL.md): Browse PeerTube federated video from the terminal — instance stats, latest videos, video detail, comment threads, channels, accounts, instance-local search, and OAuth2 login with per-instance token persistence. Set PEERTUBE_SERVER to any instance; point it at sepiasearch.org for fediverse-wide search. Use when the user mentions PeerTube, federated video, SepiaSearch, or browsing a specific PeerTube instance. Do not use this skill for YouTube/Vimeo uploads, video editing, or installing and administering a PeerTube server. - [platform-engineering](platform-engineering/SKILL.md): Use this skill when building or operating internal developer platforms: infrastructure as code, CI/CD, container orchestration, service networking, secrets, and observability. Do not use it to define release process, promotion, rollout, or rollback policy; use release-engineering for that delivery model. - [playwright](playwright/SKILL.md): Operate Playwright for browser automation end to end: author and debug E2E test suites (robust locators, network interception and mocking, parallel workers, accessibility snapshot checks), wire them into CI, and drive headless browsing and scraping with an extract -> validate -> save loop. Use when writing, running, fixing, or scraping with Playwright, when a Playwright CI failure or JSON report needs triage, or when the bundled pwrun script should analyze a run. Do not use for QA strategy or test framework selection (route to qa-methodology), for frontend component or architecture design (route to frontend-engineering), or for Cloudflare/DDoS-GUARD challenge bypass (use flaresolverr). - [postgres](postgres/SKILL.md): Operate PostgreSQL instances safely: configuration review, index and query-plan analysis, vacuum and bloat management, WAL archiving and point-in-time recovery, replication and failover, extensions, major-version upgrades, and evidence-based diagnostics with the bundled read-only pgdiag script. Use when running or inspecting a PostgreSQL server, diagnosing performance or backup health, or planning an upgrade or failover. Do not use for application-level data access patterns (that's backend-engineering) or schema design (that's data-architect/data-engineering). diff --git a/peertube/README.md b/peertube/README.md index 7ff9434..176181b 100644 --- a/peertube/README.md +++ b/peertube/README.md @@ -1,35 +1,78 @@ # PeerTube — Federated Video from the Terminal -Browse videos, channels, and server info on any PeerTube instance. Search across the fediverse, list channels, check your account stats, and manage authentication. +Browse any PeerTube instance from the command line: latest videos, video detail, comment +threads, channels and accounts, instance stats, and OAuth2 login for your own account — +plus fediverse-wide search through SepiaSearch. ## Why Install This Skill -When your agent loads this skill, it can **navigate the federated video universe** without a browser. That means: +When your agent loads this skill, it can **navigate the federated video universe** without +a browser. That means: -- **Browse videos** — recent uploads from any instance -- **Search across instances** — find content in the fediverse -- **Explore channels** — list channels and their videos -- **Check server info** — instance name, description, user/video/view stats -- **Authenticate** — OAuth2 login with token persistence +- **Browse any instance** — latest videos with real offset pagination (the API has no + `page` parameter, and most naive wrappers get this wrong) +- **Search the right scope** — instance-local search or the whole fediverse via + SepiaSearch, with the `searchTarget` semantics documented instead of guessed +- **Inspect videos deeply** — full metadata, comment threads (the hyphenated + `/comment-threads` route), channels, and accounts by handle (`name@host`) +- **Check instance health** — name, description, and user/video/view counters composed + from `/config/about` + `/server/stats` anonymously +- **Authenticate safely** — OAuth2 password grant with per-instance, owner-only token + persistence, automatic refresh, and proper server-side revocation on logout +- **Avoid the traps** — masked `client_secret` responses, token lifetimes that vary per + instance, 2FA `x-peertube-otp`, rate-limit headers, RFC7807 error bodies + +Every command is read-only except `login`/`logout`, and `--dry-run` previews any request +without touching the network. ## What You Get -| Directory | Purpose | -|-----------|---------| -| `SKILL.md` | Complete command reference with auth setup | -| `scripts/peertube-cli` | CLI tool for PeerTube API | +| Path | Purpose | +|------|---------| +| `SKILL.md` | Complete command reference with setup, gotchas, and recipes | +| `scripts/peertube` | CLI for PeerTube API operations (`--json`, `--dry-run`, `--verbose`) | +| `scripts/test_peertube.py` | Offline test suite (all HTTP mocked, zero egress) | +| `references/auth-and-tokens.md` | The full OAuth2 flow, secret masking, token hygiene | +| `references/search-and-discovery.md` | Instance-local vs SepiaSearch search scopes | +| `references/endpoint-catalog.md` | Endpoint-by-endpoint parameters and response shapes | +| `references/gotchas-field-guide.md` | Failure signatures and version drift | +| `references/worked-recipes.md` | Multi-step CLI/jq and curl workflows | +| `evals/evals.json` | Behavioral eval cases including negative triggers | ## Quick Start ```bash -export PEERTUBE_SERVER="https://your-instance.example.com" -peertube-cli auth login --username "myuser" --password "mypassword" +export PEERTUBE_SERVER="https://" # any PeerTube instance +scripts/peertube server # instance stats, anonymous +scripts/peertube videos --limit 5 --json +scripts/peertube search --query "linux" # searches THIS instance +``` + +Fediverse-wide search through SepiaSearch (same API shape, wider index): + +```bash +PEERTUBE_SERVER="https://sepiasearch.org" scripts/peertube search --query "linux" +``` + +Optional login for your own account commands: + +```bash +scripts/peertube login --username "" --prompt +scripts/peertube me --json | jq '.role.label' +scripts/peertube logout # revokes server-side + deletes token file ``` ## Triggers -Load this for PeerTube, federated video, decentralized video platforms, or browsing PeerTube content. +Load this when asking about PeerTube, federated video, decentralized video platforms, +SepiaSearch, browsing a specific PeerTube instance's videos or channels, or PeerTube API +authentication. ## Requirements -Python 3.8+ with `requests` library. +Python 3.8+ with `requests`. One thing this skill always needs from you: **an instance +host** — export `PEERTUBE_SERVER` (e.g. `https://`) or pass +`--server https://...` per command, since PeerTube is federated and every command targets +one instance. Reads are anonymous; `me`/`my-videos` need a token from `scripts/peertube +login`. Tokens persist to `~/.config/peertube/token.json` (override the directory with +`PEERTUBE_CONFIG_DIR`). Find public instances at [joinpeertube.org](https://joinpeertube.org). diff --git a/peertube/SKILL.md b/peertube/SKILL.md index 133686f..fe0c01e 100644 --- a/peertube/SKILL.md +++ b/peertube/SKILL.md @@ -1,116 +1,262 @@ --- name: peertube -description: 'Browse PeerTube federated video from the terminal: view videos and channels, - search across instances, check server stats, and manage your account. Uses OAuth2 - authentication with token persistence. Use when the user mentions PeerTube, federated - video, decentralized video platforms, or browsing/uploading to a PeerTube instance.' +description: Browse PeerTube federated video from the terminal — instance stats, latest + videos, video detail, comment threads, channels, accounts, instance-local search, and + OAuth2 login with per-instance token persistence. Set PEERTUBE_SERVER to any instance; + point it at sepiasearch.org for fediverse-wide search. Use when the user mentions + PeerTube, federated video, SepiaSearch, or browsing a specific PeerTube instance. + Do not use this skill for YouTube/Vimeo uploads, video editing, or installing and + administering a PeerTube server. license: MIT -compatibility: Requires PEERTUBE_SERVER env var (set to your instance URL, e.g. https://watch.nousresearch.com), - Python 3.8+, and the `requests` library. OAuth2 tokens persisted to ~/.config/peertube-cli/token.json. +compatibility: Requires Python 3.8+ and `requests`. Reads are anonymous; authenticated + commands (`me`, `my-videos`) need a token from `scripts/peertube login`. Tokens persist + per-instance to ~/.config/peertube/token.json (owner-only). metadata: - tags: peertube, federated-video, video-platform, activitypub, api-client - sources: https://joinpeertube.org/, https://docs.joinpeertube.org/api/reference + tags: peertube, federated-video, activitypub, video-platform, sepiasearch, api-client + sources: https://docs.joinpeertube.org/api-rest-reference.html, https://sepiasearch.org/ --- -# peertube-cli — PeerTube Federated Video +# peertube — PeerTube federated video from the terminal -Browse videos, channels, and server info on any PeerTube instance. Search across the fediverse, list channels, check your account stats, and manage authentication — all from the terminal. +Browse any PeerTube instance — a federated deployment, not a single API — from the +terminal: instance stats, latest videos, full video detail, comment threads, channels, +accounts, and instance-local search. Authenticate with OAuth2 only for your own account +commands. Every command is read-only except `login`/`logout`. ## Setup -1. Set the PeerTube instance URL: +1. Choose the instance to talk to. Every command is per-instance; the API shape is + identical everywhere, but accounts, tokens, rules, and catalogs are not: ```bash -export PEERTUBE_SERVER="https://your-instance.example.com" +export PEERTUBE_SERVER="https://" # e.g. https://tilvids.com ``` -2. (Optional) Log in for authenticated operations: + To search the whole fediverse instead of one instance, point the same variable at the + public search index: `PEERTUBE_SERVER=https://sepiasearch.org` (same API shape — see + [references/search-and-discovery.md](references/search-and-discovery.md)). + +2. Nothing else is required to browse: videos, search, channels, comments, and instance + info are anonymous reads. + +3. (Optional) Log in only for your own account commands (`me`, `my-videos`): ```bash -peertube-cli auth login --username "your-username" --password "your-password" +scripts/peertube login --username --prompt ``` -The OAuth2 token is persisted to `~/.config/peertube-cli/token.json` and automatically reused. `--dry-run` works without authentication. +### How authentication works + +PeerTube uses plain OAuth2 with per-instance client credentials: the CLI anonymously +fetches the client pair from `GET /api/v1/oauth-clients/local` (singular `local`), then +exchanges your username/password for a bearer token at `POST /api/v1/users/token` +(`grant_type=password`, form-encoded). The token rides `Authorization: Bearer `, +lives for the instance's configured lifetime (read `expires_in` from the response — do not +assume a fixed number), and is refreshed automatically when it expires. The token file is +written owner-only to `~/.config/peertube/token.json` keyed by server URL. +**Do not commit tokens** — they are account credentials; revoke with `scripts/peertube +logout` (`POST /users/revoke-token`) when done. Current production instances mask +`client_secret` in the API response; the CLI detects this and explains the workaround. +Details and wire-level error signatures: +[references/auth-and-tokens.md](references/auth-and-tokens.md). ## Essential Commands -### auth login — Authenticate to a PeerTube instance +### server — instance stats and identity (anonymous) ```bash -peertube-cli auth login --username "myuser" --password "mypassword" # login +scripts/peertube server # name, description, user/video/view counters +scripts/peertube server --json ``` -Token is saved to `~/.config/peertube-cli/token.json` and reused on subsequent calls. Tokens expire after the server-configured lifetime (typically 24h). +Composes `GET /config/about` + `GET /server/stats` (canonical paths — there is no +`/instance/stats`). -### server — Instance information +### videos — browse the instance's uploads (anonymous) ```bash -peertube-cli server # instance name, description, stats -peertube-cli server --json # machine-readable +scripts/peertube videos # latest 15, offset pagination +scripts/peertube videos --limit 50 --offset 50 +scripts/peertube videos --sort -views --json # popular first ``` -Shows: instance name, short description, total users, total videos, total views. +Pagination is `start`/`count` offsets (max count 100) — the API has **no `page` +parameter**. -### videos — Browse recent videos +### search — find videos on THIS instance (anonymous) ```bash -peertube-cli videos # last 12 videos -peertube-cli videos --limit 24 # more results -peertube-cli videos --json # machine-readable +scripts/peertube search --query "linux" # instance-local (searchTarget=local) +scripts/peertube search -q "docker" --limit 20 --json +PEERTUBE_SERVER="https://sepiasearch.org" scripts/peertube search -q "linux" # fediverse-wide ``` -Shows: title, duration, views, author/channel, publish date. +The bundled CLI performs **instance-local** search only (`searchTarget=local`). For +fediverse-wide search, point `PEERTUBE_SERVER` at SepiaSearch — same commands, wider +index. Search results carry `channel.host`/`url`, the origin instance of federated hits. -### search — Search videos across the fediverse +### video — full detail for one video (anonymous) ```bash -peertube-cli search --query "linux tutorial" # search videos -peertube-cli search -q "peer" --limit 24 # more results -peertube-cli search -q "docker" --json # machine-readable +scripts/peertube video --id # numeric id, UUID, or shortUUID all work +scripts/peertube video --id --json | jq '{name, description, views, url}' ``` -### channels — List video channels +### comments — top-level comment threads (anonymous) ```bash -peertube-cli channels # all channels on the instance -peertube-cli channels --json # machine-readable with subscriber counts +scripts/peertube comments --id # GET /videos/{id}/comment-threads +scripts/peertube comments --id --limit 30 --json ``` -Shows: display name, channel handle (@name), video count, subscriber count. - -### me — Your profile +### channels / channel / account — creators (anonymous) ```bash -peertube-cli me # your account stats -peertube-cli me --json # machine-readable +scripts/peertube channels --limit 20 --json # instance channel list +scripts/peertube channel --handle framasoft@framatube.org # name or name@host +scripts/peertube account --name chocobozzz@framatube.org ``` -Shows: username, role, video count, view count. Requires authentication. +`channel` shows metadata plus the channel's uploads (offset-paginated). -## Global Flags - -All flags work in any position: +### me / my-videos — your account (requires login) ```bash -peertube-cli --json videos # flag before subcommand -peertube-cli videos --json # flag after subcommand -peertube-cli --dry-run search --query "test" # preview (no API call) -peertube-cli --quiet videos # suppress non-essential output -peertube-cli --verbose channels # detailed logging +scripts/peertube me --json | jq '.role.label' +scripts/peertube my-videos --limit 50 --json ``` +### login / logout — OAuth2 session management + +```bash +scripts/peertube login --username --prompt # hidden prompt +echo "" | scripts/peertube login --username --password-stdin +scripts/peertube login --username --otp # 2FA-enabled accounts +scripts/peertube logout # revoke server-side + delete file +``` + +## Global flags + +```bash +scripts/peertube --json videos # flag before or after the subcommand +scripts/peertube videos --json +scripts/peertube --dry-run search --query test # request plan, zero network +scripts/peertube --verbose videos --limit 2 # trace requests on stderr +scripts/peertube --server https://tilvids.com server # per-invocation instance override +``` + +`--dry-run` emits `{"dry_run": true, "method", "path", "params"}` (login adds +`form_fields` names only, never values) — use it to verify a jq chain before running it +live. `--help` and `--dry-run` never require credentials. + +## Pipeline recipes + +### Search, then inspect the top hit + +```bash +scripts/peertube search --query "linux" --limit 5 --json | jq -r '.videos[0].uuid' +scripts/peertube video --id "$(scripts/peertube search -q linux --limit 1 --json | jq -r '.videos[0].uuid')" --json +``` + +### Page through a channel's uploads + +```bash +scripts/peertube channel --handle framasoft@framatube.org --limit 100 --offset 0 --json | jq -r '.videos[].name' +# loop: advance --offset by the returned count until you reach .total (no page param exists) +``` + +### Instance report card + +```bash +scripts/peertube server --json | jq '{name: .instance.name, videos: .stats.totalLocalVideos, users: .stats.totalUsers, views: .stats.totalLocalVideoViews}' +``` + +### Log in, check quota, log out + +```bash +scripts/peertube login --username --prompt +scripts/peertube me --json | jq '{username, role: .role.label, quota_bytes: .videoQuota}' +scripts/peertube logout +``` + +## JSON and jq + +`--json` output keys are stable snake_case wrappers around raw API objects: `videos` +(the API's `{total, data}` list objects), `channels`, `threads` (+ `total_not_deleted`), +`instance` + `stats`, `channel`, `dry_run`/`method`/`path`/`params` for plans. Video +objects keep PeerTube's own field names — `uuid`, `shortUUID`, `name`, `duration` +(seconds), `views`, `publishedAt`, `account{name,displayName,host}`, +`channel{name,displayName,host}` — so jq selectors transfer directly to raw `curl` +against `/api/v1`. Example: `jq -r '.videos[] | [.name, .views, .channel.displayName] | @tsv'`. + ## Known Gotchas -- **Set PEERTUBE_SERVER first** — Without this env var, the CLI defaults to `https://your-instance.example.com` (which won't resolve). Always export the correct instance URL. -- **Authentication is required for most commands** — `server` and public video browsing work without auth. `me`, `channels`, and personal video lists require a valid OAuth token. Use `--dry-run` to preview without auth. -- **Token is persisted automatically** — After `auth login`, the token is saved to `~/.config/peertube-cli/token.json`. No need to login again unless the token expires. Delete this file to force re-login. -- **Token expiry** — PeerTube OAuth2 tokens have a configurable expiry (default ~24h). Expired tokens cause 401 errors. Re-run `auth login` to refresh. -- **Cross-instance search** — `search --query` searches across the fediverse, not just the local instance. Results may include videos from remote instances. -- **API pagination** — PeerTube uses offset-based pagination. The `--limit` flag controls the page size (default: 12 for videos, 15 for channels). -- **Rate limits** — PeerTube instances have configurable rate limits. The CLI does not auto-retry on 429 responses. +- **Instances are independent (federated, not one API)** — accounts, tokens, rules, + enabled features, and catalogs differ per instance. A token from instance A 401s on + instance B; the CLI keys the token file by server URL. Content federated *onto* an + instance still belongs to its origin (`channel.host`, video `url`). +- **Search scope is two different things** — `searchTarget=local` searches the + instance's own catalog; `search-index` (or SepiaSearch's base URL) searches the + fediverse via an external index. Omitting `searchTarget` gives the instance's own + scope on current servers, not the fediverse. The bundled CLI is instance-local unless + you point it at sepiasearch.org. +- **`page` does not exist** — collections paginate with `start`/`count` (max 100). + Clients sending `page=` silently re-read the first page forever. +- **The comments route is `/comment-threads`** (hyphenated) — `/comments` and + `/commentthreads` are not routes (they 400 on current servers). +- **Instance metadata paths are mixed** — stats at `/server/stats` (operation titled + "instance stats"), about at `/config/about`, config at `/config`. No + `/instance/*` metadata paths exist. +- **Production masks `client_secret`** — `oauth-clients/local` answers + `"********************************"` on current production instances; a token request + with the masked value 400s. The CLI detects it and explains the front-end-asset + workaround. `response_type=code` appears in old quick-start curls but is not part of + the current token schema — the CLI omits it. +- **Token lifetimes are instance-configurable** — read `expires_in` per response; the + CLI persists the absolute `expires_at` and refreshes automatically. Store tokens + owner-only, never commit them, revoke on logout (deleting the file alone leaves the + session live). +- **2FA needs an OTP header** — `x-peertube-otp` on the token request; the CLI maps a + bare 401 to "pass --otp". +- **Rate limits** — default 50 calls/10 s per IP (token endpoint tighter); on 429 read + `Retry-After` and back off. Errors use RFC7807 `application/problem+json` bodies, and + unknown routes answer 400 (not 404) — read the body. +- **`duration` is seconds**; ids are triple (`id`, `uuid`, `shortUUID` — all accepted by + detail endpoints); `role` is an object `{id, label}`; `videoQuota` is bytes. +- **Anonymous vs authed** — browsing/search/comments/instance-info need no token; + `/users/me*` and mutations always do. -## References +## When to use -- [scripts/peertube-cli](scripts/peertube-cli) — The CLI binary. Built following the cli-builder patterns: `--json`, `--dry-run`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth, config file persistence. -- [PeerTube API Reference](https://docs.joinpeertube.org/api/reference) — Official API reference. -- [JoinPeerTube.org](https://joinpeertube.org/) — Find instances and learn about the federated video platform. +Use this skill for read-only interaction with PeerTube instances: browsing and filtering +videos, instance-local or fediverse-wide search (via SepiaSearch), video detail and +comments, channel/account exploration, instance stats, and managing your own account +session with OAuth2 (login, profile, my videos, logout). + +## When not to use + +Do not use this skill for YouTube, Vimeo, or other platform uploads or any video +editing/transcoding (route to those platforms' own tooling and ffmpeg); for installing, +hosting, or administering a PeerTube server (instance administration is out of scope — +the bundled CLI is read-only plus login/logout); or for generic ActivityPub/Mastodon +federation questions (use a Mastodon or ActivityPub skill). + +## Reference Files + +| File | Use it for | +| ---- | ---------- | +| [references/auth-and-tokens.md](references/auth-and-tokens.md) | OAuth2 flow (oauth-clients/local, password grant), secret masking, refresh/revocation, token-file hygiene, wire error signatures | +| [references/search-and-discovery.md](references/search-and-discovery.md) | searchTarget local vs search-index, SepiaSearch semantics, search parameters and sorts | +| [references/endpoint-catalog.md](references/endpoint-catalog.md) | Every read endpoint's parameters, response shapes, pagination, rate limits | +| [references/gotchas-field-guide.md](references/gotchas-field-guide.md) | Symptom → cause → fix table for every failure signature and version drift | +| [references/worked-recipes.md](references/worked-recipes.md) | Multi-step CLI/jq workflows, raw curl auth chain, jq processing patterns | + +## Available Scripts and Prerequisites + +- `scripts/peertube` — the bundled Python CLI (`--json`, `--dry-run`, `--verbose`, + `--server` override). Imports only the standard library and `requests`. +- `scripts/test_peertube.py` — offline test suite (pytest + unittest compatible); all + HTTP is mocked, zero network egress. +- Requires Python 3.8+ and `requests`. Any reachable PeerTube instance (or SepiaSearch) + works; no credentials exist or are required by default. No service is started by this + skill. diff --git a/peertube/evals/evals.json b/peertube/evals/evals.json new file mode 100644 index 0000000..fa42496 --- /dev/null +++ b/peertube/evals/evals.json @@ -0,0 +1,72 @@ +{ + "schema_version": 1, + "skill_name": "peertube", + "evals": [ + { + "id": "browse-latest-videos-json", + "prompt": "Show me the ten most recent videos on my PeerTube instance (framatube.org) as JSON.", + "expected_output": "Set PEERTUBE_SERVER=https://framatube.org (or pass --server) and run scripts/peertube videos --limit 10 --json. No login is needed: /api/v1/videos is anonymous and returns {total, data} with offset pagination (start/count, no page parameter).", + "assertions": [ + "exports PEERTUBE_SERVER or passes --server with the instance host", + "runs scripts/peertube videos with --limit 10 and --json", + "does not require login because public video listing is anonymous", + "pages with start/count offsets, never a page parameter" + ] + }, + { + "id": "search-then-detail-pipeline", + "prompt": "Search my PeerTube instance for videos about linux, then show me the full details of the best result.", + "expected_output": "Chain scripts/peertube search --query linux --json (instance-local search, searchTarget=local) to get results, extract .videos[0].uuid with jq, then run scripts/peertube video --id --json for full metadata. The video detail endpoint accepts the numeric id, UUID, or shortUUID.", + "assertions": [ + "starts with scripts/peertube search using --query", + "extracts the uuid field from the search output before the next stage", + "feeds the extracted id into scripts/peertube video --id", + "does not claim the search covered the whole fediverse since the CLI performs instance-local search" + ] + }, + { + "id": "fediverse-search-via-sepiasearch", + "prompt": "I searched my PeerTube instance for a popular video and got no results, but I know it exists on another instance. How do I search across the whole fediverse?", + "expected_output": "Instance search (searchTarget=local) only finds objects the instance knows. For fediverse-wide search, point the same CLI at SepiaSearch - the public search index that indexes public PeerTube instances and speaks the identical API shape: PEERTUBE_SERVER=https://sepiasearch.org scripts/peertube search --query ''. Follow results back to their origin instance using the account/channel host or the video url field; an instance may also support searchTarget=search-index if its admin enabled an external index.", + "assertions": [ + "explains the instance-local versus search-index/fediverse search scopes", + "uses SepiaSearch as the fediverse-wide search base host with the same API shape", + "directs following results to their origin instance via host/url fields", + "mentions searchTarget=local as the explicit instance-scope value" + ] + }, + { + "id": "oauth-client-secret-masked-login", + "prompt": "I'm writing a script that logs in to a PeerTube instance. GET /api/v1/oauth-clients/local returns client_secret as '********************************' and my subsequent POST to /users/token fails with 400 invalid client. What is going on?", + "expected_output": "Current production instances mask the client_secret in the oauth-clients/local response (the real pair reaches the web front end via its served assets). The correct flow is still: fetch /api/v1/oauth-clients/local (singular 'local'), obtain the unmasked client pair the way the instance's own front end does, then POST /api/v1/users/token with x-www-form-urlencoded fields client_id, client_secret, grant_type=password, username, password - no response_type needed. A 400 can also mean wrong credentials; the bundled CLI detects the masked secret and stops with guidance before sending a doomed token request.", + "assertions": [ + "identifies production client_secret masking as the cause of the invalid-client 400", + "names the oauth-clients/local (singular) endpoint as step one", + "lists the exact password-grant form fields including grant_type=password", + "does not treat the masked asterisk value as a usable secret" + ] + }, + { + "id": "token-persistence-and-logout-hygiene", + "prompt": "How should a PeerTube CLI store the OAuth token after login, and how do I log out properly?", + "expected_output": "Persist the token per-instance in an owner-only file (the bundled CLI uses ~/.config/peertube/token.json, override with PEERTUBE_CONFIG_DIR) recording the server URL, access token, refresh token, and absolute expires_at from the token response - lifetimes are instance-configurable, so never hard-code one. Log out with scripts/peertube logout, which calls POST /api/v1/users/revoke-token (revoking the access and refresh tokens server-side) and then deletes the local file; deleting the file alone leaves a live session. Never commit or log tokens.", + "assertions": [ + "stores the token outside the repository in an owner-only location", + "records expiry from expires_in instead of assuming a fixed lifetime", + "revokes server-side via POST /users/revoke-token before deleting the local file", + "keeps tokens per-instance because tokens are not valid across instances" + ] + }, + { + "id": "youtube-upload-not-peertube", + "prompt": "Help me upload my video to YouTube and trim the intro with ffmpeg.", + "expected_output": "This must not trigger the peertube skill: YouTube uploading and video editing are outside its read-only PeerTube API scope, and PeerTube is a different federated platform from YouTube. Use YouTube's own upload tooling (e.g. youtube-cli or YouTube Studio) and ffmpeg directly for trimming. The PeerTube skill supports no upload operation at all - it is read-only plus login/logout.", + "assertions": [ + "must not trigger peertube for YouTube uploads or video editing", + "routes the upload to YouTube's own tooling instead", + "routes the trim to ffmpeg directly", + "does not invent a peertube upload command since the bundled CLI is read-only" + ] + } + ] +} diff --git a/peertube/references/auth-and-tokens.md b/peertube/references/auth-and-tokens.md new file mode 100644 index 0000000..6b71455 --- /dev/null +++ b/peertube/references/auth-and-tokens.md @@ -0,0 +1,152 @@ +# PeerTube authentication and tokens + +How PeerTube's OAuth2 flow actually behaves on the wire, what each failure looks like, and +how to store tokens without leaking them. Every behavioral claim traces to the official +REST reference, the official quick start, or the PeerTube server source (Sources footer). +PeerTube has exactly one authenticated posture: an OAuth2 **bearer access token** minted +from per-instance client credentials. There is no API-key alternative (unlike Jellyfin) and +no header scheme beyond standard `Authorization: Bearer `. + +## Step 1 — fetch the instance's OAuth client credentials + +`GET /api/v1/oauth-clients/local` (singular `local`, not `locals`) returns the +per-instance client pair. It is anonymous — no authorization block on the operation — and +PeerTube's own web UI calls it before every login: + +```json +{ "client_id": "", "client_secret": "" } +``` + +**Production servers mask the client_secret.** The current server code returns the real +secret from this endpoint (it is the same secret the web client uses), but production +instances in recent versions respond with the secret replaced by +`"********************************"` — observed live on multiple public instances in +2026 and consistent with the reference page's own masked response example +(`client_secret: "********************************"`). Practical consequences: + +- Never persist the response of `oauth-clients/local` as if it were a working secret. +- A login attempt using the masked value fails with HTTP 400 (invalid client). This is + what you are seeing if your script fetched the client pair and the very next token + request 400s on a public instance. +- The legacy workaround mirrors what the web client does: the client pair is embedded in + the instance's front-end JavaScript, and official PeerTube tooling reads it from the + served assets when the API masks it. Treat the masked behavior as version-dependent — + always attempt the API first, then fall back to scraping the served JS bundle if the + secret comes back masked. + +The endpoint is also Host-header-guarded: the server compares the request's `Host` header +against its configured webserver hostname and answers **HTTP 403 +"Getting client tokens for host ... is forbidden"** when they disagree (proxies that +rewrite the header break clients here; the guard is skipped on test/dev instances). + +## Step 2 — the password grant + +`POST /api/v1/users/token`, `Content-Type: application/x-www-form-urlencoded`, with form +fields (names exactly as in the reference): + +| Field | Required? | Notes | +| --- | --- | --- | +| `client_id` | yes | from step 1 | +| `client_secret` | yes | from step 1 (unmasked) | +| `grant_type` | yes | `password` for login | +| `username` | yes | | +| `password` | yes | | +| `response_type` | no | the official quick-start curl sends `response_type=code`; it is absent from the current OpenAPI request schema. Sending it is harmless; omitting it works with `requests` | +| `x-peertube-otp` | conditional | request header, only when the account has 2FA enabled (server answers 401 without it) | + +```bash +curl -X POST "$BASE/users/token" \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + --data-urlencode 'client_id=' \ + --data-urlencode 'client_secret=' \ + --data-urlencode 'grant_type=password' \ + --data-urlencode 'username=' \ + --data-urlencode 'password=' +``` + +Success response fields: `access_token`, `token_type` (`"Bearer"`), `expires_in` (seconds), +`refresh_token`, and `refresh_token_expires_in` (seconds; present in the current reference +sample, `1209600` there — a sample value, not a guaranteed default). The quick-start +example shows `expires_in: 14399` (~4 hours). Sample values are not contract: instances can +configure token lifetimes server-side, so read `expires_in` from each response and schedule +refresh from it rather than hard-coding "24 hours" or any other number. + +## Refresh, revocation, and lifetime + +- **Refresh grant**: `grant_type=refresh_token` is a documented allowed value on the token + endpoint. The rendered reference does not display a `refresh_token` form-field row, so + the exact refresh request body is not fully specified in official docs; standard OAuth2 + practice (send `refresh_token` alongside the client pair) is the community-established + shape, but verify against your instance's version before relying on it. +- **Revocation**: `POST /api/v1/users/revoke-token` with `Authorization: Bearer `, + no body, returns HTTP 200 and revokes the access token **and** its associated refresh + token, destroying the session. This is the correct "logout" operation: revoke before + discarding a stored token. +- **Lifetimes**: no official page documents default lifetime values or the server config + keys that change them; the only official evidence is the sample `expires_in: 14399` / + `refresh_token_expires_in: 1209600` (~4 h / ~14 d). Server operators can adjust token + lifetimes via their production config (all options documented as living in + `config/default.yaml` overridable by `production.yaml`), so treat expiry as per-instance + and honor `expires_in`. + +## Error signatures on the wire + +| Symptom | Status | Meaning / response | +| --- | --- | --- | +| Bad client_id/client_secret, or the masked secret, or wrong username/password | `400` on `POST /users/token` | Reference documents 400 for invalid client or credentials; bodies are RFC7807-style (`application/problem+json` with `type`, `title`, `status`, `detail`, sometimes `code`) | +| 2FA enabled, no `x-peertube-otp` header | `401` on `POST /users/token` | header must be supplied on the token request | +| Expired/revoked token on any authenticated call | `401` | re-run the password grant (or refresh) | +| Wrong `Host` header reaching `oauth-clients/local` | `403` | proxy/header rewriting problem, not auth | +| Rate limit exceeded | `429` | all endpoints are rate-limited; the token endpoint is tighter than most (documented sample: 15 calls per 5 minutes). Inspect `Retry-After` (seconds) and `X-RateLimit-Limit` / `X-RateLimit-Remaining` / `X-RateLimit-Reset` (Unix timestamp) and back off | +| Connection refused / DNS failure | no HTTP response | transport failure — classify separately from API errors; usually `PEERTUBE_SERVER` is wrong or unreachable | + +Anonymous-read endpoints (videos, search, channels, `/config`, `/config/about`, +`/server/stats`) need no token at all. Authenticated-only calls include `/users/me`, +`/users/me/videos`, and any state-changing operation. The API answers `401` when a call +needs a token you did not send. + +## Token persistence hygiene + +PeerTube's docs do not prescribe storage mechanics, so a CLI should follow these +sanctioned-by-logout-support practices: + +1. **Store under the user's own profile, not in the repo.** The bundled CLI defaults to + `~/.config/peertube/token.json` (override with `PEERTUBE_CONFIG_DIR` for tests). Never + write tokens into a working tree, a shell history, or an eval manifest. +2. **Restrict file permissions.** Create the directory and file so only the owner can read + the token file (e.g. `os.makedirs(..., mode=0o700)` and `0o600` on the file). +3. **Persist the refresh token alongside the access token and the server base URL**, plus + the absolute `expires_at` computed from `expires_in`. A token file is only valid for + the instance it was minted by — re-authenticate when `PEERTUBE_SERVER` changes. +4. **Refresh before expiry; fall back to password re-grant.** Because refresh-request + semantics are underspecified in official docs, treat refresh as an optimization: try + `grant_type=refresh_token`, and on any failure re-run the password grant. +5. **Revoke on logout.** `POST /users/revoke-token` invalidates both tokens server-side, + then delete the local file. Deleting the file alone leaves a live session behind. +6. **Never commit or log tokens.** Examples everywhere in this skill use + ``-style placeholders. If a token file ever lands in a diff, revoke it — + deleting the file does not invalidate the session. +7. **Multi-instance note**: one token file per server URL (or include the server in the + file) avoids "works on instance A, 401 on instance B" confusion when switching + `PEERTUBE_SERVER`. + +## Detection and headers for API clients + +- API responses carry `x-powered-by: PeerTube` and `/api/*` is CORS-enabled; HTML pages + include ``; NodeInfo is exposed at + `/nodeinfo/2.0.json`. Any of these distinguishes a PeerTube instance from other servers. +- No special `User-Agent` is required. Use `Accept: application/json`. + +## Sources + +- https://docs.joinpeertube.org/api-rest-reference.html (Session: getOAuthClient, + getOAuthToken, revokeOAuthToken; Errors; Rate-limits; CORS; Config; Stats) +- https://docs.joinpeertube.org/api/rest-getting-started (client fetch, password grant + curl, token response example, instance detection) +- https://docs.joinpeertube.org/maintain/configuration (config file layering) +- https://github.com/Chocobozzz/PeerTube/blob/develop/support/doc/api/openapi.yaml + (generated OpenAPI spec; openapi-generator clients) +- https://raw.githubusercontent.com/Chocobozzz/PeerTube/develop/server/core/controllers/api/oauth-clients.ts + (Host-header guard; response construction) +- Live anonymous probes of public instances (`oauth-clients/local` masking, 400 on token + misuse), 2026-08-29. diff --git a/peertube/references/endpoint-catalog.md b/peertube/references/endpoint-catalog.md new file mode 100644 index 0000000..24bd6f2 --- /dev/null +++ b/peertube/references/endpoint-catalog.md @@ -0,0 +1,131 @@ +# PeerTube endpoint catalog for CLI clients + +The read surface of the PeerTube REST API with exact parameter names, response shapes, and +pagination semantics — everything a CLI needs to list, filter, and page through videos, +channels, accounts, and instance metadata. Base path: `/api/v1` on any instance +(`https:///api/v1`). Sources footer cites the official reference; a few +shapes were additionally confirmed by live anonymous probes (noted inline). + +## The one pagination model: start/count offsets + +Every collection endpoint uses **offset pagination**: query params `start` (integer >= 0) +and `count` (1–100, **default 15**). There is no `page` parameter anywhere in the current +API — a client sending `page=` silently gets default paging while believing it paginated +(this bit the original bundled CLI). Responses wrap as: + +```json +{ "total": 23792, "data": [ /* resource objects */ ] } +``` + +Loop by advancing `start` by the number of rows received until `start >= total` (or an +empty page). `skipCount=true` on video collections/search omits the `total` computation — +faster, but then you must stop on the first short/empty page. Max `count` per request is +100; a `count` above the allowed range is rejected. + +## Videos + +| Endpoint | Auth | Notes | +| --- | --- | --- | +| `GET /videos` | anonymous | instance-wide video list; filters below | +| `GET /videos/{id}` | anonymous | full detail; `{id}` accepts **numeric id, UUIDv4, or shortUUID** | +| `GET /videos/{id}/comment-threads` | anonymous | top-level comment threads; `start`, `count`, `sort` in {-createdAt, -totalReplies}; response `{total, totalNotDeletedComments, data}` | + +- The comments route is **`/comment-threads`** (hyphenated). `/comments` and + `/commentthreads` are not the route (probes: `/comments` 400s on current servers; the + OpenAPI shows `/comment-threads`). A newer `/videos/{id}/comments/{commentId}/replies` + route (v8.3 changelog) fetches replies, not top-level threads. +- Listing filters (current exact names): `start`, `count`, `sort`, `categoryOneOf`, + `tagsOneOf`, `tagsAllOf`, `languageOneOf`, `licenceOneOf`, `nsfw`, `nsfwFlagsIncluded`, + `nsfwFlagsExcluded`, `isLive`, `isLocal`, `host`, `skipCount`, `search`, plus + admin-only `include`/`privacyOneOf`/`stateOneOf` (>=8.2)/`autoTagOneOf` (>=6.2) and + file-format filters `hasHLSFiles`/`hasWebVideoFiles`. +- Sort values: `name`, `-duration`, `-createdAt`, `-publishedAt`, `-views`, `-likes`, + `-comments`, `-trending`, `-hot`, `-best`. +- List-item shape (probe-confirmed field names): `id`, `uuid`, `shortUUID`, `url`, `name`, + `category{id,label}`, `licence{id,label}`, `language{id,label}`, `privacy{id,label}`, + `nsfw`, `truncatedDescription`, `duration` (**seconds** — sample `1419` is ~23.6 min), + `views`, `likes`, `dislikes`, `comments`, `publishedAt`/`originallyPublishedAt`/`createdAt` + (ISO-8601), `isLocal`, `isLive`, thumbnail/preview `path`s, and actor summaries: + `account{id,name,displayName,host,url,avatars[]}`, + `channel{id,name,displayName,host,url,avatars[]}`. +- `account`/`channel` `host` tells you the **origin instance** of a federated video — on a + search-index result this is how you find where the video actually lives. +- Detail adds full `description`, `files[]`/`streamingPlaylists[]` (resolutions, + `fileUrl`/`fileDownloadUrl`, `metadataUrl`s), `commentsEnabled`, `downloadEnabled`, + `trackerUrls`, `support`, `tags`, `scheduledUpdate` for scheduled/live videos. + +## Channels and accounts + +| Endpoint | Auth | Notes | +| --- | --- | --- | +| `GET /video-channels` | anonymous | **does exist** (current reference): lists the instance's channels, `start`/`count`/`sort`, `{total,data}` | +| `GET /video-channels/{channelHandle}` | anonymous | handle format `my_username` or `my_username@example.com` (`name@host` for remote channels) | +| `GET /video-channels/{channelHandle}/videos` | anonymous | channel's videos, standard video filters + offset pagination | +| `GET /accounts/{name}` | anonymous | account actor; 404 for unknown; `name` accepts `chocobozzz` or `chocobozzz@example.org` | +| `GET /accounts/{name}/videos` | anonymous | account's videos, offset pagination | +| `GET /accounts/{name}/video-channels` | anonymous | an account's channels | +| `GET /search/video-channels` | anonymous | see search-and-discovery.md | + +Channel object fields include `name`, `displayName`, `host`, `url`, `avatars`, +`followersCount` (subscribers), `videosCount` — but note the **global** `/video-channels` +list rows additionally observed carrying `videosCount`/`followersCount` per channel in +list responses (probe 2026-08-29). Historical route drift: pre-1.0 `/videos/channels/*` +routes became `/video-channels/*` and `/videos/accounts/{id}/channels` became +`/accounts/{id}/video-channels` (changelog, v1.0.0-beta.4) — ancient wrappers still using +the old shapes will 404. + +## Instance metadata (all anonymous, all public) + +| Endpoint | Returns | +| --- | --- | +| `GET /config` | public runtime configuration: `client{}`, `defaults{}`, `webadmin{}`, and an `instance{}` block with `name`, `shortDescription`, classifications, customization, avatars/banners | +| `GET /config/about` | `{instance:{name, shortDescription, description, terms, codeOfConduct, hardwareInformation, administrationInformation, maintenanceInformation, businessInformation, languages, categories, banners}}` | +| `GET /server/stats` | instance counters: `totalUsers`, `totalLocalVideos`, `totalLocalVideoViews`, `totalLocalVideoDownloads`, `totalLocalVideoComments`, `totalVideos`, `totalVideoComments`, `totalLocalVideoChannels`, `totalLocalDailyActiveVideoChannels`, `totalLocalVideoChannels`, `totalLocalVideoPlaylists`, moderation/registration counters, activity-processing stats. Public and cached by the server. | +| `GET /nodeinfo/2.0.json` | standard NodeInfo document (software name/version, usage counts) — handy for instance detection | + +**Naming trap:** the stats operation is titled "Get instance stats" but the canonical +current path is **`/server/stats`** (there is no `/instance/stats`), while the config +endpoints are **`/config`** and **`/config/about`** (there is no `/instance/config` or +`/instance/about`). Mixed naming is current reality, not a docs bug. A CLI's `server` / +`info` command should compose `/config/about` + `/server/stats` to give name, description, +and user/video/view counts in one screenful. + +## My user (OAuth2 required) + +| Endpoint | Notes | +| --- | --- | +| `GET /users/me` | identity + preferences: `id`, `username`, `email`, `role{id,label}`, `videoQuota`, `videoQuotaDaily`, `account{}`, `videoChannels[]`, `twoFactorEnabled`, theme/NSFW/p2p preferences, `createdAt`. The current reference sample is rendered as an array; every live server returns a **single user object** — clients should tolerate both. | +| `GET /users/me/videos` | `{total, data}` of your uploads with the standard video-list fields and filters (`start`, `count`, `sort`, privacy/scope filters) | + +The `role` block is `{id, label}` (e.g. `{id: 1, label: "User"}`); `videoQuota` is bytes. +Channel rows inside `videoChannels` carry the same `name`/`displayName`/`host` actor shape +used everywhere else. + +## Rate limits (all endpoints) + +Default server-side limiter: **50 calls per 10 seconds** per IP across `/*` (the token +endpoint is documented at a tighter 15 per 5 minutes in its operation docs; administrators +can customize all values). On exhaustion you get **HTTP 429** with +`X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset` (Unix timestamp) and +`Retry-After` (seconds). A CLI should read `Retry-After` and back off; aggressive parallel +listing (count=100 × many pages) on a small instance will trip the limiter. + +## Error bodies + +Errors use RFC7807-style `application/problem+json` documents with `type`, `title`, +`status`, `detail`, and sometimes a `code`. Unknown routes on current servers typically +answer 400 (not the classic 404) with an `error` body — check the body, not just the +status, when a route mysteriously "doesn't exist". + +## Sources + +- https://docs.joinpeertube.org/api-rest-reference.html (getVideos, getVideo, + getVideoChannels, getVideoChannel, getVideoChannelVideos, getAccount, getAccountVideos, + searchChannels, getConfig, getAbout, getInstanceStats, getUserInfo, comment-threads + operations; Errors and Rate-limits sections) +- https://docs.joinpeertube.org/api/rest-getting-started (pagination/filter basics, + instance detection via NodeInfo / x-powered-by / og:platform) +- https://docs.joinpeertube.org/CHANGELOG (route renames v1.0.0-beta.4; v8.2 stateOneOf; + v8.3 comment routes) +- Live anonymous probes on a public instance (list shapes, channel list fields, + `/config/about`, `/server/stats`, `/comment-threads` vs `/comments` status), 2026-08-29. diff --git a/peertube/references/gotchas-field-guide.md b/peertube/references/gotchas-field-guide.md new file mode 100644 index 0000000..bcc7f98 --- /dev/null +++ b/peertube/references/gotchas-field-guide.md @@ -0,0 +1,125 @@ +# PeerTube gotchas field guide + +Failure signatures and behavioral traps, distilled from the official docs, the server +source, and live probes. Each entry: symptom → cause → what to do. + +## Instance plurality (the big one) + +- **Symptom**: same CLI command works on one instance and 400s/401s/empty-results on + another; or a token that worked on instance A 401s on instance B. +- **Cause**: PeerTube is federated software, not a single API. Every instance is an + independent deployment with its own rules, allowances, moderation policy, enabled + features (NSFW policy, search-index support, registration, transcoding) and its own user + accounts and OAuth tokens. A token minted by instance A is meaningless to instance B; + instance B may have closed registrations, disabled uploads, or set its own NSFW default. +- **Do**: always configure the instance host per operation (`PEERTUBE_SERVER` or + `--server`); keep per-instance token files; never assume an account or video exists on a + different instance. Federated content viewed on instance X still **belongs** to the + origin instance (`channel.host` / `account.host` / video `url` tell you which). + +## Search scope confusion + +- **Symptom**: "search across the fediverse" expectations return only a handful of local + results; or results reference videos the instance doesn't host. +- **Cause**: `searchTarget` has two scopes: `local` (instance-known objects only) and + `search-index` (external fediverse index, admin-enabled). Omitting the parameter gives + the instance's own scope on current servers (observed), not the fediverse. +- **Do**: pass `searchTarget=local` explicitly for instance scope; use SepiaSearch + (`https://sepiasearch.org/api/v1/search/videos`) for fediverse-wide scope. Index results + point at origin instances — follow `channel.host`/`url` rather than expecting the + queried instance to serve them. + +## Pagination: `start`/`count`, never `page` + +- **Symptom**: client pages with `page=1&count=15` and gets identical results forever. +- **Cause**: the API has no `page` parameter; unknown params are ignored, so `page=1` + requests silently return the first `count` rows every time. +- **Do**: advance `start` by the page size until `start >= total` or an empty page. Max + `count` is 100 (higher values are rejected). `skipCount=true` trades the `total` field + for speed — then you must stop on the first short page. + +## Comment route spelling + +- **Symptom**: fetching comments with `/videos/{id}/comments` or + `/videos/{id}/commentthreads` returns 400 (current servers answer 400, not 404, for bad + routes — see below) while other endpoints work. +- **Cause**: the route is `GET /videos/{id}/comment-threads` (hyphenated). The v8.3 + `/comments/{commentId}/replies` route is for replies, not top-level threads. +- **Do**: use `/comment-threads` with `start`/`count`/`sort=-createdAt|-totalReplies`. + +## Instance metadata endpoint names + +- **Symptom**: `/instance/stats`, `/instance/about`, `/instance/config` all 400/404. +- **Cause**: mixed current naming: stats live at **`/server/stats`** (operation *titled* + "Get instance stats"), about at **`/config/about`**, config at **`/config`**. +- **Do**: compose `/config/about` + `/server/stats` for a full instance picture. + +## oauth-clients/local secret masking + +- **Symptom**: `GET /oauth-clients/local` returns + `"client_secret": "********************************"`; the following token request 400s + with invalid_client. +- **Cause**: current production servers mask the secret in this response (the value is + still delivered to the web client via served front-end assets; the API response masks + it). Older instances/versions return the real secret. +- **Do**: detect the masked value; if masked, obtain the client pair from the instance's + served front-end JS (the same source its own web UI uses) before the token request. Never + persist the masked string as a secret. The endpoint is also Host-header-guarded (403 if + the `Host` header disagrees with the configured webserver hostname — mind reverse + proxies). + +## Auth error signatures + +| Status | Where | Meaning | +| --- | --- | --- | +| 400 on `POST /users/token` | invalid client pair (including the masked-secret case) or wrong credentials | RFC7807-style `application/problem+json` body; check `detail` | +| 401 on `POST /users/token` | account has 2FA and no `x-peertube-otp` header supplied | supply OTP header | +| 401 on authenticated GETs | token expired/revoked/malformed, or missing | re-run password grant | +| 403 on `oauth-clients/local` | Host-header mismatch (proxy misconfiguration) | fix the proxy/Host | +| 429 anywhere | rate limit (default 50 req/10 s; token endpoint tighter) | read `Retry-After` + `X-RateLimit-*` headers, back off | +| 400 on unknown routes | current servers answer 400 with an error body for unrecognized API routes | read the body; the classic "404 means missing route" assumption misleads here | +| connection errors | wrong/unreachable `PEERTUBE_SERVER` | no HTTP response at all; classify as transport failure | + +## Shape and value traps + +- **duration is seconds** (integer). Sample list value `1419` = 23:39, not milliseconds. +- **ids are triple**: numeric `id`, `uuid` (UUIDv4), and `shortUUID` — all three are + accepted by `/videos/{id}` and family; `uuid` is the safest portable choice in scripts. +- **`users/me` sample is an array in the docs**; live servers return a single object. + Tolerate both when writing generic parsers. +- **`role` is an object** `{id, label}` on `/users/me` — don't stringify the dict. +- **`videoQuota` is bytes** (large integer). +- **`{total, data}` everywhere**: collections never wrap in `{"videos": []}` at the API + layer (the bundled CLI adds that key in its JSON output; know which layer you're + reading). +- **`nsfw` filter is a string** (`"true"`/`"false"`) in query params. +- **filter names end in `OneOf`/`AllOf`** (`categoryOneOf`, `tagsAllOf`, ...); bare + `category=` from old wrappers is ignored silently. +- **federated results**: a video listed on instance X may be hosted on instance Y + (`account.host`/`channel.host`). Views/likes counters are local-ish and eventually + consistent across the federation — don't expect exact global numbers. + +## Version drift + +- Docs reference page currently identifies PeerTube **8.1.0** while the changelog already + carries 8.3.0 material — instance versions vary; validate optional parameters + (`stateOneOf` >= 8.2, `autoTagOneOf` >= 6.2) before relying on them. +- Historical renames worth knowing when reading old code: `/videos/channels/*` → + `/video-channels/*`, `/videos/accounts/{id}/channels` → `/accounts/{id}/video-channels` + (v1.0.0-beta.4). +- Refresh-token request fields are underspecified in official docs; don't build + refresh-critical logic without testing against your target instance. + +## Sources + +- https://docs.joinpeertube.org/api-rest-reference.html (operation pages: searchVideos, + getVideos, comment-threads, getOAuthToken, revokeOAuthToken, getInstanceStats; Errors, + Rate-limits sections) +- https://docs.joinpeertube.org/api/rest-getting-started +- https://docs.joinpeertube.org/use/search (scope semantics) +- https://docs.joinpeertube.org/admin/configuration (global-search admin enablement) +- https://docs.joinpeertube.org/CHANGELOG (route renames, version additions) +- https://raw.githubusercontent.com/Chocobozzz/PeerTube/develop/server/core/controllers/api/oauth-clients.ts + (Host guard) +- Live anonymous probes (secret masking, search default scope, route status codes, + response shapes), 2026-08-29. diff --git a/peertube/references/search-and-discovery.md b/peertube/references/search-and-discovery.md new file mode 100644 index 0000000..00b5f08 --- /dev/null +++ b/peertube/references/search-and-discovery.md @@ -0,0 +1,146 @@ +# PeerTube search: instance-local vs the fediverse-wide index + +PeerTube search has two distinct scopes, and confusing them is the single most common +mistake clients make. This file pins down exactly what each scope does, what SepiaSearch +is, and which one the bundled CLI performs. + +## The two scopes + +`GET /api/v1/search/videos` accepts `searchTarget` with exactly two documented values: + +| `searchTarget` | Scope | What you get | +| --- | --- | --- | +| `local` | platform/instance search | Results known to the platform you are querying: its own videos plus objects it has discovered/federated from instances it follows. Same behavior as the instance's web UI search box. | +| `search-index` | global/fediverse search | Results served through an **external search index** configured by the instance administrator. The result set is not scoped to objects your instance knows. The reference warns these results come from a third-party service, and the instance may not yet know (have copies of) the returned objects. | + +Facts that matter operationally: + +- `remote` is **not** a current `searchTarget` value (it appears in old blog posts and + older wrappers); the current enum is `local` | `search-index`. +- The current reference does not state what happens when `searchTarget` is omitted. + Observed behavior on a public instance (2026-08-29): omitting it returned local results + identical to `searchTarget=local`, i.e. **the default scope is the instance's own + index**, not the fediverse. Do not assume otherwise; if you need the instance's results, + pass `searchTarget=local` explicitly, and if you want the fediverse, use a search-index + host (below) rather than an undocumented default. +- `searchTarget=search-index` only works when the administrator has enabled and configured + an external search index (admin config section "Global search"); instances without one + cannot serve index results. Errors when the index is unavailable surface as HTTP 500 on + search endpoints. +- Index results may reference videos your instance has never federated. The official + recommendation for consuming them: if URI search is enabled, fetch the result's URL into + your instance first, then use the classic REST endpoint; otherwise fetch from or redirect + to the **origin instance** (every result carries its origin in `account`/`channel.host` + and the video `url`). + +## SepiaSearch: the fediverse-wide index + +[SepiaSearch](https://sepiasearch.org) is Framasoft's public search index for PeerTube: a +separately hosted service that crawls and indexes public PeerTube instances (its front +page advertises ~1,700 sites indexed) and exposes **the same REST API shape** under its own +base URL: + +``` +GET https://sepiasearch.org/api/v1/search/videos?search=&start=0&count=15 +``` + +Verified live (2026-08-29): the response is the standard `{total, data: [...]}` collection +of PeerTube-shaped video objects (`uuid`, `shortUUID`, `name`, `category`, `language`, +`privacy`, `publishedAt`, `account`, `channel`, `views`, `duration`, plus a `score` field +the instance endpoints do not return). Consequences: + +- A client only needs to swap the base host from an instance to `https://sepiasearch.org` + to get fediverse-wide search — same parameters, same pagination, same parsing. +- There is no documented indexing-latency guarantee; freshly published videos may take an + unspecified time to appear. Treat indexing lag as variable. +- SepiaSearch is a search service, not a video host: play/upload URLs in results point at + the origin instances. +- PeerTube administrators may instead configure their own index URL (Framasoft also + publishes one at `https://search.joinpeertube.org/` built on the same idea); that is what + `searchTarget=search-index` talks to on such instances. SepiaSearch is simply the + well-known public instance of this concept. +- SepiaSearch results are not moderated by anyone you are talking to; the official + documentation explicitly warns the index content is not moderated. + +## Search endpoint catalog + +| Endpoint | Notes | +| --- | --- | +| `GET /api/v1/search/videos` | required `search`; `searchTarget`, `start`, `count` (1–100, default 15), `sort`, plus video filters below | +| `GET /api/v1/search/video-channels` | required `search`; optional `handles`, `host`, `searchTarget`, `start`, `count`, `sort`; returns 500 if the search index is unavailable | + +### Sort values (search + video listing) + +`name`, `-duration`, `-createdAt`, `-publishedAt`, `-views`, `-likes`, `-comments`, +`-trending`, `-hot`, `-best`. The last three are relevance/popularity orders computed by +the instance (hot/trending window definitions are instance-side). + +### Filter parameters (exact names) + +`categoryOneOf`, `licenceOneOf`, `languageOneOf`, `tagsOneOf`, `tagsAllOf`, `nsfw` +(`"true"`/`"false"` string), `nsfwFlagsIncluded`/`nsfwFlagsExcluded`, `isLive`, +`durationMin`/`durationMax` (seconds), `startDate`/`endDate` and +`originallyPublishedStartDate`/`originallyPublishedEndDate` (ISO dates), `host`, +`uuids`, `skipCount` (`true` avoids computing `total`), plus admin-only +`autoTagOneOf` (>=6.2), `include` (bitmask), `privacyOneOf`, `stateOneOf` (>=8.2). +`category` (without `OneOf`) is not the current parameter name — older wrappers using it +silently drop the filter. + +## Which scope does the bundled CLI use? + +The bundled `scripts/peertube` performs **instance-local search only**: it issues +`GET /search/videos` with `searchTarget=local` against `PEERTUBE_SERVER` and never claims +fediverse-wide coverage. For fediverse-wide search, point the same commands at SepiaSearch +(`PEERTUBE_SERVER=https://sepiasearch.org scripts/peertube search --query ...`) — the CLI +is instance-agnostic by design, and SepiaSearch speaks the same API. The CLI's `search +--help` text states its scope so nobody mistakes local results for the whole fediverse. + +## Worked recipes + +### Instance-local search, then full video detail + +```bash +BASE="https://" +curl -G "$BASE/api/v1/search/videos" \ + --data-urlencode 'search=' \ + --data-urlencode 'searchTarget=local' \ + --data-urlencode 'start=0' --data-urlencode 'count=10' +# data[].uuid / shortUUID / id all work as the {id} path parameter below +curl "$BASE/api/v1/videos/" +``` + +### Fediverse-wide search via SepiaSearch + +```bash +curl -G 'https://sepiasearch.org/api/v1/search/videos' \ + --data-urlencode 'search=' \ + --data-urlencode 'start=0' --data-urlencode 'count=10' +# follow a result to its origin instance: +# data[0].url / data[0].channel.host tell you where the video lives +``` + +### Local search with filters and relevance sort + +```bash +curl -G "$BASE/api/v1/search/videos" \ + --data-urlencode 'search=' \ + --data-urlencode 'searchTarget=local' \ + --data-urlencode 'sort=-views' \ + --data-urlencode 'durationMin=300' \ + --data-urlencode 'languageOneOf=en' \ + --data-urlencode 'count=20' +``` + +## Sources + +- https://docs.joinpeertube.org/api-rest-reference.html (searchVideos, searchChannels + operations: searchTarget enum, parameter tables, third-party-index warning) +- https://docs.joinpeertube.org/use/search (platform search vs global search semantics) +- https://docs.joinpeertube.org/admin/configuration (Global search: external index + configuration, search.joinpeertube.org, non-moderation warning) +- https://sepiasearch.org/ (what SepiaSearch is; indexed-site count) +- https://sepiasearch.org/api/v1/search/videos?search=peertube&start=0&count=1 + (live response shape, 2026-08-29) +- https://docs.joinpeertube.org/CHANGELOG (version-drift notes) +- Live anonymous probe of a public instance's `/search/videos` with and without + `searchTarget` (default-scope observation), 2026-08-29. diff --git a/peertube/references/worked-recipes.md b/peertube/references/worked-recipes.md new file mode 100644 index 0000000..1e96987 --- /dev/null +++ b/peertube/references/worked-recipes.md @@ -0,0 +1,151 @@ +# Worked recipes and CLI workflows + +Multi-step workflows for the bundled `scripts/peertube` CLI, plus raw curl/jq equivalents. +Every stage's output field names and JSON types are what the next stage consumes — the +pipelines are proven by the CLI's offline test suite. `PEERTUBE_SERVER` must be exported +for all commands (any instance host works; SepiaSearch works too — see below). + +```bash +export PEERTUBE_SERVER="https://" # e.g. https://tilvids.com +``` + +## CLI command map + +| Command | Does | Auth needed | +| --- | --- | --- | +| `server` | instance name + description (`/config/about`) + stats (`/server/stats`) | no | +| `videos` | latest instance videos (`/videos`, offset paging) | no | +| `search --query Q` | **instance-local** search (`/search/videos`, `searchTarget=local`) | no | +| `video --id ID` | full video detail (id, UUID, or shortUUID) | no | +| `comments --id ID` | top-level comment threads (`/comment-threads`) | no | +| `channels` | instance channel list (`/video-channels`) | no | +| `channel --handle H` | one channel's metadata + recent uploads | no | +| `account --name N` | account metadata (`/accounts/{name}`) | no | +| `me` | your profile (`/users/me`) | yes | +| `my-videos` | your uploads (`/users/me/videos`) | yes | +| `login` | OAuth2 password grant → persists token file | yes (credentials) | +| `logout` | revoke token server-side + delete token file | yes (token) | + +Global flags: `--json` (machine output), `--dry-run` (print the request plan, zero +network), `--limit N` (page size, max 100), `--offset N` (start offset). All flags work +before or after the subcommand. `--help` and `--dry-run` never require credentials. + +## Recipe 1 — browse what's new, then inspect one video + +```bash +scripts/peertube videos --limit 5 --json | jq -r '.videos[] | [.name, .uuid, .duration] | @tsv' +UUID=$(scripts/peertube videos --limit 1 --json | jq -r '.videos[0].uuid') +scripts/peertube video --id "$UUID" --json | jq '{name, description, views, likes, url}' +``` + +`videos` emits `{"total": , "videos": [...each raw video object with uuid/name/ +duration/views/publishedAt/channel/account...]}`; `video` emits the raw detail object +(fields include `description`, `files[]`, `commentsEnabled`). + +## Recipe 2 — instance-local search, then pull the description + +```bash +scripts/peertube search --query "linux" --limit 10 --json | jq -r '.videos[0].uuid' +scripts/peertube search --query "linux" --limit 5 --json \ + | jq -r '.videos[] | select(.language.label == "English") | .name' +# detail for the top hit: +scripts/peertube video --id "$(scripts/peertube search --query linux --limit 1 --json | jq -r '.videos[0].uuid')" --json +``` + +Search results are the same video-object shape as `videos` (plus nothing missing that the +detail call needs — `uuid` is always present). To search the **whole fediverse** instead of +one instance, point the same CLI at SepiaSearch: + +```bash +PEERTUBE_SERVER="https://sepiasearch.org" scripts/peertube search --query "linux" --limit 10 +``` + +## Recipe 3 — channels: find the busy ones, then page their uploads + +```bash +scripts/peertube channels --json | jq -r '.channels[] | [.displayName, .name, .host, .videosCount, .followersCount] | @tsv' \ + | sort -t$'\t' -k4,4nr | head +# page through a channel's uploads with offsets (no page param exists): +scripts/peertube channel --handle "framasoft@framatube.org" --limit 100 --offset 0 --json | jq -r '.videos[].name' +scripts/peertube channel --handle "framasoft@framatube.org" --limit 100 --offset 100 --json | jq -c '{returned: (.videos | length), total}' +``` + +Handles accept `name` (local) or `name@host` (remote). The offset loop is the only +pagination mechanism — stop when `returned` is 0 or `offset >= total`. + +## Recipe 4 — log in, check your quota, upload-aware housekeeping, log out + +```bash +scripts/peertube login --username "" # prompts for password (hidden) +scripts/peertube me --json | jq '{username, role: .role.label, quota_bytes: .videoQuota}' +scripts/peertube my-videos --limit 100 --json | jq -r '.videos[] | [.name, .privacy.label, .duration] | @tsv' +scripts/peertube logout # revokes server-side + deletes local file +``` + +The token file lands in `~/.config/peertube/token.json` (owner-only permissions; +`PEERTUBE_CONFIG_DIR` overrides the directory for tests). It records the server URL, +access token, refresh token, and absolute `expires_at`; the CLI re-authenticates if the +server changes or the token is expired. `login --dry-run --json` previews the token +request (fields only — no secret values) without network. + +## Recipe 5 — instance report card (compose three anonymous endpoints) + +```bash +scripts/peertube server --json \ + | jq '{name: .instance.name, description: .instance.shortDescription, + local_videos: .stats.totalLocalVideos, total_videos: .stats.totalVideos, + users: .stats.totalUsers, views: .stats.totalLocalVideoViews}' +``` + +Equivalent raw curl: `/api/v1/config/about` for identity, `/api/v1/server/stats` for the +counters (note: the stats path is `/server/stats`, not `/instance/stats`). + +## Recipe 6 — jq processing patterns + +```bash +# TSV table of the five most-viewed local videos +scripts/peertube videos --limit 100 --json \ + | jq -r '.videos | sort_by(-.views)[:5][] | [.name, .views, .channel.displayName] | @tsv' + +# Count videos per origin host on a search-index-style result set +PEERTUBE_SERVER="https://sepiasearch.org" scripts/peertube search --query "peertube" --limit 100 --json \ + | jq -r '.videos | group_by(.channel.host) | map({host: .[0].channel.host, n: length}) | sort_by(-.n)[] | "\(.n)\t\(.host)"' + +# Comments of a video, flattening thread counts +scripts/peertube comments --id "" --json | jq '{total, total_not_deleted: .totalNotDeletedComments, threads: (.threads | length)}' + +# Verify the request plan before running it live (zero network) +scripts/peertube --dry-run --json search --query "test" | jq '{path, params: (.params | keys)}' +``` + +`--dry-run` output shape: `{"dry_run": true, "method": "GET", "url": "...", "path": +"/api/v1/...", "params": {...}}` (token requests additionally carry `"form"` keys without +values). Every handler's dry-run plan uses the same keys, so one jq pattern audits any +command. + +## Raw curl equivalents (auth chain end-to-end) + +```bash +BASE="$PEERTUBE_SERVER/api/v1" +CLIENT_ID=$(curl -sS "$BASE/oauth-clients/local" | jq -r .client_id) +# NOTE: production instances mask client_secret ("****...") in this response; if masked, +# obtain the secret as the web client does (served front-end assets) before proceeding. +curl -sS -X POST "$BASE/users/token" \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + --data-urlencode "client_id=$CLIENT_ID" \ + --data-urlencode 'client_secret=' \ + --data-urlencode 'grant_type=password' \ + --data-urlencode 'username=' \ + --data-urlencode 'password=' +ACCESS_TOKEN="" +curl -sS -H "Authorization: Bearer $ACCESS_TOKEN" "$BASE/users/me" +``` + +## Sources + +- https://docs.joinpeertube.org/api/rest-getting-started (auth chain, pagination basics) +- https://docs.joinpeertube.org/api-rest-reference.html (endpoint parameter tables and + response shapes referenced per recipe) +- https://sepiasearch.org/api/v1/search/videos (fediverse-wide search base URL) +- Field names/types corroborated by live anonymous probes and the CLI's offline mocked + tests, 2026-08-29. diff --git a/peertube/scripts/peertube b/peertube/scripts/peertube new file mode 100755 index 0000000..c0f7d76 --- /dev/null +++ b/peertube/scripts/peertube @@ -0,0 +1,821 @@ +#!/usr/bin/env python3 +"""peertube — browse PeerTube federated video from the terminal. + +Read-only client for any PeerTube instance's REST API, plus OAuth2 login. +Commands: server, videos, video, search, comments, channels, channel, +account, me, my-videos, login, logout. Set PEERTUBE_SERVER (or pass +--server) to choose the instance; point it at https://sepiasearch.org +for fediverse-wide search — the API shape is identical. `--json` emits +machine-readable output, `--dry-run` prints the exact request plan with +zero network activity, and `--help` works without credentials. +""" + +import argparse +import getpass +import json +import os +import sys +import time +import warnings +from typing import Any, Dict, List, Optional, Tuple + +warnings.simplefilter("ignore") + +import requests + +ENV_SERVER = os.getenv("PEERTUBE_SERVER", "") +ENV_CONFIG_DIR = os.getenv("PEERTUBE_CONFIG_DIR", "") +API_BASE = "/api/v1" +DEFAULT_CONFIG_DIR = "~/.config/peertube" +TOKEN_FILE_NAME = "token.json" +MAX_COUNT = 100 +DEFAULT_COUNT = 15 + +GLOBAL_FLAGS: Dict[str, Any] = {"json": False, "dry_run": False, "quiet": False, "verbose": False} + + +def die(message, exit_code=1): + print(f"Error: {message}", file=sys.stderr) + sys.exit(exit_code) + + +def warn(message): + print(f"Warning: {message}", file=sys.stderr) + + +def emit(human, data): + if GLOBAL_FLAGS.get("json"): + print(json.dumps(data, default=str)) + else: + print(human) + + +def log(message): + if GLOBAL_FLAGS.get("verbose") and not GLOBAL_FLAGS.get("json"): + print(message, file=sys.stderr) + + +def _preparse_global_flags(argv): + """Pull global flags out so they work before or after the subcommand. + + Boolean flags are recorded as True; --server consumes its value. Anything + else passes through to the subparsers untouched.""" + bools = {"--json", "--dry-run", "--quiet", "--verbose"} + flags, filtered = {}, [argv[0]] + i = 1 + while i < len(argv): + arg = argv[i] + if arg in bools: + flags[arg.lstrip("-").replace("-", "_")] = True + i += 1 + elif arg == "--server": + if i + 1 >= len(argv): + die("--server requires a value (the instance URL).") + flags["server"] = argv[i + 1] + i += 2 + elif arg in ("--help", "-h"): + return flags, argv + elif arg == "--": + filtered.extend(argv[i:]) + break + else: + filtered.append(arg) + i += 1 + return flags, filtered + + +def default_config_dir(): + if ENV_CONFIG_DIR: + return os.path.expanduser(ENV_CONFIG_DIR) + return os.path.expanduser(DEFAULT_CONFIG_DIR) + + +def fmt_duration(seconds): + try: + total = int(seconds or 0) + except (TypeError, ValueError): + return "?" + return f"{total // 60}:{total % 60:02d}" + + +def fmt_date(value): + return str(value or "")[:10] + + +def is_masked_secret(value): + """Production instances reply to oauth-clients/local with the secret + replaced by a run of '*' characters.""" + return bool(value) and set(value) == {"*"} + + +def request_plan(method, path, params=None, **extra): + plan: Dict[str, Any] = {"dry_run": True, "method": method, "path": path, "params": params or {}} + plan.update(extra) + return plan + + +def channel_label(video): + channel = video.get("channel") or {} + account = video.get("account") or {} + display = channel.get("displayName") or account.get("displayName") or "?" + host = channel.get("host") or account.get("host") + return f"{display}@{host}" if host else display + + +def video_line(video, index=None): + name = str(video.get("name") or "?")[:52] + views = video.get("views") or 0 + when = fmt_date(video.get("publishedAt")) + prefix = f"{index:>3}. " if index else " - " + return f" {prefix}{name:<52} {fmt_duration(video.get('duration')):>6} {views:>8} views {channel_label(video)} {when}" + + +class PeerTubeClient: + """REST client for one PeerTube instance (federated: tokens are per-instance).""" + + def __init__(self, server="", dry_run=False, config_dir=None): + self.server = (server or ENV_SERVER).rstrip("/") + self.dry_run = dry_run + self.config_dir = config_dir or default_config_dir() + self._token: Optional[str] = None + self._refresh_token: Optional[str] = None + self._expires_at: Optional[float] = None + self._load_token() + + # ----- instance / URL helpers ------------------------------------- + + def base_url(self): + if not self.server: + die("No instance configured. Export PEERTUBE_SERVER (e.g. https://) " + "or pass --server.") + return self.server + + def display_server(self): + return self.server or "https://" + + def _url(self, path): + return f"{self.base_url()}{API_BASE}{path}" + + def _headers(self, with_token=True): + headers = {"Accept": "application/json"} + if with_token and self._token: + headers["Authorization"] = f"Bearer {self._token}" + return headers + + # ----- token file persistence (per-instance, owner-only) ---------- + + def token_path(self): + return os.path.join(self.config_dir, TOKEN_FILE_NAME) + + def _load_token(self): + try: + with open(self.token_path()) as handle: + data = json.load(handle) + except (OSError, ValueError): + return + stored_server = str(data.get("server") or "").rstrip("/") + if self.server and stored_server and stored_server != self.server: + return # token was minted by a different instance + self._token = data.get("access_token") or None + self._refresh_token = data.get("refresh_token") or None + expires_at = data.get("expires_at") + self._expires_at = float(expires_at) if expires_at else None + + def save_session(self, token_response): + access = token_response.get("access_token") + if not access: + die("Login response carried no access_token; nothing to persist.") + expires_in = token_response.get("expires_in") + expires_at = time.time() + float(expires_in) if expires_in else None + record = { + "server": self.server, + "access_token": access, + "refresh_token": token_response.get("refresh_token"), + "token_type": token_response.get("token_type", "Bearer"), + "expires_at": expires_at, + "expires_in": expires_in, + } + os.makedirs(self.config_dir, mode=0o700, exist_ok=True) + fd = os.open(self.token_path(), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as handle: + json.dump(record, handle, indent=2) + self._token = access + self._refresh_token = record["refresh_token"] + self._expires_at = expires_at + + def clear_token(self): + try: + os.remove(self.token_path()) + except OSError: + pass + self._token = None + self._refresh_token = None + self._expires_at = None + + def token_is_valid(self): + if not self._token: + return False + return self._expires_at is None or time.time() < self._expires_at + + def ensure_token(self): + """Authed commands need a live token; try refresh before giving up.""" + if self.token_is_valid(): + return + if self._refresh_token and not self.dry_run and self.refresh(): + return + die(f"Not authenticated for {self.display_server()}. Run " + f"'peertube login --username ' first, or use --dry-run to preview.") + + # ----- OAuth2 flows ------------------------------------------------ + + def fetch_oauth_client(self): + """Anonymous GET /oauth-clients/local (singular 'local').""" + url = self._url("/oauth-clients/local") + log(f"--> GET {url}") + try: + response = requests.get(url, headers={"Accept": "application/json"}, timeout=15) + except requests.exceptions.RequestException as exc: + die(f"Cannot reach {self.display_server()} ({exc.__class__.__name__}). " + f"Check PEERTUBE_SERVER and connectivity.") + if response.status_code >= 400: + die(f"oauth-clients/local returned {response.status_code}: {response.text[:200]}") + try: + payload = response.json() + except ValueError: + die(f"Non-JSON response from oauth-clients/local; is {self.display_server()} a PeerTube instance?") + client_id = payload.get("client_id") + if not client_id: + die("oauth-clients/local response missing client_id.") + return client_id, payload.get("client_secret") or "" + + def request_token(self, client_id, client_secret, username, password, otp=None): + """POST /users/token with the password grant (x-www-form-urlencoded).""" + form = { + "client_id": client_id, + "client_secret": client_secret, + "grant_type": "password", + "username": username, + "password": password, + } + headers = {"Accept": "application/json"} + if otp: + headers["x-peertube-otp"] = otp + url = self._url("/users/token") + log("--> POST /users/token (password grant, values not logged)") + try: + response = requests.post(url, data=form, headers=headers, timeout=15) + except requests.exceptions.RequestException as exc: + die(f"Cannot reach {self.display_server()} ({exc.__class__.__name__}) during token request.") + if response.status_code == 401: + die("Login failed (401): token request rejected. If the account uses two-factor " + "authentication, pass --otp.") + if response.status_code >= 400: + detail = "" + try: + body = response.json() + if isinstance(body, dict): + detail = body.get("detail") or body.get("title") or body.get("error") or "" + except ValueError: + detail = (response.text or "")[:200] + die(f"Login failed ({response.status_code}): invalid client credentials or wrong " + f"username/password. {detail}".rstrip()) + try: + return response.json() + except ValueError: + die("Token endpoint returned non-JSON.") + + def refresh(self): + """grant_type=refresh_token. The official reference does not render the + exact refresh form fields, so this uses the community-established shape + and treats any failure as a signal to fall back to a password re-grant.""" + if not self._refresh_token: + return False + try: + client_id, client_secret = self.fetch_oauth_client() + except SystemExit: + return False + if is_masked_secret(client_secret) or not client_secret: + return False + form = { + "client_id": client_id, + "client_secret": client_secret, + "grant_type": "refresh_token", + "refresh_token": self._refresh_token, + } + url = self._url("/users/token") + log("--> POST /users/token (refresh grant, values not logged)") + try: + response = requests.post(url, data=form, headers={"Accept": "application/json"}, timeout=15) + except requests.exceptions.RequestException: + return False + if response.status_code >= 400: + return False + try: + data = response.json() + except ValueError: + return False + if not data.get("access_token"): + return False + self.save_session(data) + return True + + def revoke_token(self): + """POST /users/revoke-token invalidates the access and refresh tokens.""" + url = self._url("/users/revoke-token") + log(f"--> POST {url}") + try: + return requests.post(url, headers=self._headers(), timeout=15) + except requests.exceptions.RequestException as exc: + die(f"Cannot reach {self.display_server()} ({exc.__class__.__name__}) during revocation.") + + # ----- read endpoints ---------------------------------------------- + + def _get(self, path, params=None): + url = self._url(path) + if self.dry_run: + return request_plan("GET", f"{API_BASE}{path}", params) + log(f"--> GET {url}") + try: + response = requests.get(url, params=params, headers=self._headers(), timeout=30) + except requests.exceptions.RequestException as exc: + die(f"Cannot reach {self.display_server()} ({exc.__class__.__name__}). " + f"Check PEERTUBE_SERVER and connectivity.") + return self._handle(response, f"{API_BASE}{path}") + + def _handle(self, response, path): + status = response.status_code + if status >= 400: + detail = "" + try: + body = response.json() + if isinstance(body, dict): + detail = body.get("detail") or body.get("title") or body.get("error") or "" + except ValueError: + detail = (response.text or "")[:200] + if status == 401: + die(f"401 Unauthorized on {path}: {detail or 'token missing, expired, or revoked'}. " + f"Run 'peertube login' or check the instance.") + if status == 429: + retry = response.headers.get("Retry-After", "?") + die(f"429 rate limited on {path}; Retry-After: {retry}s " + f"(default server limit: 50 calls per 10 seconds).") + die(f"API error {status} on {path}: {detail or 'no detail in body'}") + try: + return response.json() + except ValueError: + die(f"Non-JSON response from {path} (status {status}); " + f"is {self.display_server()} a PeerTube instance?") + + def get_about(self): + return self._get("/config/about") + + def get_stats(self): + return self._get("/server/stats") + + def list_videos(self, start=0, count=DEFAULT_COUNT, sort="-publishedAt"): + return self._get("/videos", {"start": start, "count": count, "sort": sort}) + + def get_video(self, video_id): + return self._get(f"/videos/{video_id}") + + def search_videos(self, query, start=0, count=DEFAULT_COUNT, search_target="local", sort=None): + params: Dict[str, Any] = {"search": query, "searchTarget": search_target, + "start": start, "count": count} + if sort: + params["sort"] = sort + return self._get("/search/videos", params) + + def list_comment_threads(self, video_id, start=0, count=DEFAULT_COUNT): + return self._get(f"/videos/{video_id}/comment-threads", + {"start": start, "count": count, "sort": "-createdAt"}) + + def list_channels(self, start=0, count=DEFAULT_COUNT): + return self._get("/video-channels", {"start": start, "count": count}) + + def get_channel(self, handle): + return self._get(f"/video-channels/{handle}") + + def list_channel_videos(self, handle, start=0, count=DEFAULT_COUNT): + return self._get(f"/video-channels/{handle}/videos", {"start": start, "count": count}) + + def get_account(self, name): + return self._get(f"/accounts/{name}") + + def my_profile(self): + return self._get("/users/me") + + def my_videos(self, start=0, count=DEFAULT_COUNT): + return self._get("/users/me/videos", {"start": start, "count": count}) + + +# ----- command handlers ------------------------------------------------ + + +def validate_page_args(count, start): + if count < 1 or count > MAX_COUNT: + die(f"--limit must be between 1 and {MAX_COUNT} (server maximum).") + if start < 0: + die("--offset must be >= 0.") + + +def cmd_server(client, args): + if client.dry_run: + plan = {"dry_run": True, "requests": [ + {"method": "GET", "path": f"{API_BASE}/config/about", "params": {}}, + {"method": "GET", "path": f"{API_BASE}/server/stats", "params": {}}, + ]} + return emit("[dry-run] GET /config/about + /server/stats", plan) + about = client.get_about() or {} + stats = client.get_stats() or {} + instance = about.get("instance") or {} + name = instance.get("name", "?") + description = instance.get("shortDescription") or "" + keys = ("totalUsers", "totalLocalVideos", "totalVideos", + "totalLocalVideoViews", "totalLocalVideoDownloads", "totalLocalVideoChannels") + payload = { + "instance": { + "name": name, + "shortDescription": instance.get("shortDescription"), + "description": instance.get("description"), + }, + "stats": {key: stats.get(key) for key in keys}, + } + stats_line = " ".join(f"{key}={payload['stats'][key]}" for key in keys if payload['stats'][key] is not None) + emit(f"Instance {name}\n {description[:100]}\n {stats_line}", payload) + + +def cmd_videos(client, args): + parser = argparse.ArgumentParser(prog="peertube videos") + parser.add_argument("--limit", type=int, default=DEFAULT_COUNT) + parser.add_argument("--offset", type=int, default=0) + parser.add_argument("--sort", default="-publishedAt") + parsed, _ = parser.parse_known_args(args) + validate_page_args(parsed.limit, parsed.offset) + params = {"start": parsed.offset, "count": parsed.limit, "sort": parsed.sort} + if client.dry_run: + return emit("[dry-run] GET /api/v1/videos " + json.dumps(params), + request_plan("GET", f"{API_BASE}/videos", params)) + data = client._get("/videos", params) or {} + videos = data.get("data", []) + total = data.get("total", len(videos)) + if not videos: + return emit("No videos.", {"total": total, "start": parsed.offset, "count": 0, "videos": []}) + lines = [video_line(v, i + 1) for i, v in enumerate(videos)] + emit(f"{total} video(s) on {client.display_server()}:\n" + "\n".join(lines), + {"total": total, "start": parsed.offset, "count": len(videos), "videos": videos}) + + +def cmd_video(client, args): + parser = argparse.ArgumentParser(prog="peertube video") + parser.add_argument("--id", required=True, help="Numeric id, UUID, or shortUUID") + parsed, _ = parser.parse_known_args(args) + if client.dry_run: + return emit(f"[dry-run] GET {API_BASE}/videos/{parsed.id}", + request_plan("GET", f"{API_BASE}/videos/{parsed.id}", {})) + detail = client._get(f"/videos/{parsed.id}") or {} + emit(f"{detail.get('name', '?')} [{fmt_duration(detail.get('duration'))}] " + f"{detail.get('views', 0)} views by {channel_label(detail)}", detail) + + +def cmd_search(client, args): + parser = argparse.ArgumentParser( + prog="peertube search", + description="Search the configured instance's OWN catalog (searchTarget=local). " + "For fediverse-wide search set PEERTUBE_SERVER=https://sepiasearch.org — " + "same commands, wider index.") + parser.add_argument("--query", "-q", required=True) + parser.add_argument("--limit", type=int, default=DEFAULT_COUNT) + parser.add_argument("--offset", type=int, default=0) + parser.add_argument("--sort", default=None) + parser.add_argument("--search-target", choices=["local", "search-index"], default="local") + parsed, _ = parser.parse_known_args(args) + validate_page_args(parsed.limit, parsed.offset) + params: Dict[str, Any] = {"search": parsed.query, "searchTarget": parsed.search_target, + "start": parsed.offset, "count": parsed.limit} + if parsed.sort: + params["sort"] = parsed.sort + if client.dry_run: + return emit(f"[dry-run] GET {API_BASE}/search/videos " + json.dumps(params), + request_plan("GET", f"{API_BASE}/search/videos", params)) + data = client._get("/search/videos", params) or {} + videos = data.get("data", []) + total = data.get("total", len(videos)) + if not videos: + return emit("No results.", {"total": total, "start": parsed.offset, "count": 0, "videos": []}) + lines = [video_line(v, i + 1) for i, v in enumerate(videos)] + scope = ("instance-local" if parsed.search_target == "local" else "search-index") + emit(f"{total} result(s) [{scope} search on {client.display_server()}]:\n" + "\n".join(lines), + {"total": total, "start": parsed.offset, "count": len(videos), "videos": videos}) + + +def cmd_comments(client, args): + parser = argparse.ArgumentParser(prog="peertube comments") + parser.add_argument("--id", required=True, help="Video id, UUID, or shortUUID") + parser.add_argument("--limit", type=int, default=DEFAULT_COUNT) + parser.add_argument("--offset", type=int, default=0) + parsed, _ = parser.parse_known_args(args) + validate_page_args(parsed.limit, parsed.offset) + params = {"start": parsed.offset, "count": parsed.limit, "sort": "-createdAt"} + if client.dry_run: + return emit(f"[dry-run] GET {API_BASE}/videos/{parsed.id}/comment-threads " + json.dumps(params), + request_plan("GET", f"{API_BASE}/videos/{parsed.id}/comment-threads", params)) + data = client._get(f"/videos/{parsed.id}/comment-threads", params) or {} + threads = data.get("data", []) + total = data.get("total", len(threads)) + lines = [] + for thread in threads: + comment = thread.get("comment") or {} + account = comment.get("account") or {} + text = str(comment.get("text") or "")[:70].replace("\n", " ") + lines.append(f" {thread.get('totalReplies', 0):>3} replies @{account.get('name', '?')} {text}") + emit(f"{total} comment thread(s):\n" + ("\n".join(lines) if lines else " (none)"), + {"total": total, "total_not_deleted": data.get("totalNotDeletedComments"), + "start": parsed.offset, "threads": threads}) + + +def cmd_channels(client, args): + parser = argparse.ArgumentParser(prog="peertube channels") + parser.add_argument("--limit", type=int, default=DEFAULT_COUNT) + parser.add_argument("--offset", type=int, default=0) + parsed, _ = parser.parse_known_args(args) + validate_page_args(parsed.limit, parsed.offset) + params = {"start": parsed.offset, "count": parsed.limit} + if client.dry_run: + return emit(f"[dry-run] GET {API_BASE}/video-channels " + json.dumps(params), + request_plan("GET", f"{API_BASE}/video-channels", params)) + data = client._get("/video-channels", params) or {} + channels = data.get("data", []) + total = data.get("total", len(channels)) + if not channels: + return emit("No channels.", {"total": total, "start": parsed.offset, "count": 0, "channels": []}) + lines, _ = [], [] + for channel in channels: + display = channel.get("displayName") or channel.get("name") or "?" + handle = f"{channel.get('name', '?')}@{channel.get('host', '?')}" + lines.append(f" {display:<30} {handle:<40} " + f"{channel.get('videosCount', '?')} videos {channel.get('followersCount', '?')} followers") + emit(f"{total} channel(s) on {client.display_server()}:\n" + "\n".join(lines), + {"total": total, "start": parsed.offset, "count": len(channels), "channels": channels}) + + +def cmd_channel(client, args): + parser = argparse.ArgumentParser(prog="peertube channel") + parser.add_argument("--handle", required=True, help="Channel name or name@host") + parser.add_argument("--limit", type=int, default=DEFAULT_COUNT) + parser.add_argument("--offset", type=int, default=0) + parsed, _ = parser.parse_known_args(args) + validate_page_args(parsed.limit, parsed.offset) + params = {"start": parsed.offset, "count": parsed.limit} + if client.dry_run: + plan = {"dry_run": True, "requests": [ + {"method": "GET", "path": f"{API_BASE}/video-channels/{parsed.handle}", "params": {}}, + {"method": "GET", "path": f"{API_BASE}/video-channels/{parsed.handle}/videos", "params": params}, + ]} + return emit(f"[dry-run] GET /video-channels/{parsed.handle} (+ /videos)", plan) + meta = client._get(f"/video-channels/{parsed.handle}") or {} + videos_data = client._get(f"/video-channels/{parsed.handle}/videos", params) or {} + videos = videos_data.get("data", []) + display = meta.get("displayName") or meta.get("name") or parsed.handle + lines = [video_line(v, i + 1) for i, v in enumerate(videos)] + emit(f"{display} ({meta.get('followersCount', '?')} followers, {meta.get('videosCount', '?')} videos)\n" + + ("\n".join(lines) if lines else " (no videos listed)"), + {"channel": meta, "total": videos_data.get("total", len(videos)), + "start": parsed.offset, "videos": videos}) + + +def cmd_account(client, args): + parser = argparse.ArgumentParser(prog="peertube account") + parser.add_argument("--name", required=True, help="Account name or name@host") + parsed, _ = parser.parse_known_args(args) + if client.dry_run: + return emit(f"[dry-run] GET {API_BASE}/accounts/{parsed.name}", + request_plan("GET", f"{API_BASE}/accounts/{parsed.name}", {})) + account = client._get(f"/accounts/{parsed.name}") or {} + display = account.get("displayName") or account.get("name") or parsed.name + handle = f"{account.get('name', parsed.name)}@{account.get('host', '?')}" + emit(f"{display} ({handle}) {account.get('followersCount', '?')} followers", account) + + +def cmd_me(client, args): + if client.dry_run: + return emit(f"[dry-run] GET {API_BASE}/users/me (Authorization: Bearer )", + request_plan("GET", f"{API_BASE}/users/me", {})) + client.ensure_token() + data = client._get("/users/me") or {} + if isinstance(data, list): # docs render an array sample; live servers return one object + data = data[0] if data else {} + role = data.get("role") or {} + quota = data.get("videoQuota") + quota_text = f"{quota} bytes" if quota is not None else "?" + emit(f"@{data.get('username', '?')} Role: {role.get('label', '?')} Quota: {quota_text}", data) + + +def cmd_my_videos(client, args): + parser = argparse.ArgumentParser(prog="peertube my-videos") + parser.add_argument("--limit", type=int, default=DEFAULT_COUNT) + parser.add_argument("--offset", type=int, default=0) + parsed, _ = parser.parse_known_args(args) + validate_page_args(parsed.limit, parsed.offset) + params = {"start": parsed.offset, "count": parsed.limit} + if client.dry_run: + return emit(f"[dry-run] GET {API_BASE}/users/me/videos " + json.dumps(params), + request_plan("GET", f"{API_BASE}/users/me/videos", params)) + client.ensure_token() + data = client._get("/users/me/videos", params) or {} + videos = data.get("data", []) + total = data.get("total", len(videos)) + lines = [video_line(v, i + 1) for i, v in enumerate(videos)] + emit(f"{total} of your video(s):\n" + ("\n".join(lines) if lines else " (none)"), + {"total": total, "start": parsed.offset, "count": len(videos), "videos": videos}) + + +def cmd_login(client, args): + parser = argparse.ArgumentParser( + prog="peertube login", + description="OAuth2 password grant: fetch the instance's client pair, exchange your " + "credentials for a bearer token, and persist it (owner-only file, " + "per-instance). Password is never echoed.") + parser.add_argument("--username", "-u", required=True) + parser.add_argument("--server", help="Instance URL override, e.g. https://") + password_group = parser.add_mutually_exclusive_group() + password_group.add_argument("--password", help="Account password (prefer --password-stdin or --prompt)") + password_group.add_argument("--password-stdin", action="store_true", help="Read password from stdin") + password_group.add_argument("--prompt", action="store_true", help="Prompt for the password (hidden)") + parser.add_argument("--otp", help="One-time password when the account has 2FA enabled") + parsed, _ = parser.parse_known_args(args) + if client.dry_run: + return emit("[dry-run] POST /api/v1/users/token (password grant)", + request_plan("POST", f"{API_BASE}/users/token", {}, + form_fields=["client_id", "client_secret", "grant_type", + "username", "password"], + note="field values suppressed; run without --dry-run to authenticate")) + server = (parsed.server or client.server).rstrip("/") + if not server: + die("No instance configured. Export PEERTUBE_SERVER or pass --server https://.") + login_client = PeerTubeClient(server=server, config_dir=client.config_dir) + if not parsed.password_stdin and not parsed.prompt and parsed.password is None: + die("No password supplied. Use --password, --password-stdin, or --prompt.") + if parsed.password_stdin: + password = sys.stdin.readline().rstrip("\n") + elif parsed.prompt: + password = getpass.getpass(f"Password for {parsed.username}@{server}: ") + else: + password = parsed.password + client_id, client_secret = login_client.fetch_oauth_client() + if is_masked_secret(client_secret): + die("This instance masks client_secret in oauth-clients/local ('*' characters). " + "Production PeerTube front ends obtain the real pair from their own served assets; " + "see references/auth-and-tokens.md for the workaround before retrying login.") + token_response = login_client.request_token(client_id, client_secret, parsed.username, + password, otp=parsed.otp) + login_client.save_session(token_response) + expires_in = token_response.get("expires_in") + expires_text = f", expires in {expires_in}s" if expires_in else "" + emit(f"Logged in to {server} as {parsed.username}{expires_text}\n" + f"Token file: {login_client.token_path()}", + {"status": "logged_in", "server": server, "username": parsed.username, + "token_file": login_client.token_path(), "expires_in": expires_in}) + + +def cmd_logout(client, args): + if client.dry_run: + return emit(f"[dry-run] POST {API_BASE}/users/revoke-token", + request_plan("POST", f"{API_BASE}/users/revoke-token", {})) + if not client.server: + die("No instance configured. Export PEERTUBE_SERVER or pass --server.") + if not client._token and not client._refresh_token: + die(f"No stored token for {client.server}; nothing to revoke.") + response = client.revoke_token() + status = response.status_code + if status == 200: + client.clear_token() + return emit("Token revoked and local token file removed.", + {"status": "logged_out", "revoked": True, "server": client.server}) + if status == 401: + client.clear_token() + warn("Server rejected revocation (401); the token was likely already invalid. " + "Local token file removed.") + return emit("Logged out (token was already invalid server-side).", + {"status": "logged_out", "revoked": False, "server": client.server}) + die(f"Revocation failed ({status}). Local token file kept; retry or remove " + f"{client.token_path()} manually.") + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="peertube", + description="PeerTube federated video from the terminal.", + epilog="Set PEERTUBE_SERVER (any instance host, e.g. https://) or pass " + "--server. Fediverse-wide search: PEERTUBE_SERVER=https://sepiasearch.org. " + "Login: peertube login --username NAME --prompt") + parser.add_argument("--server", + help="Instance URL (default: PEERTUBE_SERVER env var)") + parser.add_argument("--json", action="store_true", help="Machine-readable JSON output") + parser.add_argument("--dry-run", action="store_true", + help="Print the request plan without any network call") + parser.add_argument("--quiet", action="store_true", + help="Accepted for compatibility; no effect") + parser.add_argument("--verbose", action="store_true", help="Trace requests on stderr") + sub = parser.add_subparsers(dest="command") + + sub.add_parser("server", help="Instance name, description, and stats (anonymous)", + description="Compose /config/about and /server/stats for an instance picture.", + epilog="Example: peertube server --json") + vp = sub.add_parser("videos", help="List the instance's videos (offset pagination)", + description="GET /videos with start/count pagination (there is no page parameter).", + epilog="Example: peertube videos --limit 5 --json") + vp.add_argument("--limit", type=int, default=DEFAULT_COUNT, help="Page size, 1-100") + vp.add_argument("--offset", type=int, default=0, help="Zero-based result offset") + vp.add_argument("--sort", default="-publishedAt", help="e.g. -publishedAt, -views, -trending") + vi = sub.add_parser("video", help="Full detail for one video", + description="GET /videos/{id}; id accepts numeric id, UUID, or shortUUID.", + epilog="Example: peertube video --id ") + vi.add_argument("--id", required=True) + sp = sub.add_parser("search", help="Search this instance's own catalog", + description="Instance-local search (searchTarget=local). For fediverse-wide " + "search point PEERTUBE_SERVER at https://sepiasearch.org.", + epilog="Example: peertube search --query linux --limit 10") + sp.add_argument("--query", "-q", required=True) + sp.add_argument("--limit", type=int, default=DEFAULT_COUNT, help="Page size, 1-100") + sp.add_argument("--offset", type=int, default=0) + sp.add_argument("--sort", default=None, help="e.g. -views, -match, -publishedAt") + sp.add_argument("--search-target", choices=["local", "search-index"], default="local", + help="local = this instance's catalog; search-index = external index if the " + "instance has one") + cp = sub.add_parser("comments", help="Top-level comment threads of a video", + description="GET /videos/{id}/comment-threads (hyphenated route).", + epilog="Example: peertube comments --id ") + cp.add_argument("--id", required=True) + cp.add_argument("--limit", type=int, default=DEFAULT_COUNT, help="Page size, 1-100") + cp.add_argument("--offset", type=int, default=0) + hp = sub.add_parser("channels", help="List the instance's channels", + description="GET /video-channels with start/count pagination.", + epilog="Example: peertube channels --limit 20 --json") + hp.add_argument("--limit", type=int, default=DEFAULT_COUNT, help="Page size, 1-100") + hp.add_argument("--offset", type=int, default=0) + cv = sub.add_parser("channel", help="One channel's metadata and uploads", + description="GET /video-channels/{handle} and its videos; handle is name or " + "name@host for remote channels.", + epilog="Example: peertube channel --handle framasoft@framatube.org") + cv.add_argument("--handle", required=True) + cv.add_argument("--limit", type=int, default=DEFAULT_COUNT, help="Page size, 1-100") + cv.add_argument("--offset", type=int, default=0) + ap = sub.add_parser("account", help="One account's metadata", + description="GET /accounts/{name}; name accepts name or name@host.", + epilog="Example: peertube account --name chocobozzz@framatube.org") + ap.add_argument("--name", required=True) + sub.add_parser("me", help="Your profile (requires login)", + description="GET /users/me with the persisted bearer token.", + epilog="Example: peertube me --json | jq .role.label") + mv = sub.add_parser("my-videos", help="Your uploads (requires login)", + description="GET /users/me/videos with start/count pagination.", + epilog="Example: peertube my-videos --limit 50 --json") + mv.add_argument("--limit", type=int, default=DEFAULT_COUNT, help="Page size, 1-100") + mv.add_argument("--offset", type=int, default=0) + lg = sub.add_parser("login", help="Authenticate (OAuth2 password grant) and persist a token", + description="Fetch the instance's OAuth client pair, run the password grant, " + "and persist the token per-instance in an owner-only file.", + epilog="Example: peertube login --username NAME --prompt") + lg.add_argument("--username", "-u", help="PeerTube username") + lg_password_group = lg.add_mutually_exclusive_group() + lg_password_group.add_argument("--password", help="Account password (prefer --password-stdin or --prompt)") + lg_password_group.add_argument("--password-stdin", action="store_true", help="Read the password from stdin") + lg_password_group.add_argument("--prompt", action="store_true", help="Prompt for the password (hidden)") + lg.add_argument("--otp", help="One-time password when the account has 2FA enabled") + lg.add_argument("--server", help="Instance URL override, e.g. https://") + lo = sub.add_parser("logout", help="Revoke the token server-side and delete the local token file", + description="POST /users/revoke-token, then remove the local token file.", + epilog="Example: peertube logout") + return parser + + +def main(): + global GLOBAL_FLAGS + GLOBAL_FLAGS, filtered_argv = _preparse_global_flags(sys.argv) + if GLOBAL_FLAGS.get("json"): + warnings.simplefilter("ignore") + + parser = build_parser() + args = parser.parse_args(filtered_argv[1:]) + if not args.command: + parser.print_help() + sys.exit(1) + + client = PeerTubeClient(server=args.server or GLOBAL_FLAGS.get("server") or "", + dry_run=GLOBAL_FLAGS.get("dry_run", False)) + handlers = { + "server": cmd_server, "videos": cmd_videos, "video": cmd_video, + "search": cmd_search, "comments": cmd_comments, "channels": cmd_channels, + "channel": cmd_channel, "account": cmd_account, "me": cmd_me, + "my-videos": cmd_my_videos, "login": cmd_login, "logout": cmd_logout, + } + handler = handlers.get(args.command) + if not handler: + parser.print_help() + sys.exit(1) + remaining = filtered_argv[filtered_argv.index(args.command) + 1:] + handler(client, remaining) + + +if __name__ == "__main__": + main() diff --git a/peertube/scripts/peertube-cli b/peertube/scripts/peertube-cli deleted file mode 100755 index 4e37cf4..0000000 --- a/peertube/scripts/peertube-cli +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env python3 -"""peertube-cli — PeerTube federated video from the terminal. - -Browse videos, channels, and playlists on any PeerTube instance. -Login with OAuth2 for authenticated operations. -""" - -import argparse, json, os, sys, time, warnings -from typing import Any, Dict, List, Optional, Tuple -warnings.simplefilter("ignore") -import requests - -ENV_SERVER = os.getenv("PEERTUBE_SERVER", "") -CONFIG_DIR = os.path.expanduser(os.getenv("PEERTUBE_CONFIG_DIR", "~/.config/peertube-cli")) -API_BASE = "/api/v1" - -QUIET = False -GLOBAL_FLAGS: Dict[str, Any] = {"json": False, "dry_run": False, "quiet": False, "verbose": False} -def log(m): global QUIET; (not QUIET and not GLOBAL_FLAGS.get("json")) and print(m) -def warn(m): print(f"Warning: {m}", file=sys.stderr) -def die(m, c=1): print(f"Error: {m}", file=sys.stderr); sys.exit(c) -def emit(h, d): - if GLOBAL_FLAGS.get("json"): print(json.dumps(d, default=str)) - else: print(h) - -def _preparse(argv): - BOOLS = {"--json","--dry-run","--quiet","--verbose"} - f, fl = {}, [argv[0]] - i = 1 - while i < len(argv): - a = argv[i] - if a in BOOLS: f[a.lstrip("-").replace("-","_")] = True; i += 1 - elif a in ("--help","-h"): return f, argv - elif a == "--": fl.extend(argv[i:]); break - else: fl.append(a); i += 1 - return f, fl - -class PeerTubeClient: - def __init__(self, server="", dry_run=False): - self.server = (server or ENV_SERVER or "https://your-instance.example.com").rstrip("/") - self.dry_run = dry_run - self._token = None - # Try to load saved token - token_path = os.path.join(CONFIG_DIR, "token.json") - if os.path.isfile(token_path): - try: - with open(token_path) as f: data = json.load(f) - if data.get("expires_at", 0) > time.time(): - self._token = data.get("access_token") - except: pass - - def _oauth_login(self): - """Fetch OAuth client creds and exchange for token.""" - try: - r = requests.get(f"{self.server}{API_BASE}/oauth-clients/local", timeout=15) - clients = r.json() - except: die(f"Cannot reach {self.server}. Check PEERTUBE_SERVER.") - return clients.get("client_id", ""), clients.get("client_secret", "") - - def login(self, username, password): - """Login and persist OAuth token.""" - cid, csec = self._oauth_login() - try: - r = requests.post(f"{self.server}{API_BASE}/users/token", data={ - "client_id": cid, "client_secret": csec, - "grant_type": "password", "username": username, - "password": password, - "response_type": "code"}, timeout=15) - except ConnectionError as e: die(f"Cannot connect: {e}") - if r.status_code >= 400: die(f"Login failed: {r.text[:200]}") - data = r.json() - self._token = data.get("access_token") - # Persist token - os.makedirs(CONFIG_DIR, exist_ok=True) - with open(os.path.join(CONFIG_DIR, "token.json"), "w") as f: - json.dump({"access_token": self._token, "expires_at": time.time() + data.get("expires_in", 86400)}, f) - return data - - def _headers(self): - h = {"Accept": "application/json"} - if self._token: h["Authorization"] = f"Bearer {self._token}" - return h - - def _get(self, path, params=None): - url = f"{self.server}{API_BASE}{path}" - if self.dry_run: return {"dry_run":True, "url":url, "params":params, "total":0, "data":[]} - try: - r = requests.get(url, params=params, headers=self._headers(), timeout=30) - except ConnectionError as e: die(f"Cannot connect: {e}") - if r.status_code == 401: - die("Not authenticated. Run 'peertube auth login' first.") - if r.status_code >= 400: - try: d = r.json() - except: d = r.text[:200] - die(f"API error ({r.status_code}): {d}") - return r.json() - - def get_server_info(self): return self._get("/server/") - - def list_videos(self, page=1, limit=12, sort="-publishedAt"): - return self._get("/videos", {"page":page, "count":limit, "sort":sort}) - - def search_videos(self, query, page=1, limit=12): - return self._get("/search/videos", {"search":query, "page":page, "count":limit}) - - def get_video(self, vid): return self._get(f"/videos/{vid}") - - def list_video_comments(self, vid, page=1, limit=10): - return self._get(f"/videos/{vid}/comments", {"page":page, "count":limit}) - - def list_channels(self, page=1, limit=15): - return self._get("/video-channels", {"page":page, "count":limit}) - - def get_channel(self, name): return self._get(f"/video-channels/{name}") - - def list_channel_videos(self, name, page=1, limit=12): - return self._get(f"/video-channels/{name}/videos", {"page":page, "count":limit}) - - def list_playlists(self, page=1, limit=10): - return self._get("/video-playlists/user", {"page":page, "count":limit}) - - def get_playlist(self, pid): return self._get(f"/video-playlists/{pid}") - - def my_profile(self): return self._get("/users/me") - def my_videos(self, page=1, limit=12): return self._get("/users/me/videos", {"page":page, "count":limit}) - - -def fmt_video(v, idx=None): - prefix = f"{idx}. " if idx else "" - name = v.get("name", "?") - author = v.get("channel",{}).get("displayName", v.get("account",{}).get("displayName","?")) - dur = v.get("duration", 0) - dur_str = f"{int(dur//60)}:{int(dur%60):02d}" - views = v.get("views", 0) - pub = (v.get("publishedAt") or "")[:10] - return f" {prefix}{name:55} {dur_str} {views} views {author} {pub}" - - -def cmd_server(client, args): - if client.dry_run: return emit("[dry-run] Get server info", {"dry_run":True}) - d = client.get_server_info() or {} - name = d.get("instance",{}).get("name","?") - desc = (d.get("instance",{}).get("shortDescription","") or "")[:80] - users = d.get("users","?"); videos = d.get("videos","?"); views = d.get("views","?") - emit(f"🖥️ {name}\n {desc}\n Users: {users} Videos: {videos} Views: {views}", - {"instance":{"name":name,"shortDescription":desc},"users":users,"videos":videos}) - -def cmd_videos(client, args): - p = argparse.ArgumentParser(prog="peertube videos") - p.add_argument("--limit", type=int, default=12) - parsed, _ = p.parse_known_args(args) - if client.dry_run: return emit("[dry-run] List videos", {"dry_run":True}) - data = client.list_videos(limit=parsed.limit) or {} - videos = data.get("data",[]) - if not videos: return emit("No videos.", {"videos":[]}) - lines = [fmt_video(v, i+1) for i, v in enumerate(videos)] - total = data.get("total", len(videos)) - emit(f"{total} video(s):\n"+"\n".join(lines), {"total":total,"videos":videos}) - -def cmd_search(client, args): - p = argparse.ArgumentParser(prog="peertube search") - p.add_argument("--query", "-q", required=True) - p.add_argument("--limit", type=int, default=12) - parsed, _ = p.parse_known_args(args) - if client.dry_run: return emit(f"[dry-run] Search: {parsed.query}", {"dry_run":True}) - data = client.search_videos(parsed.query, limit=parsed.limit) or {} - videos = data.get("data",[]) - if not videos: return emit("No results.", {"videos":[]}) - lines = [fmt_video(v, i+1) for i, v in enumerate(videos)] - total = data.get("total", len(videos)) - emit(f"{total} result(s):\n"+"\n".join(lines), {"total":total,"videos":videos}) - -def cmd_channels(client, args): - if client.dry_run: return emit("[dry-run] List channels", {"dry_run":True}) - data = client.list_channels() or {} - channels = data.get("data",[]) - if not channels: return emit("No channels.", {"channels":[]}) - lines, out = [], [] - for c in channels: - dn = c.get("displayName","?"); name = c.get("name","?"); vid = c.get("videosCount",0); subs = c.get("subscribersCount",0) - lines.append(f" {dn:30} @{name} {vid} videos {subs} subscribers") - out.append({"displayName":dn,"name":name,"videosCount":vid,"subscribersCount":subs}) - emit(f"{len(channels)} channel(s):\n"+"\n".join(lines), {"channels":out}) - -def cmd_me(client, args): - if client.dry_run: return emit("[dry-run] Get profile", {"dry_run":True}) - d = client.my_profile() or {} - uname = d.get("username","?"); role = d.get("role","?") - vid = d.get("videosCount",0); views = d.get("viewsCount",0) - emit(f"👤 @{uname} Role: {role} Videos: {vid} Views: {views}", - {"username":uname,"role":role,"videosCount":vid,"viewsCount":views}) - -def main(): - global GLOBAL_FLAGS, QUIET - GLOBAL_FLAGS, filtered_argv = _preparse(sys.argv) - if GLOBAL_FLAGS.get("quiet"): QUIET = True - if GLOBAL_FLAGS.get("json"): warnings.simplefilter("ignore") - - parser = argparse.ArgumentParser(prog="peertube", description="PeerTube federated video.", - epilog="Set PEERTUBE_SERVER. Login: peertube auth login --username ... --password ...") - sub = parser.add_subparsers(dest="command") - - # auth - ap = sub.add_parser("auth", help="Authentication") - asub = ap.add_subparsers(dest="auth_action") - lp = asub.add_parser("login", help="Login to PeerTube instance") - lp.add_argument("--username", required=True); lp.add_argument("--password", required=True) - - # other commands - sub.add_parser("server", help="Server info") - sub.add_parser("videos", help="List videos").add_argument("--limit",type=int,default=12) - sub.add_parser("channels", help="List channels") - sp = sub.add_parser("search", help="Search videos") - sp.add_argument("--query","-q",required=True); sp.add_argument("--limit",type=int,default=12) - sub.add_parser("me", help="My profile") - - args = parser.parse_args(filtered_argv[1:]) - if not args.command: parser.print_help(); sys.exit(1) - - client = PeerTubeClient(dry_run=GLOBAL_FLAGS.get("dry_run",False)) - remaining = filtered_argv[filtered_argv.index(args.command)+1:] - - if args.command == "auth": - if args.auth_action == "login": - result = client.login(args.username, args.password) - emit(f"✅ Logged in to {client.server}", {"status":"logged_in","server":client.server}) - else: ap.print_help() - return - - # Commands that need auth - if not client._token and not GLOBAL_FLAGS.get("dry_run"): - die("Not logged in. Run 'peertube auth login --username ... --password ...' first, or use --dry-run.") - - handlers = {"server":cmd_server,"videos":cmd_videos,"search":cmd_search,"channels":cmd_channels,"me":cmd_me} - h = handlers.get(args.command) - if not h: parser.print_help(); sys.exit(1) - h(client, remaining) - -if __name__ == "__main__": main() diff --git a/peertube/scripts/test_peertube.py b/peertube/scripts/test_peertube.py new file mode 100644 index 0000000..1bfa41d --- /dev/null +++ b/peertube/scripts/test_peertube.py @@ -0,0 +1,961 @@ +"""Offline test suite for the bundled peertube CLI. + +All HTTP is mocked at the requests seam; the only live call in the file is the +single anonymous instance probe behind the PEERTUBE_LIVE_TESTS=1 guard (skipped +by default, so the suite is fully offline and passes the proxy-trap rerun). +Covers: help output, argument-error paths, dry-run plans, mocked OAuth2 token +persistence/refresh/revocation, handler output contracts, and the documented +multi-step pipeline stages (each stage's output fields/types feed the next). +""" + +import contextlib +import importlib.machinery +import importlib.util +import io +import json +import os +import pathlib +import stat +import subprocess +import sys +import tempfile +import time +import unittest +from unittest.mock import patch + +SCRIPT = pathlib.Path(__file__).resolve().parent / "peertube" +LOADER = importlib.machinery.SourceFileLoader("peertube_cli", str(SCRIPT)) +SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER) +pt = importlib.util.module_from_spec(SPEC) +LOADER.exec_module(pt) + + +def clean_env(): + env = os.environ.copy() + for var in ("PEERTUBE_SERVER", "PEERTUBE_CONFIG_DIR", "PEERTUBE_LIVE_TESTS"): + env.pop(var, None) + return env + + +class FakeResponse: + def __init__(self, status_code=200, json_body=None, text="", headers=None): + self.status_code = status_code + self._json = json_body + self.text = text if text else (json.dumps(json_body) if json_body is not None else "") + self.headers = headers or {} + + def json(self): + if self._json is None: + raise ValueError("no json body") + return self._json + + +VIDEO_ONE = { + "id": 1, + "uuid": "uuid-one", + "shortUUID": "sOne1", + "url": "https://inst.example/w/uuid-one", + "name": "First video", + "duration": 125, + "views": 42, + "likes": 7, + "publishedAt": "2026-08-01T10:00:00.000Z", + "privacy": {"id": 1, "label": "Public"}, + "account": {"name": "alice", "displayName": "Alice", "host": "inst.example"}, + "channel": {"name": "alice-channel", "displayName": "Alice Channel", "host": "inst.example"}, +} +VIDEO_TWO = dict( + VIDEO_ONE, + id=2, + uuid="uuid-two", + shortUUID="sTwo2", + name="Second video", + duration=3661, + views=5, + channel={"name": "bob-channel", "displayName": "Bob Channel", "host": "other.example"}, +) + +TOKEN_RESPONSE = { + "access_token": "tok-1", + "refresh_token": "ref-1", + "token_type": "Bearer", + "expires_in": 3600, + "refresh_token_expires_in": 7200, +} +OAUTH_CLIENT = {"client_id": "cid-1", "client_secret": "client-secret-1"} +MASKED_OAUTH_CLIENT = {"client_id": "cid-1", "client_secret": "*" * 32} + + +def run_cli(*args, env=None): + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + env=env if env is not None else clean_env(), + ) + + +class ModuleStateTestCase(unittest.TestCase): + """Base that restores module globals mutated by in-process tests.""" + + def setUp(self): + self._flags = dict(pt.GLOBAL_FLAGS) + self._env_server = pt.ENV_SERVER + self._env_config = pt.ENV_CONFIG_DIR + pt.GLOBAL_FLAGS = {"json": True, "dry_run": False, "quiet": False, "verbose": False} + + def tearDown(self): + pt.GLOBAL_FLAGS = self._flags + pt.ENV_SERVER = self._env_server + pt.ENV_CONFIG_DIR = self._env_config + + +class HelpOutputTests(unittest.TestCase): + """Class 1: --help output.""" + + def test_help_lists_every_subcommand(self): + result = run_cli("--help") + self.assertEqual(result.returncode, 0, result.stderr) + for noun in ( + "server", + "videos", + "video", + "search", + "comments", + "channels", + "channel", + "account", + "me", + "my-videos", + "login", + "logout", + ): + self.assertIn(noun, result.stdout) + + def test_help_names_the_instance_env_var(self): + result = run_cli("--help") + self.assertIn("PEERTUBE_SERVER", result.stdout) + self.assertIn("sepiasearch.org", result.stdout) + + def test_search_help_states_its_scope(self): + result = run_cli("search", "--help") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("searchTarget", result.stdout) + self.assertIn("sepiasearch", result.stdout.lower()) + + def test_leaf_help_carries_examples(self): + for leaf in ("videos", "comments", "login", "logout"): + result = run_cli(leaf, "--help") + with self.subTest(leaf=leaf): + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Example:", result.stdout) + + +class ArgumentErrorTests(unittest.TestCase): + """Class 2: argument-error paths fail cleanly before any network call.""" + + def test_search_requires_query(self): + result = run_cli("search") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--query", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + def test_video_requires_id(self): + result = run_cli("video") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--id", result.stderr) + + def test_channel_requires_handle(self): + result = run_cli("channel") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--handle", result.stderr) + + def test_no_subcommand_prints_help_and_exits(self): + result = run_cli() + self.assertNotEqual(result.returncode, 0) + self.assertIn("usage", result.stdout) + + def test_limit_above_server_maximum_rejected(self): + result = run_cli("--dry-run", "--json", "videos", "--limit", "101") + self.assertNotEqual(result.returncode, 0) + self.assertIn("100", result.stderr) + + def test_limit_zero_rejected(self): + result = run_cli("--dry-run", "--json", "videos", "--limit", "0") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--limit", result.stderr) + + def test_negative_offset_rejected(self): + result = run_cli("--dry-run", "--json", "videos", "--offset", "-1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--offset", result.stderr) + + def test_login_without_password_errors(self): + result = run_cli("login", "--username", "alice", "--server", "https://inst.example") + self.assertNotEqual(result.returncode, 0) + self.assertIn("password", result.stderr.lower()) + self.assertNotIn("Traceback", result.stderr) + + def test_missing_server_dies_before_network(self): + result = run_cli("videos", "--limit", "1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("PEERTUBE_SERVER", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + +class DryRunPlanTests(ModuleStateTestCase): + """Class 3: --dry-run emits valid JSON plans with zero network activity.""" + + def run_json(self, *args): + pt.GLOBAL_FLAGS = {"json": True, "dry_run": True, "quiet": False, "verbose": False} + return io.StringIO() + + def test_single_endpoint_plans_emit_method_path_params(self): + cases = ( + ( + pt.cmd_videos, + ["--limit", "3"], + { + "method": "GET", + "path": "/api/v1/videos", + "params.start": 0, + "params.count": 3, + "params.sort": "-publishedAt", + }, + ), + ( + pt.cmd_search, + ["--query", "linux", "--limit", "5"], + { + "method": "GET", + "path": "/api/v1/search/videos", + "params.searchTarget": "local", + "params.search": "linux", + }, + ), + ( + pt.cmd_video, + ["--id", "uuid-one"], + {"method": "GET", "path": "/api/v1/videos/uuid-one"}, + ), + ( + pt.cmd_comments, + ["--id", "uuid-one"], + {"method": "GET", "path": "/api/v1/videos/uuid-one/comment-threads"}, + ), + (pt.cmd_channels, [], {"method": "GET", "path": "/api/v1/video-channels"}), + (pt.cmd_me, [], {"method": "GET", "path": "/api/v1/users/me"}), + (pt.cmd_my_videos, [], {"method": "GET", "path": "/api/v1/users/me/videos"}), + (pt.cmd_logout, [], {"method": "POST", "path": "/api/v1/users/revoke-token"}), + ) + for handler, args, expectations in cases: + with self.subTest(handler=handler.__name__): + client = pt.PeerTubeClient( + server="https://inst.example", + dry_run=True, + config_dir=tempfile.mkdtemp(prefix="pt-dry-"), + ) + out = io.StringIO() + with contextlib.redirect_stdout(out): + handler(client, args) + plan = json.loads(out.getvalue()) + self.assertTrue(plan["dry_run"]) + self.assertEqual(plan["method"], expectations["method"]) + self.assertEqual(plan["path"], expectations["path"]) + + def test_dry_run_videos_plan_never_sends_page_param(self): + client = pt.PeerTubeClient( + server="https://inst.example", + dry_run=True, + config_dir=tempfile.mkdtemp(prefix="pt-dry-"), + ) + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_videos(client, ["--limit", "9", "--offset", "18"]) + plan = json.loads(out.getvalue()) + self.assertNotIn("page", plan["params"]) + self.assertEqual(plan["params"]["start"], 18) + self.assertEqual(plan["params"]["count"], 9) + + def test_search_plan_defaults_to_instance_local_scope(self): + client = pt.PeerTubeClient( + server="https://inst.example", + dry_run=True, + config_dir=tempfile.mkdtemp(prefix="pt-dry-"), + ) + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_search(client, ["--query", "peertube"]) + plan = json.loads(out.getvalue()) + self.assertEqual(plan["params"]["searchTarget"], "local") + + def test_composite_commands_plan_every_request(self): + client = pt.PeerTubeClient( + server="https://inst.example", + dry_run=True, + config_dir=tempfile.mkdtemp(prefix="pt-dry-"), + ) + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_server(client, []) + plan = json.loads(out.getvalue()) + paths = [req["path"] for req in plan["requests"]] + self.assertEqual(paths, ["/api/v1/config/about", "/api/v1/server/stats"]) + + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_channel(client, ["--handle", "alice-channel"]) + plan = json.loads(out.getvalue()) + paths = [req["path"] for req in plan["requests"]] + self.assertEqual( + paths, + ["/api/v1/video-channels/alice-channel", "/api/v1/video-channels/alice-channel/videos"], + ) + + def test_login_dry_run_lists_form_fields_without_values(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_login( + pt.PeerTubeClient( + server="https://inst.example", + dry_run=True, + config_dir=tempfile.mkdtemp(prefix="pt-dry-"), + ), + ["--username", "alice"], + ) + plan = json.loads(out.getvalue()) + self.assertTrue(plan["dry_run"]) + self.assertEqual(plan["path"], "/api/v1/users/token") + self.assertIn("grant_type", plan["form_fields"]) + self.assertIn("client_secret", plan["form_fields"]) + self.assertNotIn("form", plan) # no values leak in the plan + + def test_dry_run_works_without_any_server_configured(self): + pt.ENV_SERVER = "" + client = pt.PeerTubeClient( + server="", dry_run=True, config_dir=tempfile.mkdtemp(prefix="pt-dry-") + ) + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_videos(client, ["--limit", "2"]) + self.assertTrue(json.loads(out.getvalue())["dry_run"]) + + def test_dry_run_never_touches_network(self): + client = pt.PeerTubeClient( + server="https://inst.example", + dry_run=True, + config_dir=tempfile.mkdtemp(prefix="pt-dry-"), + ) + with ( + patch.object(pt.requests, "get") as getter, + patch.object(pt.requests, "post") as poster, + ): + for handler, args in ( + (pt.cmd_videos, ["--limit", "2"]), + (pt.cmd_search, ["--query", "x"]), + (pt.cmd_server, []), + (pt.cmd_me, []), + (pt.cmd_login, ["--username", "a"]), + (pt.cmd_logout, []), + ): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + handler(client, args) + getter.assert_not_called() + poster.assert_not_called() + + def test_flags_work_before_and_after_subcommand(self): + result = run_cli("--json", "--dry-run", "search", "--query", "x", "--limit", "2") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(json.loads(result.stdout)["dry_run"]) + result = run_cli("search", "--query", "x", "--limit", "2", "--json", "--dry-run") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(json.loads(result.stdout)["dry_run"]) + + +class ClientContractTests(ModuleStateTestCase): + """Class 4a: mocked requests — paths, params, and error handling.""" + + def mocked_get(self, responses, server="https://inst.example"): + client = pt.PeerTubeClient(server=server, config_dir=tempfile.mkdtemp(prefix="pt-cc-")) + return client, patch.object(pt.requests, "get", side_effect=responses) + + def test_video_listing_sends_start_count_sort(self): + client, patcher = self.mocked_get( + [FakeResponse(200, {"total": 2, "data": [VIDEO_ONE, VIDEO_TWO]})] + ) + with patcher as getter: + pt.cmd_videos(client, ["--limit", "2", "--offset", "10"]) + args, kwargs = getter.call_args + self.assertEqual(args[0], "https://inst.example/api/v1/videos") + self.assertEqual(kwargs["params"], {"start": 10, "count": 2, "sort": "-publishedAt"}) + self.assertNotIn("page", kwargs["params"]) + + def test_search_defaults_to_local_target_and_omits_empty_sort(self): + client, patcher = self.mocked_get([FakeResponse(200, {"total": 0, "data": []})]) + with patcher as getter: + pt.cmd_search(client, ["--query", "linux"]) + params = getter.call_args[1]["params"] + self.assertEqual(params["searchTarget"], "local") + self.assertEqual(params["search"], "linux") + self.assertNotIn("sort", params) + + def test_comment_threads_route_is_hyphenated(self): + client, patcher = self.mocked_get( + [FakeResponse(200, {"total": 0, "totalNotDeletedComments": 0, "data": []})] + ) + with patcher as getter: + pt.cmd_comments(client, ["--id", "uuid-one"]) + self.assertEqual( + getter.call_args[0][0], "https://inst.example/api/v1/videos/uuid-one/comment-threads" + ) + + def test_server_composes_about_and_stats(self): + about = FakeResponse( + 200, {"instance": {"name": "Inst", "shortDescription": "Desc", "description": "Long"}} + ) + stats = FakeResponse( + 200, + { + "totalUsers": 9, + "totalLocalVideos": 933, + "totalVideos": 23890, + "totalLocalVideoViews": 1001751, + "totalLocalVideoDownloads": 32569, + "totalLocalVideoChannels": 28, + }, + ) + client, patcher = self.mocked_get([about, stats]) + with patcher: + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_server(client, []) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["instance"]["name"], "Inst") + self.assertEqual(payload["stats"]["totalLocalVideos"], 933) + self.assertIsInstance(payload["stats"]["totalUsers"], int) + + def test_401_names_login_remedy(self): + client, patcher = self.mocked_get([FakeResponse(401, {"detail": "token expired"})]) + client._token = "stale-token" # authed command proceeds, then server rejects + with patcher: + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + pt.cmd_me(client, []) + self.assertIn("401", err.getvalue()) + self.assertIn("login", err.getvalue().lower()) + + def test_429_surfaces_retry_after(self): + client, patcher = self.mocked_get( + [FakeResponse(429, {"detail": "rate limit"}, headers={"Retry-After": "7"})] + ) + with patcher: + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + pt.cmd_videos(client, ["--limit", "2"]) + self.assertIn("429", err.getvalue()) + self.assertIn("Retry-After", err.getvalue()) + + def test_rfc7807_detail_extracted_on_generic_error(self): + client, patcher = self.mocked_get( + [ + FakeResponse( + 400, + { + "type": "about:blank", + "title": "Bad Request", + "status": 400, + "detail": "unknown route shape", + }, + ) + ] + ) + with patcher: + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + pt.cmd_videos(client, ["--limit", "2"]) + self.assertIn("unknown route shape", err.getvalue()) + + def test_non_json_instance_response_is_diagnosed(self): + client, patcher = self.mocked_get([FakeResponse(200, text="not peertube")]) + with patcher: + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + pt.cmd_videos(client, ["--limit", "2"]) + self.assertIn("Non-JSON", err.getvalue()) + + +class OAuthFlowTests(ModuleStateTestCase): + """Class 4b: mocked OAuth2 — client fetch, password grant, persistence, + refresh, revocation. Token files live in TemporaryDirectories only.""" + + def setUp(self): + super().setUp() + self.tmp = tempfile.TemporaryDirectory(prefix="pt-oauth-") + self.config_dir = self.tmp.name + pt.ENV_SERVER = "https://inst.example" + + def tearDown(self): + self.tmp.cleanup() + super().tearDown() + + def client(self, **kwargs): + return pt.PeerTubeClient( + server="https://inst.example", config_dir=self.config_dir, **kwargs + ) + + def test_fetch_oauth_client_hits_singular_local_route(self): + client = self.client() + with patch.object( + pt.requests, "get", return_value=FakeResponse(200, OAUTH_CLIENT) + ) as getter: + client_id, client_secret = client.fetch_oauth_client() + self.assertEqual(getter.call_args[0][0], "https://inst.example/api/v1/oauth-clients/local") + self.assertEqual((client_id, client_secret), ("cid-1", "client-secret-1")) + + def test_password_grant_sends_form_encoded_fields(self): + client = self.client() + with ( + patch.object(pt.requests, "get", return_value=FakeResponse(200, OAUTH_CLIENT)), + patch.object( + pt.requests, "post", return_value=FakeResponse(200, TOKEN_RESPONSE) + ) as poster, + ): + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_login(client, ["--username", "alice", "--password", "pw"]) + args, kwargs = poster.call_args + self.assertEqual(args[0], "https://inst.example/api/v1/users/token") + form = kwargs["data"] + self.assertEqual(form["grant_type"], "password") + self.assertEqual(form["username"], "alice") + self.assertEqual(form["client_id"], "cid-1") + self.assertNotIn("response_type", form) # not part of the documented schema + payload = json.loads(out.getvalue()) + self.assertEqual(payload["status"], "logged_in") + + def test_bad_password_400_exits_with_guidance(self): + client = self.client() + with ( + patch.object(pt.requests, "get", return_value=FakeResponse(200, OAUTH_CLIENT)), + patch.object( + pt.requests, "post", return_value=FakeResponse(400, {"detail": "invalid_grant"}) + ), + ): + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + pt.cmd_login(client, ["--username", "alice", "--password", "wrong"]) + self.assertIn("400", err.getvalue()) + self.assertIn("invalid_grant", err.getvalue()) + + def test_two_factor_401_suggests_otp_flag(self): + client = self.client() + with ( + patch.object(pt.requests, "get", return_value=FakeResponse(200, OAUTH_CLIENT)), + patch.object(pt.requests, "post", return_value=FakeResponse(401, {})), + ): + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + pt.cmd_login(client, ["--username", "alice", "--password", "pw"]) + self.assertIn("--otp", err.getvalue()) + + def test_otp_header_attached_when_provided(self): + client = self.client() + with ( + patch.object(pt.requests, "get", return_value=FakeResponse(200, OAUTH_CLIENT)), + patch.object( + pt.requests, "post", return_value=FakeResponse(200, TOKEN_RESPONSE) + ) as poster, + ): + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_login(client, ["--username", "alice", "--password", "pw", "--otp", "123456"]) + self.assertEqual(poster.call_args[1]["headers"]["x-peertube-otp"], "123456") + + def test_masked_client_secret_stops_login_with_guidance(self): + client = self.client() + with ( + patch.object(pt.requests, "get", return_value=FakeResponse(200, MASKED_OAUTH_CLIENT)), + patch.object(pt.requests, "post") as poster, + ): + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + pt.cmd_login(client, ["--username", "alice", "--password", "pw"]) + self.assertIn("masks client_secret", err.getvalue()) + poster.assert_not_called() + + def test_is_masked_secret_detection(self): + self.assertTrue(pt.is_masked_secret("*" * 32)) + self.assertFalse(pt.is_masked_secret("client-secret-1")) + self.assertFalse(pt.is_masked_secret("")) + self.assertFalse(pt.is_masked_secret("*-mixed-*")) + + def test_token_file_persisted_owner_only_with_expiry(self): + client = self.client() + with ( + patch.object(pt.requests, "get", return_value=FakeResponse(200, OAUTH_CLIENT)), + patch.object(pt.requests, "post", return_value=FakeResponse(200, TOKEN_RESPONSE)), + ): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + pt.cmd_login(client, ["--username", "alice", "--password", "pw"]) + token_path = client.token_path() + self.assertTrue(os.path.isfile(token_path)) + mode = stat.S_IMODE(os.stat(token_path).st_mode) + self.assertEqual(mode & 0o077, 0, "token file must be owner-only") + with open(token_path) as handle: + record = json.load(handle) + self.assertEqual(record["server"], "https://inst.example") + self.assertEqual(record["access_token"], "tok-1") + self.assertEqual(record["refresh_token"], "ref-1") + self.assertIsNotNone(record["expires_at"]) + self.assertGreater(record["expires_at"], time.time()) + self.assertLess(record["expires_at"], time.time() + 7200) + + def test_token_from_another_instance_is_ignored(self): + client = self.client() + with ( + patch.object(pt.requests, "get", return_value=FakeResponse(200, OAUTH_CLIENT)), + patch.object(pt.requests, "post", return_value=FakeResponse(200, TOKEN_RESPONSE)), + ): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + pt.cmd_login(client, ["--username", "alice", "--password", "pw"]) + other = pt.PeerTubeClient(server="https://other.example", config_dir=self.config_dir) + self.assertIsNone(other._token) + + def test_expired_token_triggers_refresh_then_success(self): + client = self.client() + client.save_session(dict(TOKEN_RESPONSE, expires_in=-10)) # already expired + refreshed = dict(TOKEN_RESPONSE, access_token="tok-2", expires_in=3600) + with ( + patch.object(pt.requests, "get", return_value=FakeResponse(200, OAUTH_CLIENT)), + patch.object(pt.requests, "post", return_value=FakeResponse(200, refreshed)) as poster, + ): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + pt.cmd_me(client, []) + form = poster.call_args[1]["data"] + self.assertEqual(form["grant_type"], "refresh_token") + self.assertEqual(form["refresh_token"], "ref-1") + self.assertEqual(client._token, "tok-2") + with open(client.token_path()) as handle: + self.assertEqual(json.load(handle)["access_token"], "tok-2") + + def test_failed_refresh_falls_back_to_login_guidance(self): + client = self.client() + client.save_session(dict(TOKEN_RESPONSE, expires_in=-10)) + with ( + patch.object(pt.requests, "get", return_value=FakeResponse(200, OAUTH_CLIENT)), + patch.object(pt.requests, "post", return_value=FakeResponse(400, {})), + ): + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + pt.cmd_me(client, []) + self.assertIn("Not authenticated", err.getvalue()) + + def test_logout_revokes_and_deletes_token_file(self): + client = self.client() + client.save_session(TOKEN_RESPONSE) + self.assertTrue(os.path.isfile(client.token_path())) + with patch.object(pt.requests, "post", return_value=FakeResponse(200, {})) as poster: + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_logout(client, []) + args, kwargs = poster.call_args + self.assertEqual(args[0], "https://inst.example/api/v1/users/revoke-token") + self.assertEqual(kwargs["headers"]["Authorization"], "Bearer tok-1") + self.assertFalse(os.path.exists(client.token_path())) + + def test_logout_keeps_file_when_revocation_fails(self): + client = self.client() + client.save_session(TOKEN_RESPONSE) + with patch.object(pt.requests, "post", return_value=FakeResponse(500, {})): + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + pt.cmd_logout(client, []) + self.assertTrue(os.path.isfile(client.token_path())) + + def test_logout_without_token_errors_cleanly(self): + client = self.client() + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + pt.cmd_logout(client, []) + self.assertIn("No stored token", err.getvalue()) + + +class HandlerOutputTests(ModuleStateTestCase): + """Class 4c: handler output contracts consumed by jq pipelines.""" + + def test_videos_output_carries_raw_video_objects(self): + client = pt.PeerTubeClient( + server="https://inst.example", config_dir=tempfile.mkdtemp(prefix="pt-ho-") + ) + with patch.object( + pt.requests, + "get", + return_value=FakeResponse(200, {"total": 2, "data": [VIDEO_ONE, VIDEO_TWO]}), + ): + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_videos(client, ["--limit", "2"]) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["total"], 2) + self.assertEqual(payload["start"], 0) + self.assertEqual(payload["count"], 2) + self.assertIsInstance(payload["videos"], list) + first = payload["videos"][0] + self.assertEqual(first["uuid"], "uuid-one") + self.assertIsInstance(first["duration"], int) # seconds + self.assertEqual(first["channel"]["host"], "inst.example") + + def test_search_output_marks_scope_and_carries_uuids(self): + client = pt.PeerTubeClient( + server="https://inst.example", config_dir=tempfile.mkdtemp(prefix="pt-ho-") + ) + with patch.object( + pt.requests, "get", return_value=FakeResponse(200, {"total": 1, "data": [VIDEO_TWO]}) + ): + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_search(client, ["--query", "x"]) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["videos"][0]["uuid"], "uuid-two") + + def test_me_tolerates_docs_array_sample_and_object(self): + client = pt.PeerTubeClient( + server="https://inst.example", config_dir=tempfile.mkdtemp(prefix="pt-ho-") + ) + profile = { + "username": "alice", + "role": {"id": 1, "label": "User"}, + "videoQuota": 1073741824, + "videoChannels": [], + } + for body in (profile, [profile]): + client.save_session(TOKEN_RESPONSE) + with patch.object(pt.requests, "get", return_value=FakeResponse(200, body)): + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_me(client, []) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["username"], "alice") + self.assertEqual(payload["role"]["label"], "User") + client.clear_token() + + def test_comments_output_exposes_thread_counts(self): + client = pt.PeerTubeClient( + server="https://inst.example", config_dir=tempfile.mkdtemp(prefix="pt-ho-") + ) + body = { + "total": 1, + "totalNotDeletedComments": 3, + "data": [ + {"totalReplies": 3, "comment": {"text": "nice video", "account": {"name": "bob"}}} + ], + } + with patch.object(pt.requests, "get", return_value=FakeResponse(200, body)): + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_comments(client, ["--id", "uuid-one"]) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["total"], 1) + self.assertEqual(payload["total_not_deleted"], 3) + self.assertEqual(payload["threads"][0]["comment"]["text"], "nice video") + + def test_channels_output_carries_handles_and_counts(self): + client = pt.PeerTubeClient( + server="https://inst.example", config_dir=tempfile.mkdtemp(prefix="pt-ho-") + ) + body = { + "total": 1, + "data": [ + { + "name": "alice-channel", + "displayName": "Alice Channel", + "host": "inst.example", + "videosCount": 12, + "followersCount": 34, + } + ], + } + with patch.object(pt.requests, "get", return_value=FakeResponse(200, body)): + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_channels(client, []) + payload = json.loads(out.getvalue()) + channel = payload["channels"][0] + self.assertEqual(channel["name"], "alice-channel") + self.assertIsInstance(channel["videosCount"], int) + + +class PipelineChainTests(ModuleStateTestCase): + """Documented multi-step recipes must execute stage by stage, each stage's + output field names AND JSON types consumable by the next.""" + + @classmethod + def setUpClass(cls): + cls.tmpdir = tempfile.TemporaryDirectory(prefix="pt-pipeline-") + + @classmethod + def tearDownClass(cls): + cls.tmpdir.cleanup() + + def run_cli(self, *args): + env = clean_env() + env["PEERTUBE_SERVER"] = "https://inst.example" + env["PEERTUBE_CONFIG_DIR"] = self.tmpdir.name + return subprocess.run( + [sys.executable, str(SCRIPT), "--json", "--dry-run", *args], + capture_output=True, + text=True, + env=env, + ) + + def run_jq(self, *jq_args, stdin_text=""): + return subprocess.run( + ["jq", *jq_args], input=stdin_text, capture_output=True, text=True, env=clean_env() + ) + + def stage_file(self, name, document): + path = pathlib.Path(self.tmpdir.name) / name + path.write_text(json.dumps(document)) + return str(path) + + def test_browse_then_detail_chain_consumability(self): + # Stage 1: videos plan; jq proves the path and the start-offset type + # (number) that stage two consumes when picking an id from the listing. + r1 = self.run_cli("videos", "--limit", "2") + self.assertEqual(r1.returncode, 0, r1.stderr) + self.stage_file("s1.json", json.loads(r1.stdout)) + self.assertEqual( + self.run_jq("-r", ".path", stdin_text=r1.stdout).stdout.strip(), "/api/v1/videos" + ) + self.assertEqual( + self.run_jq("-r", ".params.start | type", stdin_text=r1.stdout).stdout.strip(), "number" + ) + # Stage 2: detail plan consumes an id into the URL path. + r2 = self.run_cli("video", "--id", "uuid-one") + self.assertEqual(r2.returncode, 0, r2.stderr) + self.stage_file("s2.json", json.loads(r2.stdout)) + self.assertEqual( + self.run_jq("-r", ".path", stdin_text=r2.stdout).stdout.strip(), + "/api/v1/videos/uuid-one", + ) + + def test_search_then_video_chain_consumability(self): + r1 = self.run_cli("search", "--query", "linux", "--limit", "3") + self.assertEqual(r1.returncode, 0, r1.stderr) + self.assertEqual( + self.run_jq("-r", ".params.searchTarget", stdin_text=r1.stdout).stdout.strip(), "local" + ) + self.assertEqual( + self.run_jq("-r", ".params.count | type", stdin_text=r1.stdout).stdout.strip(), "number" + ) + # The documented jq selector .videos[0].uuid maps to detail --id. + r2 = self.run_cli("video", "--id", "uuid-from-search") + self.assertEqual(r2.returncode, 0, r2.stderr) + self.assertEqual( + self.run_jq("-r", ".path", stdin_text=r2.stdout).stdout.strip(), + "/api/v1/videos/uuid-from-search", + ) + + def test_channel_offset_paging_chain_consumability(self): + r1 = self.run_cli("channels", "--limit", "100", "--offset", "0") + self.assertEqual(r1.returncode, 0, r1.stderr) + self.assertEqual( + self.run_jq("-r", ".path", stdin_text=r1.stdout).stdout.strip(), + "/api/v1/video-channels", + ) + r2 = self.run_cli( + "channel", "--handle", "alice-channel", "--limit", "100", "--offset", "100" + ) + self.assertEqual(r2.returncode, 0, r2.stderr) + plan = json.loads(r2.stdout) + video_req = plan["requests"][1] + self.assertEqual(video_req["params"]["start"], 100) + self.assertEqual(video_req["params"]["count"], 100) + self.assertNotIn("page", video_req["params"]) + + def test_login_to_me_chain_handoff(self): + # Stage 1: login plan lists the form fields (no values). + r1 = self.run_cli("login", "--username", "alice") + self.assertEqual(r1.returncode, 0, r1.stderr) + self.assertEqual( + self.run_jq("-r", ".path", stdin_text=r1.stdout).stdout.strip(), "/api/v1/users/token" + ) + fields = json.loads(self.run_jq("-c", ".form_fields", stdin_text=r1.stdout).stdout) + self.assertIn("grant_type", fields) + # Stage 2: me plan rides the Authorization header the login persisted. + r2 = self.run_cli("me") + self.assertEqual(r2.returncode, 0, r2.stderr) + self.assertEqual( + self.run_jq("-r", ".path", stdin_text=r2.stdout).stdout.strip(), "/api/v1/users/me" + ) + # Stage 3: logout plan revokes on the same instance. + r3 = self.run_cli("logout") + self.assertEqual(r3.returncode, 0, r3.stderr) + self.assertEqual( + self.run_jq("-r", ".path", stdin_text=r3.stdout).stdout.strip(), + "/api/v1/users/revoke-token", + ) + + def test_server_composition_plan_targets_both_endpoints(self): + r1 = self.run_cli("server") + self.assertEqual(r1.returncode, 0, r1.stderr) + paths = json.loads(self.run_jq("-c", "[.requests[].path]", stdin_text=r1.stdout).stdout) + self.assertEqual(paths, ["/api/v1/config/about", "/api/v1/server/stats"]) + + def test_mocked_browse_to_detail_stage_types(self): + """Live-shape variant of recipe 1: the videos output's uuid (string) + feeds video --id, and the detail object carries description/url.""" + pt.GLOBAL_FLAGS = {"json": True, "dry_run": False, "quiet": False, "verbose": False} + client = pt.PeerTubeClient(server="https://inst.example", config_dir=self.tmpdir.name) + detail = dict(VIDEO_ONE, description="full text", commentsEnabled=True) + with patch.object( + pt.requests, + "get", + side_effect=[ + FakeResponse(200, {"total": 1, "data": [VIDEO_ONE]}), + FakeResponse(200, detail), + ], + ): + first = io.StringIO() + with contextlib.redirect_stdout(first): + pt.cmd_videos(client, ["--limit", "1"]) + listing = json.loads(first.getvalue()) + consumed_id = listing["videos"][0]["uuid"] + self.assertIsInstance(consumed_id, str) + + second = io.StringIO() + with contextlib.redirect_stdout(second): + pt.cmd_video(client, ["--id", consumed_id]) + video_detail = json.loads(second.getvalue()) + self.assertEqual(video_detail["uuid"], consumed_id) + self.assertIsInstance(video_detail["description"], str) + self.assertIsInstance(video_detail["commentsEnabled"], bool) + + +class EnvGuardedLiveProbeTests(unittest.TestCase): + """Optional anonymous instance probe (keyless public endpoint). Runs only + with PEERTUBE_LIVE_TESTS=1; skipped cleanly otherwise so the suite stays + fully offline under the proxy-trap.""" + + def test_public_instance_oauth_client_probe(self): + if os.getenv("PEERTUBE_LIVE_TESTS") != "1": + self.skipTest("live probe disabled (set PEERTUBE_LIVE_TESTS=1)") + result = subprocess.run( + [sys.executable, str(SCRIPT), "--json", "server", "--server", "https://framatube.org"], + capture_output=True, + text=True, + env=clean_env(), + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(payload["instance"]["name"], "Framatube") + self.assertIsInstance(payload["stats"]["totalLocalVideos"], int) + + +if __name__ == "__main__": + unittest.main() diff --git a/references/skill-triggers.md b/references/skill-triggers.md index a455567..80fe1f7 100644 --- a/references/skill-triggers.md +++ b/references/skill-triggers.md @@ -22,6 +22,7 @@ Each skill's `description` field is the canonical routing contract. This conveni | "Jellyfin", "Jellyfin media server", "recently added movies", "recently added episodes", "media library", "JELLYFIN_API_KEY", "Jellyfin API authentication" | [jellyfin](../jellyfin/SKILL.md) | | "Jira", "Atlassian Jira", "JQL", "ticket PROJ-123", "sprint work", "JIRA_API_TOKEN" | [jira](../jira/SKILL.md) | | "Open Library", "openlibrary", "book search", "ISBN lookup", "author records", "work details", "book editions", "book ratings", "cover image" | [openlibrary](../openlibrary/SKILL.md) | +| "PeerTube", "peertube", "federated video", "SepiaSearch", "decentralized video platform", "PEERTUBE_SERVER", "my PeerTube instance" | [peertube](../peertube/SKILL.md) | | "weather", "forecast", "temperature", "is it raining", "Tempest" | [tempest](../tempest/SKILL.md) | | "TMDb", "The Movie Database", "movie search", "trending movies", "upcoming TV releases", "TMDB_ACCESS_TOKEN" | [tmdb](../tmdb/SKILL.md) | | "traefik", "reverse proxy", "load balancer", "API gateway", "Let's Encrypt", "ACME", "Docker routing", "traefik.yml", "entry point", "middleware", "TLS termination", "forward auth", "rate limit" | [traefik](../traefik/SKILL.md) | From a20b66e6c103f6595d8782675b9a6bf725fe85c4 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sat, 29 Aug 2026 19:53:50 -0400 Subject: [PATCH 34/40] fix(ghost): send source=html on html writes and repair scrutiny round-1 findings Admin API writes carrying an html payload (create-post, update-post, create-page) now attach the docs-required ?source=html query flag; _post/_put gained params plumbing, and the create-post dry-run plan includes params for plan/request parity. Regression tests pin the param whenever an html payload is present (mocked request capture and dry-run plans) and assert its absence on mobiledoc/lexical writes. Docs: fix the jq interpolation typo in worked-recipes recipe 2 (missing backslash made the scheduled-posts line exit 5), replace the blanket "exit code 2" claim in admin-auth-and-basics with the script's actual 2-5 failure-class mapping, and state the source=html requirement in the SKILL.md --html gotcha. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- ghost/SKILL.md | 2 +- ghost/references/admin-auth-and-basics.md | 2 +- ghost/references/worked-recipes.md | 2 +- ghost/scripts/ghost | 28 ++++-- ghost/scripts/test_ghost.py | 114 ++++++++++++++++++++++ 5 files changed, 136 insertions(+), 12 deletions(-) diff --git a/ghost/SKILL.md b/ghost/SKILL.md index 02e4a6d..9767a77 100644 --- a/ghost/SKILL.md +++ b/ghost/SKILL.md @@ -141,7 +141,7 @@ JSON shapes worth knowing: - **HS256 only, decoded-secret keying.** Tokens signed with HS512 are refused ("invalid algorithm"); signing without first hex-decoding the secret half produces "valid-looking" garbage that 401s. The CLI handles both rules. - **`Authorization: Ghost`, not Bearer.** `Bearer` scheme answers 401 `INVALID_AUTH_HEADER`. - **Edits require collision guards.** PUTs without the post's current `updated_at` fail with 409; relation arrays (`tags`, `authors`) replace wholesale rather than merge. -- **HTML ingestion is lossy without cards.** Send proper Lexical, or wrap fixed markup in HTML card comments when using `--html`. +- **`--html` requires `source=html` and stays lossy.** Every write carrying an `html` payload (create-post, update-post, create-page) must send the `?source=html` query flag — the CLI attaches it automatically, and dry-run plans show it in `params` — or Ghost parses the body as mobiledoc/lexical. Even with the flag, conversion is lossy: send proper Lexical, or wrap fixed markup in HTML card comments. - **Pagination caps at 100** since Ghost 6 removed `limit=all`; oversized limits silently return ≤100 rows, so always loop by `next`. - **Deletion is permanent** and takes effect on the public site immediately. diff --git a/ghost/references/admin-auth-and-basics.md b/ghost/references/admin-auth-and-basics.md index db8c1a6..59a3a43 100644 --- a/ghost/references/admin-auth-and-basics.md +++ b/ghost/references/admin-auth-and-basics.md @@ -88,7 +88,7 @@ Ghost returns JSON errors shaped like `{"errors": [{"message", "context", "type" | Malformed token JSON/base64, `INVALID_JWT` | 400 | Structurally undecodable token | | No auth at all → `Authorization failed`, type `NoPermissionError` | 403 | Missing `Authorization` header entirely | -The CLI maps each of these to exit code 2 with the server message plus a hint. +The CLI surfaces each of these on stderr with the server message plus a hint, mapped to exit codes by failure class: `401` and `403` exit `2` (auth/permission), `404` exits `3` (missing resource), `409` exits `4` (update collision), and `429` exits `5` (rate limited). ## Request conventions diff --git a/ghost/references/worked-recipes.md b/ghost/references/worked-recipes.md index 8755869..c57d78f 100644 --- a/ghost/references/worked-recipes.md +++ b/ghost/references/worked-recipes.md @@ -40,7 +40,7 @@ Drafts, scheduled, and published live on different filters; one call per status: ```bash ghost posts --status draft --limit 50 --json | jq -r '.posts[] | "\(.title)\t\(.slug)"' -ghost posts --status scheduled --json | jq -r '.posts[] | "\(.title)\t(.published_at // "-")"' +ghost posts --status scheduled --json | jq -r '.posts[] | "\(.title)\t\(.published_at // "-")"' ``` Pair it with an authoring-wide sanity check via jq types before feeding slugs onward: diff --git a/ghost/scripts/ghost b/ghost/scripts/ghost index d9057d6..f2da259 100755 --- a/ghost/scripts/ghost +++ b/ghost/scripts/ghost @@ -210,11 +210,11 @@ class GhostClient: def _get(self, path, params=None): return self._request("get", path, params=params) - def _post(self, path, json_data): - return self._request("post", path, json_data=json_data) + def _post(self, path, json_data, params=None): + return self._request("post", path, params=params, json_data=json_data) - def _put(self, path, json_data): - return self._request("put", path, json_data=json_data) + def _put(self, path, json_data, params=None): + return self._request("put", path, params=params, json_data=json_data) def _delete(self, path): return self._request("delete", path) @@ -241,10 +241,16 @@ class GhostClient: post["slug"] = slug if published_at: post["published_at"] = published_at - return self._post("/posts", {"posts": [post]}) + # html payloads need the docs-required source=html flag; Ghost otherwise + # parses the body as mobiledoc/lexical and rejects or mangles it. + params = {"source": "html"} if html else None + return self._post("/posts", {"posts": [post]}, params=params) def update_post(self, post_id, **kwargs): - return self._put(f"/posts/{post_id}", {"posts": [kwargs]}) + # html payloads need the docs-required source=html flag; Ghost otherwise + # parses the body as mobiledoc/lexical and rejects or mangles it. + params = {"source": "html"} if kwargs.get("html") else None + return self._put(f"/posts/{post_id}", {"posts": [kwargs]}, params=params) def delete_post(self, post_id): return self._delete(f"/posts/{post_id}") @@ -261,7 +267,10 @@ class GhostClient: page["html"] = html if slug: page["slug"] = slug - return self._post("/pages", {"pages": [page]}) + # html payloads need the docs-required source=html flag; Ghost otherwise + # parses the body as mobiledoc/lexical and rejects or mangles it. + params = {"source": "html"} if html else None + return self._post("/pages", {"pages": [page]}, params=params) def get_tags(self, limit=50, page=None): params: Dict[str, Any] = {"limit": limit, "include": "count.posts"} @@ -339,9 +348,10 @@ def cmd_create_post(client, args): slug=parsed.slug, published_at=parsed.published_at) or {} if data.get("dry_run"): # Plan and real request share one code path, so the previewed URL, - # method, and JSON envelope are exactly what execution would send. + # method, params, and JSON envelope are exactly what execution would send. plan = {"dry_run": True, "method": data.get("method"), - "url": data.get("url"), "json": data.get("json"), **vars(parsed)} + "url": data.get("url"), "params": data.get("params"), + "json": data.get("json"), **vars(parsed)} return emit(f"[dry-run] Create post '{parsed.title}' " f"-> {data.get('method', 'POST').upper()} {data.get('url')}", plan) posts = data.get("posts", []) diff --git a/ghost/scripts/test_ghost.py b/ghost/scripts/test_ghost.py index ef07814..0be40a3 100644 --- a/ghost/scripts/test_ghost.py +++ b/ghost/scripts/test_ghost.py @@ -123,6 +123,59 @@ class GhostCliTests(unittest.TestCase): self.assertEqual(fields["updated_at"], "2026-08-26T12:00:00.000Z") self.assertEqual(fields["status"], "published") + def test_create_post_dry_run_sends_source_html_only_with_html_payload(self): + # Ghost parses html write payloads as mobiledoc/lexical unless the + # docs-required ?source=html query flag rides along; the plan must + # preview exactly that request. + result = self.run_cli("--dry-run", "--json", "create-post", + "--title", "Doc", "--html", "

Hi

") + self.assertEqual(result.returncode, 0) + payload = json.loads(result.stdout) + self.assertEqual(payload["params"], {"source": "html"}) + self.assertIn("html", payload["json"]["posts"][0]) + + no_html = self.run_cli("--dry-run", "--json", "create-post", + "--title", "Doc") + self.assertEqual(no_html.returncode, 0) + no_html_payload = json.loads(no_html.stdout) + self.assertNotIn("html", no_html_payload["json"]["posts"][0]) + self.assertNotIn("source=html", no_html_payload["url"]) + self.assertIsNone(no_html_payload["params"]) + + def test_update_post_dry_run_sends_source_html_only_with_html_payload(self): + result = self.run_cli("--dry-run", "--json", "update-post", "abc123", + "--html", "

Edited

", + "--updated-at", "2026-08-26T12:00:00.000Z") + self.assertEqual(result.returncode, 0) + payload = json.loads(result.stdout) + self.assertEqual(payload["params"], {"source": "html"}) + self.assertIn("html", payload["fields"]) + + no_html = self.run_cli("--dry-run", "--json", "update-post", "abc123", + "--status", "draft", + "--updated-at", "2026-08-26T12:00:00.000Z") + self.assertEqual(no_html.returncode, 0) + no_html_payload = json.loads(no_html.stdout) + self.assertNotIn("html", no_html_payload["fields"]) + self.assertNotIn("source=html", no_html_payload["url"]) + self.assertIsNone(no_html_payload["params"]) + + def test_create_page_dry_run_sends_source_html_only_with_html_payload(self): + result = self.run_cli("--dry-run", "--json", "create-page", + "--title", "About", "--html", "

About us

") + self.assertEqual(result.returncode, 0) + payload = json.loads(result.stdout) + self.assertEqual(payload["params"], {"source": "html"}) + self.assertIn("html", payload["json"]["pages"][0]) + + no_html = self.run_cli("--dry-run", "--json", "create-page", + "--title", "About") + self.assertEqual(no_html.returncode, 0) + no_html_payload = json.loads(no_html.stdout) + self.assertNotIn("html", no_html_payload["json"]["pages"][0]) + self.assertNotIn("source=html", no_html_payload["url"]) + self.assertIsNone(no_html_payload["params"]) + def test_scheduled_post_requires_published_at_even_in_dry_run(self): result = self.run_cli("--dry-run", "--json", "create-post", "--title", "Later", "--status", "scheduled") @@ -442,5 +495,66 @@ class MockedClientTests(unittest.TestCase): self.assertEqual(called_url, "https://example.com/ghost/api/admin/site") +class HtmlSourceFlagTests(unittest.TestCase): + """Regression: html write payloads must carry the docs-required + ?source=html query flag; mobiledoc/lexical writes must not send it.""" + + def setUp(self): + self.cli = load_cli() + self.cli.GLOBAL_FLAGS = {"json": True, "dry_run": False, + "quiet": False, "verbose": False} + ok = Mock(status_code=201) + ok.json = Mock(return_value={"posts": []}) + self.cli.requests.post = Mock(return_value=ok) + self.client = self.cli.GhostClient(url="https://example.com", key=FIXED_KEY) + + def _put_ok(self): + ok = Mock(status_code=200) + ok.json = Mock(return_value={"posts": []}) + self.cli.requests.put = Mock(return_value=ok) + return self.cli.requests.put + + def test_create_post_with_html_sends_source_html_query_param(self): + self.client.create_post("Doc", html="

Hi

") + call = self.cli.requests.post.call_args + self.assertEqual(call.kwargs["params"], {"source": "html"}) + self.assertIn("html", call.kwargs["json"]["posts"][0]) + + def test_create_post_without_html_omits_source_flag(self): + self.client.create_post("Doc") + call = self.cli.requests.post.call_args + self.assertNotIn("source", (call.kwargs.get("params") or {})) + self.assertNotIn("html", call.kwargs["json"]["posts"][0]) + + def test_update_post_with_html_sends_source_html_query_param(self): + put = self._put_ok() + self.client.update_post("abc123", html="

Edited

", + updated_at="2026-08-26T12:00:00.000Z") + call = put.call_args + self.assertEqual(call.kwargs["params"], {"source": "html"}) + self.assertIn("html", call.kwargs["json"]["posts"][0]) + + def test_update_post_without_html_omits_source_flag(self): + put = self._put_ok() + self.client.update_post("abc123", status="published", + updated_at="2026-08-26T12:00:00.000Z") + call = put.call_args + self.assertNotIn("source", (call.kwargs.get("params") or {})) + self.assertNotIn("html", call.kwargs["json"]["posts"][0]) + + def test_create_page_with_html_sends_source_html_query_param(self): + self.client.create_page("About", html="

About us

") + call = self.cli.requests.post.call_args + self.assertEqual(call.kwargs["params"], {"source": "html"}) + self.assertEqual(call.args[0], "https://example.com/ghost/api/admin/pages") + self.assertIn("html", call.kwargs["json"]["pages"][0]) + + def test_create_page_without_html_omits_source_flag(self): + self.client.create_page("About") + call = self.cli.requests.post.call_args + self.assertNotIn("source", (call.kwargs.get("params") or {})) + self.assertNotIn("html", call.kwargs["json"]["pages"][0]) + + if __name__ == "__main__": unittest.main() From 08689dfd8f3aa57305aa4f69cf3b2bc6f0b50945 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sat, 29 Aug 2026 20:32:06 -0400 Subject: [PATCH 35/40] fix(media-home): repair scrutiny round-1 doc-vs-reality findings jellyfin: send the access token over exactly ONE channel per request (drop the simultaneous X-Emby-Token header; the MediaBrowser Token= parameter is the sole transport, legacy fallback remains documented as substitute-never- stack and is request-capture tested); remove the dead no-op conditional in JellyfinClient.__init__; make the login error-path test exception-safe with patch.object; document the test-pinned dry-run plan keys and the actual 0/1/2 exit-code mapping in worked-recipes and SKILL.md. peertube: replace the stale dry-run shape prose ('url'/'form') with the test-pinned {dry_run, method, path, params} / form_fields keys; harden cmd_me against a non-dict role (no AttributeError) with regression tests; remove the dead client facade, the unused cmd_channels variable, and the unused List/Tuple imports (ruff F401/F841 clean). ghost: fold the 5 nested with-statements (ruff SIM117) in test_ghost.py into single with-statements. All three skills double-runner + proxy-trap green (25/56/37 tests); validate-evals, paired smoke, quality validator, core gates, and catalog check modes green. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- ghost/scripts/test_ghost.py | 27 ++++------ jellyfin/SKILL.md | 12 +++-- jellyfin/references/auth-and-sessions.md | 10 ++-- jellyfin/references/gotchas-field-guide.md | 5 +- jellyfin/references/worked-recipes.md | 60 +++++++++++++++++++--- jellyfin/scripts/jellyfin | 16 +++--- jellyfin/scripts/test_jellyfin_cli.py | 39 +++++++++++--- peertube/references/worked-recipes.md | 8 +-- peertube/scripts/peertube | 53 +++---------------- peertube/scripts/test_peertube.py | 29 +++++++++++ 10 files changed, 164 insertions(+), 95 deletions(-) diff --git a/ghost/scripts/test_ghost.py b/ghost/scripts/test_ghost.py index 0be40a3..cab94c6 100644 --- a/ghost/scripts/test_ghost.py +++ b/ghost/scripts/test_ghost.py @@ -238,17 +238,15 @@ class JwtSigningTests(unittest.TestCase): def test_malformed_secret_half_is_graceful_error_not_traceback(self): client = self.cli.GhostClient(url="https://example.com", key="5f9d4b1c8e2a43d7b6c0a1e9:not-hex!") stderr = io.StringIO() - with self.assertRaises(SystemExit): - with contextlib.redirect_stderr(stderr): - client._jwt_token() + with self.assertRaises(SystemExit), contextlib.redirect_stderr(stderr): + client._jwt_token() self.assertIn("hexadecimal", stderr.getvalue()) def test_key_without_colon_names_required_format(self): client = self.cli.GhostClient(url="https://example.com", key="justonepart") stderr = io.StringIO() - with self.assertRaises(SystemExit): - with contextlib.redirect_stderr(stderr): - client._jwt_token() + with self.assertRaises(SystemExit), contextlib.redirect_stderr(stderr): + client._jwt_token() self.assertIn("id:secret", stderr.getvalue()) @@ -431,10 +429,9 @@ class MockedClientTests(unittest.TestCase): cli.requests.put = Mock(return_value=collision) client = cli.GhostClient(url="https://example.com", key=FIXED_KEY) stderr = io.StringIO() - with self.assertRaises(SystemExit): - with contextlib.redirect_stderr(stderr): - client.update_post("abc123", status="published", - updated_at="2026-01-01T00:00:00.000Z") + with self.assertRaises(SystemExit), contextlib.redirect_stderr(stderr): + client.update_post("abc123", status="published", + updated_at="2026-01-01T00:00:00.000Z") message = stderr.getvalue() self.assertIn("409", message) self.assertIn("Someone else is editing this post", message) @@ -449,9 +446,8 @@ class MockedClientTests(unittest.TestCase): cli.requests.get = Mock(return_value=missing) client = cli.GhostClient(url="https://example.com", key=FIXED_KEY) stderr = io.StringIO() - with self.assertRaises(SystemExit): - with contextlib.redirect_stderr(stderr): - client._get("/posts/does-not-exist") + with self.assertRaises(SystemExit), contextlib.redirect_stderr(stderr): + client._get("/posts/does-not-exist") message = stderr.getvalue() self.assertIn("404", message) self.assertIn("Admin API", message) @@ -468,9 +464,8 @@ class MockedClientTests(unittest.TestCase): cli.requests.get = Mock(return_value=bad_scheme) client = cli.GhostClient(url="https://example.com", key=FIXED_KEY) stderr = io.StringIO() - with self.assertRaises(SystemExit): - with contextlib.redirect_stderr(stderr): - client._get("/posts") + with self.assertRaises(SystemExit), contextlib.redirect_stderr(stderr): + client._get("/posts") self.assertIn("Ghost [token]", stderr.getvalue()) def test_authorization_header_uses_ghost_scheme_and_version_headers(self): diff --git a/jellyfin/SKILL.md b/jellyfin/SKILL.md index 188b6a3..7f7569c 100644 --- a/jellyfin/SKILL.md +++ b/jellyfin/SKILL.md @@ -49,7 +49,8 @@ endpoint requires its `Client=..., Device=..., DeviceId=..., Version=...` quarte any token exists** — the server rejects `POST /Users/AuthenticateByName` with `400 Error processing request.` otherwise. Afterwards the access token (or API key) rides the same header as `Token="..."`; the legacy `X-Emby-Token` header means the same thing -and is scheduled for removal from Jellyfin 12.0. The bundled CLI sends the modern form. +and is scheduled for removal from Jellyfin 12.0. The bundled CLI sends the modern form and +puts the token in exactly that one channel per request (never co-sends `X-Emby-Token`). See [references/auth-and-sessions.md](references/auth-and-sessions.md). ## Essential Commands @@ -156,10 +157,11 @@ scripts/jellyfin login --username alice --prompt --json | jq -r '"\(.user_id) \( Put `--json` before or after the subcommand. Output keys are stable snake_case: `items` (with `id`, `name`, `type`, `year`, `series`, `season_number`, `episode_number`), -`results`, `libraries`, `total_record_count`, `start_index`. `--dry-run` emits the exact -`path` + `params` (or `authorization_header` for `login`) that would be sent, so jq can -verify a chain before running it live. Use `jq -r '.items[] | [.name, .year] | @tsv'` for -tabular handoff. +`results`, `libraries`, `total_record_count`, `start_index`. `--dry-run` emits a plan +carrying `dry_run`, `path`, and `params` (`login` adds `authorization_header`; `info` +composes a `requests` list), matching what would be sent, so jq can verify a chain before +running it live. Exit codes: `0` success (including dry-run), `1` CLI/API errors, `2` +argument errors. Use `jq -r '.items[] | [.name, .year] | @tsv'` for tabular handoff. ## Known Gotchas diff --git a/jellyfin/references/auth-and-sessions.md b/jellyfin/references/auth-and-sessions.md index 9060abb..7300ea5 100644 --- a/jellyfin/references/auth-and-sessions.md +++ b/jellyfin/references/auth-and-sessions.md @@ -123,7 +123,10 @@ It is wire-equivalent to the legacy `X-Emby-Token` header on every server that s legacy auth, and it keeps working when legacy channels are switched off. Never send two different tokens in one request — server precedence across channels is -unspecified at the contract level and the value used becomes uncertain. +unspecified at the contract level and the value used becomes uncertain. The bundled CLI +takes this literally: after login it puts the token in exactly ONE channel per request +(the `Token=` parameter of the modern header) and never attaches `X-Emby-Token` alongside +it; the offline suite pins that single-channel contract with a request-capture test. ### Legacy kill-switch and deprecation timeline @@ -133,8 +136,9 @@ unspecified at the contract level and the value used becomes uncertain. `X-Emby-Authorization` stop resolving — a client that "worked yesterday" and now gets uniform 401s has almost certainly met this toggle. - Maintainers have targeted disabling the deprecated options starting with the 12.0 - release. Speak the modern `Authorization` header natively; keep `X-Emby-Token` only as a - compat fallback for old servers. + release. Speak the modern `Authorization` header natively; if you must support a server + with legacy auth locked off, substitute the legacy header for the modern one on that + server's requests — never send both on the same request. ## Error signatures worth mocking diff --git a/jellyfin/references/gotchas-field-guide.md b/jellyfin/references/gotchas-field-guide.md index 1286351..80fce26 100644 --- a/jellyfin/references/gotchas-field-guide.md +++ b/jellyfin/references/gotchas-field-guide.md @@ -52,7 +52,10 @@ proxy, not Jellyfin. - One access token per `(DeviceId, user)` pair: re-logging-in the same pair revokes the pair's previous token. Multi-profile CLIs must vary the DeviceId per profile or they will keep logging each other out. -- Never send two token channels in one request; which one wins is not contractual. +- Never send two token channels in one request; which one wins is not contractual. The + bundled CLI honors this literally: the token rides only the MediaBrowser `Token=` + parameter and the legacy `X-Emby-Token` header is never attached (request-capture-tested + in the offline suite). ## User-scoping pitfalls diff --git a/jellyfin/references/worked-recipes.md b/jellyfin/references/worked-recipes.md index 55b09e2..90bc52d 100644 --- a/jellyfin/references/worked-recipes.md +++ b/jellyfin/references/worked-recipes.md @@ -41,10 +41,12 @@ request.`; wrong password → 401; server restarting → 503 + `Retry-After` (re With the bundled CLI, steps 2–5 collapse to env vars: export `JELLYFIN_URL`, `JELLYFIN_API_KEY` (or `JELLYFIN_TOKEN`), `JELLYFIN_USER_ID`; then -`scripts/jellyfin recent --json`. The login path itself is available as -`scripts/jellyfin login --username alice` (reads `JELLYFIN_PASSWORD` interactively or via -`--password`/`--password-stdin`), which prints the captured `user_id`/`access_token` for -exporting. +`scripts/jellyfin recent --json`. Post-login requests carry the token in exactly one +channel — the `Token=` parameter of the MediaBrowser `Authorization` header (verified by +the suite's request-capture test); the legacy `X-Emby-Token` header is not co-sent. The +login path itself is available as `scripts/jellyfin login --username alice` (reads +`JELLYFIN_PASSWORD` interactively or via `--password`/`--password-stdin`), which prints the +captured `user_id`/`access_token` for exporting. ## Recipe 2 — Libraries → paged browse of one collection @@ -129,10 +131,56 @@ scripts/jellyfin next-up --limit 5 --json `RunTimeTicks` are 100-nanosecond ticks (divide by 600,000,000 for minutes). +## Bundled CLI `--dry-run` and exit-code contract + +The CLI's dry-run plans are pinned by its offline test suite (`scripts/test_jellyfin_cli.py`), +so jq keys match tested reality exactly. Every plan carries: + +```json +{ "dry_run": true, "path": "/Items/Latest", "params": { "userId": null, "limit": 10 } } +``` + +- `dry_run` (bool, always true), `path` (string) and `params` (object) appear on every + command plan; `login` instead emits `path: "/Users/AuthenticateByName"`, `server`, + `username`, `authorization_header`, and `pre_token_header: true` (its + `authorization_header` is the complete pre-token MediaBrowser header, no `Token=` + segment); `info` composes a `requests` array of `{path, params}` steps instead of a + single `path`/`params` pair. +- `params` mirrors the exact query the live call would send (`userId` is JSON `null` + when not supplied). + +Exit codes: `0` on success (including dry-run previews), `1` on CLI errors (missing +credentials, unreachable server, API 4xx/5xx, missing required `--user-id`), `2` on +argparse misuse such as `--movies --episodes` together or a missing required flag. + +## Bundled CLI `--dry-run` and exit-code contract + +The CLI's dry-run plans are pinned by its offline test suite (`scripts/test_jellyfin_cli.py`), +so jq keys match tested reality exactly. Every plan carries: + +```json +{ "dry_run": true, "path": "/Items/Latest", "params": { "userId": null, "limit": 10 } } +``` + +- `dry_run` (bool, always true), `path` (string) and `params` (object) appear on every + command plan; `login` instead emits `path: "/Users/AuthenticateByName"`, + `server`, `username`, `authorization_header`, and `pre_token_header: true` (its + `authorization_header` is the complete pre-token MediaBrowser header, no `Token=` + segment); `info` composes a `requests` array of `{path, params}` steps instead of a + single `path`/`params` pair. +- `params` mirrors the exact query the live call would send (`userId` is JSON `null` + when not supplied). + +Exit codes: `0` on success (including dry-run previews), `1` on CLI errors (missing +credentials, unreachable server, API 4xx/5xx, missing required `--user-id`), `2` on +argparse misuse such as `--movies --episodes` together or a missing required flag. + ## Cross-version-safe baseline (derive your own recipes from these rules) -1. Speak modern auth (`Authorization: MediaBrowser ...`); treat `X-Emby-Token` as a compat - fallback for old servers only. +1. Speak modern auth (`Authorization: MediaBrowser ...`) and put the token in exactly ONE + channel per request (its `Token=` parameter). Never co-send the legacy `X-Emby-Token` + header with it — precedence across channels is unspecified; if a legacy-only server + needs the old header, substitute it there, do not stack channels. 2. Send a complete pre-token header everywhere — zero cost, avoids the 400-on-login trap. 3. Always send explicit `userId` on `/Items*`, `/UserViews`, `/Shows/*`, `/Items/Latest`. 4. Resolve identity once: `USER_ID = AuthenticationResult.User.Id` (fallback `/Users/Me` diff --git a/jellyfin/scripts/jellyfin b/jellyfin/scripts/jellyfin index 66a1a54..909e611 100755 --- a/jellyfin/scripts/jellyfin +++ b/jellyfin/scripts/jellyfin @@ -103,20 +103,20 @@ class JellyfinClient: self.token = token or ENV_TOKEN self.device_id = device_id or ENV_DEVICE_ID or default_device_id() self.dry_run = dry_run - if not self.key and self.token: - # User access tokens are only meaningful together with their user id; - # user-scoped commands resolve that separately via --user-id. - pass + # A user access token is only meaningful together with its user id; the + # user-scoped commands below resolve that separately via --user-id + # rather than guessing a session user here. def _headers(self, with_token=True): - headers = { + # Exactly ONE token channel per request: the access token (or API key) + # rides the MediaBrowser Authorization header's Token= parameter. The + # legacy X-Emby-Token header is deliberately NOT also sent — combining + # channels is undefined behavior (see references/auth-and-sessions.md). + return { "Accept": "application/json", "Authorization": build_authorization_header( self.device_id, token=(self.key or self.token) if with_token else "") } - if with_token and self.key: - headers["X-Emby-Token"] = self.key - return headers def _get(self, path, params=None): url = f"{self.url}{path}" diff --git a/jellyfin/scripts/test_jellyfin_cli.py b/jellyfin/scripts/test_jellyfin_cli.py index fadab56..c14ae18 100644 --- a/jellyfin/scripts/test_jellyfin_cli.py +++ b/jellyfin/scripts/test_jellyfin_cli.py @@ -10,7 +10,6 @@ import tempfile import unittest from unittest.mock import Mock, patch - SCRIPT = pathlib.Path(__file__).resolve().parent / "jellyfin" LOADER = importlib.machinery.SourceFileLoader("jellyfin_cli", str(SCRIPT)) SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER) @@ -347,16 +346,15 @@ class LoginAuthSequenceTests(unittest.TestCase): def test_mocked_login_error_paths_do_not_crash(self): cli = jellyfin_cli cli.GLOBAL_FLAGS = {"json": False, "dry_run": False} - original_post = cli.requests.post for status, expected_fragment in ((400, "400"), (401, "401"), (403, "403")): with self.subTest(status=status): - cli.requests.post = Mock(return_value=FakeResponse(status, text="Error processing request.")) - stderr = io.StringIO() - with contextlib.redirect_stderr(stderr), self.assertRaises(SystemExit): + response = FakeResponse(status, text="Error processing request.") + with patch.object(cli.requests, "post", return_value=response), \ + contextlib.redirect_stderr(io.StringIO()) as stderr, \ + self.assertRaises(SystemExit): cli.cmd_login(cli.JellyfinClient(url="http://s:8096", device_id="d"), ["--username", "alice", "--password", "bad"]) self.assertIn(expected_fragment, stderr.getvalue()) - cli.requests.post = original_post cli.GLOBAL_FLAGS = {"json": False, "dry_run": False} def test_reads_require_a_credential_before_network(self): @@ -375,8 +373,35 @@ class LoginAuthSequenceTests(unittest.TestCase): headers = client._headers() self.assertIn('Token="k-1"', headers["Authorization"]) self.assertTrue(headers["Authorization"].startswith("MediaBrowser ")) + self.assertNotIn("X-Emby-Token", headers) # one token channel per request token_client = cli.JellyfinClient(token="t-1", device_id="dev-1") - self.assertIn('Token="t-1"', token_client._headers()["Authorization"]) + token_headers = token_client._headers() + self.assertIn('Token="t-1"', token_headers["Authorization"]) + self.assertNotIn("X-Emby-Token", token_headers) + + def test_captured_requests_carry_token_in_exactly_one_channel(self): + """VAL-JF-009: the access token (or API key) appears in exactly ONE + channel per request — the MediaBrowser Token= parameter — and the + legacy X-Emby-Token header is never sent alongside it.""" + cli = jellyfin_cli + captured = [] + response = FakeResponse(200, json_body={"Items": [], "TotalRecordCount": 0}) + for client in (cli.JellyfinClient(key="k-capture", device_id="dev-cap"), + cli.JellyfinClient(token="t-capture", device_id="dev-cap")): + with patch.object(cli.requests, "get", + side_effect=lambda url, **kw: captured.append(kw) or response): + client._get("/Items") + self.assertEqual(len(captured), 2) + for kwargs in captured: + self.assertIn("headers", kwargs) + headers = kwargs["headers"] + self.assertIn("Token=", headers["Authorization"]) + self.assertNotIn("X-Emby-Token", headers) + channel_count = sum( + 1 for value in headers.values() + if "Token=" in value or value.lower() == "x-emby-token" + ) + self.assertEqual(channel_count, 1) class TvNavigationCommandTests(unittest.TestCase): diff --git a/peertube/references/worked-recipes.md b/peertube/references/worked-recipes.md index 1e96987..d971a32 100644 --- a/peertube/references/worked-recipes.md +++ b/peertube/references/worked-recipes.md @@ -118,9 +118,11 @@ scripts/peertube comments --id "" --json | jq '{total, total_not_deleted: scripts/peertube --dry-run --json search --query "test" | jq '{path, params: (.params | keys)}' ``` -`--dry-run` output shape: `{"dry_run": true, "method": "GET", "url": "...", "path": -"/api/v1/...", "params": {...}}` (token requests additionally carry `"form"` keys without -values). Every handler's dry-run plan uses the same keys, so one jq pattern audits any +`--dry-run` output shape: `{"dry_run": true, "method": "GET", "path": "/api/v1/...", +"params": {...}}` — every plan carries exactly `dry_run`, `method`, `path`, and `params` +(test-pinned in `scripts/test_peertube.py`); composite commands emit a `requests` array of +those same keyed steps, and the `login` plan is a `POST /api/v1/users/token` whose +`form_fields` lists the field NAMES only (never values). One jq pattern audits any command. ## Raw curl equivalents (auth chain end-to-end) diff --git a/peertube/scripts/peertube b/peertube/scripts/peertube index c0f7d76..824b97a 100755 --- a/peertube/scripts/peertube +++ b/peertube/scripts/peertube @@ -17,7 +17,7 @@ import os import sys import time import warnings -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, Optional warnings.simplefilter("ignore") @@ -366,47 +366,6 @@ class PeerTubeClient: die(f"Non-JSON response from {path} (status {status}); " f"is {self.display_server()} a PeerTube instance?") - def get_about(self): - return self._get("/config/about") - - def get_stats(self): - return self._get("/server/stats") - - def list_videos(self, start=0, count=DEFAULT_COUNT, sort="-publishedAt"): - return self._get("/videos", {"start": start, "count": count, "sort": sort}) - - def get_video(self, video_id): - return self._get(f"/videos/{video_id}") - - def search_videos(self, query, start=0, count=DEFAULT_COUNT, search_target="local", sort=None): - params: Dict[str, Any] = {"search": query, "searchTarget": search_target, - "start": start, "count": count} - if sort: - params["sort"] = sort - return self._get("/search/videos", params) - - def list_comment_threads(self, video_id, start=0, count=DEFAULT_COUNT): - return self._get(f"/videos/{video_id}/comment-threads", - {"start": start, "count": count, "sort": "-createdAt"}) - - def list_channels(self, start=0, count=DEFAULT_COUNT): - return self._get("/video-channels", {"start": start, "count": count}) - - def get_channel(self, handle): - return self._get(f"/video-channels/{handle}") - - def list_channel_videos(self, handle, start=0, count=DEFAULT_COUNT): - return self._get(f"/video-channels/{handle}/videos", {"start": start, "count": count}) - - def get_account(self, name): - return self._get(f"/accounts/{name}") - - def my_profile(self): - return self._get("/users/me") - - def my_videos(self, start=0, count=DEFAULT_COUNT): - return self._get("/users/me/videos", {"start": start, "count": count}) - # ----- command handlers ------------------------------------------------ @@ -425,8 +384,8 @@ def cmd_server(client, args): {"method": "GET", "path": f"{API_BASE}/server/stats", "params": {}}, ]} return emit("[dry-run] GET /config/about + /server/stats", plan) - about = client.get_about() or {} - stats = client.get_stats() or {} + about = client._get("/config/about") or {} + stats = client._get("/server/stats") or {} instance = about.get("instance") or {} name = instance.get("name", "?") description = instance.get("shortDescription") or "" @@ -548,7 +507,7 @@ def cmd_channels(client, args): total = data.get("total", len(channels)) if not channels: return emit("No channels.", {"total": total, "start": parsed.offset, "count": 0, "channels": []}) - lines, _ = [], [] + lines = [] for channel in channels: display = channel.get("displayName") or channel.get("name") or "?" handle = f"{channel.get('name', '?')}@{channel.get('host', '?')}" @@ -604,7 +563,9 @@ def cmd_me(client, args): data = client._get("/users/me") or {} if isinstance(data, list): # docs render an array sample; live servers return one object data = data[0] if data else {} - role = data.get("role") or {} + role = data.get("role") if isinstance(data, dict) else {} + if not isinstance(role, dict): # some instances/versions may send a scalar role id + role = {"id": role, "label": str(role)} quota = data.get("videoQuota") quota_text = f"{quota} bytes" if quota is not None else "?" emit(f"@{data.get('username', '?')} Role: {role.get('label', '?')} Quota: {quota_text}", data) diff --git a/peertube/scripts/test_peertube.py b/peertube/scripts/test_peertube.py index 1bfa41d..88174c5 100644 --- a/peertube/scripts/test_peertube.py +++ b/peertube/scripts/test_peertube.py @@ -743,6 +743,35 @@ class HandlerOutputTests(ModuleStateTestCase): self.assertEqual(payload["role"]["label"], "User") client.clear_token() + def test_me_tolerates_scalar_role_without_attribute_error(self): + """VAL-PT-010: a non-dict `role` (scalar id from a version-drifted + server) must degrade to a readable line, never AttributeError.""" + client = pt.PeerTubeClient( + server="https://inst.example", config_dir=tempfile.mkdtemp(prefix="pt-ho-") + ) + profile = {"username": "bob", "role": 2, "videoQuota": None} + client.save_session(TOKEN_RESPONSE) + with patch.object(pt.requests, "get", return_value=FakeResponse(200, profile)): + out = io.StringIO() + err = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + pt.cmd_me(client, []) + self.assertNotIn("Traceback", err.getvalue()) + self.assertEqual(json.loads(out.getvalue())["role"], 2) + client.clear_token() + + def test_me_tolerates_missing_role(self): + client = pt.PeerTubeClient( + server="https://inst.example", config_dir=tempfile.mkdtemp(prefix="pt-ho-") + ) + client.save_session(TOKEN_RESPONSE) + with patch.object(pt.requests, "get", return_value=FakeResponse(200, {"username": "carol"})): + out = io.StringIO() + with contextlib.redirect_stdout(out): + pt.cmd_me(client, []) + self.assertEqual(json.loads(out.getvalue())["username"], "carol") + client.clear_token() + def test_comments_output_exposes_thread_counts(self): client = pt.PeerTubeClient( server="https://inst.example", config_dir=tempfile.mkdtemp(prefix="pt-ho-") From 05b99be6ec9c6c9f309b10535ae46d879e2aeebf Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sat, 29 Aug 2026 22:02:23 -0400 Subject: [PATCH 36/40] docs(transistor): thicken podcast hosting skill against current API research Full skill-builder rebuild of transistor per issue #407: - scripts/transistor (renamed from transistor-cli): 19 commands covering the verified API surface - user probe (GET /v1; the /v1/user route does not exist), shows/episodes with corrected pagination[page]/pagination[per], the dedicated episode publish endpoint (PATCH /v1/episodes/:id/publish with episode[status]=draft|scheduled|published), authorize-upload flow, the three real analytics routes with downloads[] array summing, subscriber management incl. batch, and webhooks. Write bodies are form-encoded bracket keys exactly as documented; dry-run plans carry method/path/params/body; publish guard refuses audio-less episodes. Fixed stale claims: /analytics/show -> /v1/analytics/..., totals -> downloads arrays, pagination[limit] -> pagination[per], user email -> name/time_zone, dropped invented episodes_count/subscribers_count and POST /v1/shows (show creation is dashboard-only). - scripts/test_transistor.py: 50 offline tests (pytest + unittest green, proxy-trap clean) covering help, argument errors, dry-run plans, canned JSON:API compound-document parsing (data/attributes/relationships/ included[]), write-path body shapes, publish guard, create->audio-> publish pipeline, and HTTP error signatures. - references/: auth+JSON:API envelope with jq patterns, endpoint catalog, publish lifecycle with documented request/response shapes, gotchas field guide + worked recipes; all cited to live-verified sources. - evals/evals.json: 8 schema-v1 cases incl. two should-not-trigger probes. - SKILL.md rewritten (308 lines), README refreshed, root README blurb and generated catalogs synced (marketplace.json + llms.txt descriptions). Publish-body shape reconciliation: the contract's data.id+data.type JSON:API PATCH premise was falsified by current official docs (verified 2026-08-29) and the flimzy/transistor Go SDK; implemented reality escalated in handoff (see library/transistor-api-facts.md). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/marketplace.json | 2 +- README.md | 2 +- llms.txt | 2 +- transistor/README.md | 60 +- transistor/SKILL.md | 316 +++++- transistor/evals/evals.json | 94 ++ transistor/references/auth-and-basics.md | 165 ++++ transistor/references/endpoint-catalog.md | 137 +++ .../references/episode-publish-lifecycle.md | 201 ++++ transistor/references/gotchas-and-recipes.md | 256 +++++ transistor/scripts/test_transistor.py | 912 ++++++++++++++++++ transistor/scripts/transistor | 757 +++++++++++++++ transistor/scripts/transistor-cli | 160 --- 13 files changed, 2832 insertions(+), 232 deletions(-) create mode 100644 transistor/evals/evals.json create mode 100644 transistor/references/auth-and-basics.md create mode 100644 transistor/references/endpoint-catalog.md create mode 100644 transistor/references/episode-publish-lifecycle.md create mode 100644 transistor/references/gotchas-and-recipes.md create mode 100644 transistor/scripts/test_transistor.py create mode 100755 transistor/scripts/transistor delete mode 100755 transistor/scripts/transistor-cli diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 91266f9..24e8eab 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1299,7 +1299,7 @@ "./transistor" ], "strict": false, - "description": "Manage Transistor.fm podcast hosting from the terminal: view shows, list episodes, check analytics, and get subscriber counts. Use when the user mentions Transistor, podcast hosting, podcast analytics, show management, or episode tracking." + "description": "Operate Transistor.fm podcast hosting from the terminal: verify API access, browse shows and episodes with JSON:API-aware output, run the episode publish lifecycle (create draft, attach audio, publish or schedule via the dedicated publish endpoint), pull download analytics, and manage private podcast subscribers and webhooks. Use when the user mentions Transistor, Transistor.fm, podcast hosting, episode publishing, private podcast subscribers, or podcast download analytics. Do not use this skill for other podcast hosts (Buzzsprout, Libsyn, Megaphone, Spotify for Creators), for editing or producing audio, or for feed/RSS parsing — the bundled CLI manages a Transistor account through its v1 API and cannot create new shows (dashboard-only)." }, { "name": "travel-guide", diff --git a/README.md b/README.md index 274b871..622c382 100644 --- a/README.md +++ b/README.md @@ -589,7 +589,7 @@ Trakt.tv media discovery from the terminal. Browse trending, anticipated, and po ### [transistor](transistor/SKILL.md) -Transistor.fm podcast hosting from the terminal. Manage shows and episodes, view subscriber analytics. API key from transistor.fm settings. +Operate Transistor.fm podcast hosting from the terminal: verify API access, browse shows and episodes with JSON:API-aware output, run the episode publish lifecycle (create draft, attach audio, publish or schedule via the dedicated publish endpoint), pull download analytics, and manage private podcast subscribers and webhooks. ### [travel-guide](travel-guide/SKILL.md) diff --git a/llms.txt b/llms.txt index 0f1a4bf..342cc38 100644 --- a/llms.txt +++ b/llms.txt @@ -145,7 +145,7 @@ - [tmdb](tmdb/SKILL.md): Query TMDb metadata for films and television, then enrich results with details, credits, providers, and external IDs. Do not use this skill for torrent search, streaming playback, or personal watch-history tracking. - [traefik](traefik/SKILL.md): Deploy, configure, and troubleshoot Traefik v3 reverse proxy — covers all providers, routing, TLS/ACME, middlewares, and production patterns with YAML examples. Load when setting up or debugging a Traefik instance. - [trakt](trakt/SKILL.md): Discover and compare Trakt.tv trending, popular, and anticipated movies and shows from the terminal. Do not use this skill for TMDb catalog metadata, credits, images, or provider lookups; use the tmdb skill for those tasks. -- [transistor](transistor/SKILL.md): Manage Transistor.fm podcast hosting from the terminal: view shows, list episodes, check analytics, and get subscriber counts. Use when the user mentions Transistor, podcast hosting, podcast analytics, show management, or episode tracking. +- [transistor](transistor/SKILL.md): Operate Transistor.fm podcast hosting from the terminal: verify API access, browse shows and episodes with JSON:API-aware output, run the episode publish lifecycle (create draft, attach audio, publish or schedule via the dedicated publish endpoint), pull download analytics, and manage private podcast subscribers and webhooks. Use when the user mentions Transistor, Transistor.fm, podcast hosting, episode publishing, private podcast subscribers, or podcast download analytics. Do not use this skill for other podcast hosts (Buzzsprout, Libsyn, Megaphone, Spotify for Creators), for editing or producing audio, or for feed/RSS parsing — the bundled CLI manages a Transistor account through its v1 API and cannot create new shows (dashboard-only). - [travel-guide](travel-guide/SKILL.md): Create personalized, source-grounded travel dossiers from a destination, dates, duration, travelers, and constraints. Ask only the questions that change the plan, use explicitly permitted personal context without exposing it, research current logistics, and produce a cited, visually coherent PDF or responsive companion web page. Use when someone wants an individualized itinerary, trip brief, travel field guide, or shareable travel website. Do not use for real-time booking, ticket purchasing, visa or legal advice, or generic destination summaries without a specific traveler and trip. - [vercel-eve](vercel-eve/SKILL.md): Build, develop, deploy, self-host, secure, and troubleshoot durable backend AI agents with Vercel Eve. Use when creating an Eve agent, adding tools, skills, subagents, channels, schedules, sandboxing, durable sessions, observability, or deploying Eve on Vercel or a Node host. Do not use for the separate Vercel AI SDK Agent APIs such as ToolLoopAgent or WorkflowAgent; use an AI SDK-specific skill for those. - [verification-methodology](verification-methodology/SKILL.md): Verify work against explicit criteria using direct, source-faithful evidence, reproducible checks, and clear verdicts. Use before declaring an artifact, implementation, or claim complete; do not use for exploratory research without pass/fail criteria. diff --git a/transistor/README.md b/transistor/README.md index 655bde6..7ead58e 100644 --- a/transistor/README.md +++ b/transistor/README.md @@ -1,37 +1,65 @@ # Transistor.fm — Podcast Hosting from the Terminal -Manage your Transistor.fm podcast shows, episodes, subscribers, and analytics — all from the terminal. +Manage your Transistor.fm podcast account over its official API: browse +shows and episodes, publish episodes, pull download analytics, and run +private-podcast subscriber lists — all from the terminal. ## Why Install This Skill -When your agent loads this skill, it can **manage your podcast hosting** without the web dashboard. That means: +When your agent loads this skill, it can **operate your Transistor.fm +podcast hosting** without the dashboard, including the part no other tool +gives an agent: the full episode publish lifecycle. -- **List shows** — all your podcasts with episode and subscriber counts -- **Browse episodes** — recent episodes with publish dates -- **Check analytics** — subscriber counts and trends -- **Filter by show** — drill into a specific podcast's episodes +- **Publish episodes end to end** — create a draft, attach audio (URL or + authorized local-file upload), then publish or schedule it through + Transistor's dedicated publish endpoint +- **Browse your catalog** — shows, episodes, drafts, season/number + metadata, with JSON:API compound documents unwrapped for jq +- **Track downloads** — per-day analytics windows for shows and episodes, + summed and ready for reports +- **Run private podcasts** — list, add (single or batch), and revoke + subscribers; register webhooks so you push instead of poll +- **Stay under the rate limit** — dry-run request plans and clear 429 + guidance (Transistor allows 10 requests per 10 seconds) ## What You Get -| Directory | Purpose | -|-----------|---------| -| `SKILL.md` | Complete command reference with examples | -| `scripts/transistor-cli` | CLI tool for Transistor.fm v1 REST API | +| Path | Purpose | +|------|---------| +| `SKILL.md` | Command reference, publish-lifecycle recipe, jq guidance, gotchas | +| `scripts/transistor` | Bundled Python CLI for the Transistor.fm v1 API (read + write commands) | +| `scripts/test_transistor.py` | Offline mocked test suite (canned JSON:API documents, zero network) | +| `references/auth-and-basics.md` | API-key auth, JSON:API envelope and jq patterns, pagination, errors | +| `references/endpoint-catalog.md` | Every endpoint's method, path, and parameters | +| `references/episode-publish-lifecycle.md` | Draft → audio → publish/schedule/unpublish, exact request shapes | +| `references/gotchas-and-recipes.md` | Symptom → cause → fix guide plus multi-step workflows | ## Quick Start ```bash -export TRANSISTOR_API_KEY="your-transistor-api-key" -transistor-cli shows -transistor-cli episodes +export TRANSISTOR_API_KEY="" # Dashboard -> Account -> API Access + +transistor user # verify the key +transistor shows # list your podcasts +transistor episodes --status draft # what is not out yet? + +# Publish pipeline: create (draft) -> attach audio -> publish +EP=$(transistor episode-create --show --title "Ep 12" \ + --audio-url "https://example.com/ep12.mp3" --json | jq -r '.id') +transistor episode-publish --id "$EP" ``` -API key from Settings → API Keys in the Transistor.fm dashboard. +`--help` and `--dry-run` work without an API key; preview any request with +`transistor --dry-run episode-publish --id 123`. ## Triggers -Load this for Transistor.fm, podcast hosting, podcast analytics, show management, or episode tracking. +Load this skill when the user mentions Transistor or Transistor.fm, podcast +hosting, publishing a podcast episode, scheduling or unpublishing episodes, +podcast download analytics, or private podcast subscribers. ## Requirements -Python 3.8+ with `requests` library. +Python 3.8+ with `requests`, plus a Transistor.fm API key (Account page → +API Access). The key carries your dashboard role per podcast; treat it like +a password. No other services or credentials are involved. diff --git a/transistor/SKILL.md b/transistor/SKILL.md index 7d6b168..a776b08 100644 --- a/transistor/SKILL.md +++ b/transistor/SKILL.md @@ -1,98 +1,308 @@ --- name: transistor -description: 'Manage Transistor.fm podcast hosting from the terminal: view shows, - list episodes, check analytics, and get subscriber counts. Use when the user mentions - Transistor, podcast hosting, podcast analytics, show management, or episode tracking.' +description: >- + Operate Transistor.fm podcast hosting from the terminal: verify API access, + browse shows and episodes with JSON:API-aware output, run the episode + publish lifecycle (create draft, attach audio, publish or schedule via the + dedicated publish endpoint), pull download analytics, and manage private + podcast subscribers and webhooks. Use when the user mentions Transistor, + Transistor.fm, podcast hosting, episode publishing, private podcast + subscribers, or podcast download analytics. Do not use this skill for other + podcast hosts (Buzzsprout, Libsyn, Megaphone, Spotify for Creators), for + editing or producing audio, or for feed/RSS parsing — the bundled CLI + manages a Transistor account through its v1 API and cannot create new + shows (dashboard-only). license: MIT -compatibility: Requires TRANSISTOR_API_KEY env var (from Settings → API Keys in the - Transistor.fm dashboard), Python 3.8+, and the `requests` library. Uses the Transistor.fm - v1 REST API with JSON:API format responses. +compatibility: Requires TRANSISTOR_API_KEY env var (Account page -> API Access + at https://dashboard.transistor.fm/account), Python 3.8+, and `requests`. + Read commands need a working key; `--help` and `--dry-run` never do. metadata: - tags: transistor, podcast, podcast-hosting, analytics, api-client - sources: https://transistor.fm/, https://developers.transistor.fm/ + tags: transistor, podcast, podcast-hosting, episodes, analytics, api-client + sources: https://developers.transistor.fm/, https://support.transistor.fm/ --- -# transistor-cli — Transistor.fm Podcast Hosting +# transistor — Transistor.fm podcast hosting from the terminal -Manage your Transistor.fm podcast shows, episodes, subscribers, and analytics from the terminal. Uses the Transistor.fm v1 REST API. +Drive a Transistor.fm account over its v1 JSON:API: shows, episodes, the +draft→publish lifecycle, per-day download analytics, private-podcast +subscribers, and webhooks. Responses are JSON:API documents; the bundled CLI +unwraps them (`--json`) while preserving the raw shapes agents need for jq. +Write commands are guarded: episode creation is always a draft, and +publishing goes through its own dedicated endpoint. ## Setup -1. Get your API key from Transistor.fm: **Settings → API Keys** (bottom of the page) -2. Set the environment variable: +1. Find your API key on the Transistor dashboard **Account page → API + Access** (https://dashboard.transistor.fm/account) and export it: ```bash -export TRANSISTOR_API_KEY="your-transistor-api-key" +export TRANSISTOR_API_KEY="" ``` -`--help` and `--dry-run` work without credentials. +2. Verify the key (GET /v1 — the authorization probe; there is no + /v1/user route): + +```bash +transistor user # name and time zone +transistor user --json | jq '{id, name, time_zone}' +``` + +A key carries the dashboard role of its user (owner / admin / team member) +per podcast. `--help` and `--dry-run` work without credentials. Requests are +rate-limited to 10 per 10 seconds; the CLI dies with a clear 429 message +instead of hammering. ## Essential Commands -### user — Current user info +### user — authorization probe ```bash -transistor-cli user # email and timezone -transistor-cli user --json # machine-readable +transistor user # who does this key belong to? +transistor user --json ``` -### shows — List all shows +### shows / show — browse podcasts ```bash -transistor-cli shows # all shows with episode/subscriber counts -transistor-cli shows --json # machine-readable with IDs +transistor shows # newest-updated first +transistor shows --private --json # private podcasts only +transistor show --id # full attributes incl. feed_url +transistor shows --page 1 --per 20 --json ``` -Shows title, episode count, subscriber count, and ID. +Show ids and slugs are interchangeable on most show-scoped routes. Show +resources carry no counts fields — count via `episodes --show ... --json`, +then `meta.totalCount`. -### episodes — List episodes +### episodes / episode — browse episodes ```bash -transistor-cli episodes # last 20 episodes across all shows -transistor-cli episodes --show 12345 # filter by show ID -transistor-cli episodes --limit 50 # more results -transistor-cli episodes --show 12345 --json # machine-readable +transistor episodes # newest first, all shows +transistor episodes --show --status draft # drafts for one show +transistor episodes --show --per 50 --page 1 --json +transistor episode --id --include show # compound doc + parent show ``` -Get show IDs from `transistor-cli shows --json`. Shows season/episode numbers, title, status, duration, and publish date. +`--include show` adds `included[]` (the JSON:API compound document); every +episode item in `--json` output already carries `show_id` resolved from +relationships. `--limit` works as an alias for `--per` for old scripts. -### analytics — Download and play analytics +### episode-create / episode-update — drafts and metadata ```bash -transistor-cli analytics # totals across all shows -transistor-cli analytics --show 12345 # filter by show ID -transistor-cli analytics --json # machine-readable with full totals +transistor episode-create --show --title "Ep 12: Roasting" \ + --season 2 --number 4 --audio-url "https://uploads.example.com/ep12.mp3" +transistor episode-update --id --title "New title" +transistor episode-update --id --audio-url "" # attach audio ``` -Shows total downloads and plays (when available). +`episode-create` ALWAYS produces a draft (`status: "draft"`, +`published_at: null`) — it never publishes. `episode-update` changes +metadata or attaches audio and never touches publishing state. -## Global Flags - -All flags work in any position: +### episode-publish — the lifecycle switch ```bash -transistor-cli --json shows # flag before subcommand -transistor-cli shows --json # flag after subcommand -transistor-cli --dry-run episodes # preview (no API call) -transistor-cli --force episodes --limit 100 # override safety checks -transistor-cli --quiet shows # suppress non-essential output -transistor-cli --verbose episodes # detailed logging +transistor episode-publish --id # publish now +transistor episode-publish --id --status scheduled \ + --published-at "2026-09-03 09:00:00" # schedule +transistor episode-publish --id --status draft # unpublish ``` -Extra flag vs other CLIs: `--force` overrides internal safety checks (e.g. large limits). +Hits `PATCH /v1/episodes//publish` with +`episode[status]=draft|scheduled|published` — the documented dedicated +endpoint. The CLI refuses to publish an episode whose `media_url` is still +empty (an unplayable item would hit every subscriber's feed); pass +`--force` to override. + +### authorize-upload — local audio (max 5GB) + +```bash +transistor authorize-upload --filename ep12.mp3 # plan only +transistor authorize-upload --filename ep12.mp3 --file ./ep12.mp3 +``` + +Returns (and, with `--file`, performs) the signed PUT; the printed +`audio_url` is what you attach with `episode-create`/`episode-update`. The +signed URL expires (~600 s in the docs' example). + +### analytics / episode-analytics — downloads per day + +```bash +transistor analytics --show # last 14 days +transistor analytics --show \ + --start-date 01-08-2026 --end-date 28-08-2026 --json +transistor episode-analytics --id --json +``` + +Dates are dd-mm-yyyy and must come in pairs. Analytics attributes are +per-day `downloads[]` arrays, not totals; the CLI sums them into +`downloads_total` and keeps the raw array. + +### subscribers — private podcast audience + +```bash +transistor subscribers --show --json +transistor subscriber-create --show --email "listener@example.com" +transistor subscriber-batch --show --email "a@example.com" --email "b@example.com" +transistor subscriber-delete --show --email "a@example.com" # or --id +``` + +### webhooks — push instead of poll + +```bash +transistor webhooks --show +transistor webhook-create --show --event episode_published \ + --url "https://example.com/hooks/transistor" +transistor webhook-delete --id +``` + +Events: `episode_created`, `episode_published`, `subscriber_created`, +`subscriber_deleted`. Cap: 50 per account. With a 10 req / 10 s limit, +webhooks beat polling for freshness. + +## Global flags + +```bash +transistor --json shows # flags work in any position +transistor --dry-run episodes --show # request plan, zero network +transistor --force episode-publish --id # skip the audio guard +transistor --quiet shows # suppress non-essential output +transistor --verbose episodes # detailed stderr logging +``` + +`--dry-run` emits `{"dry_run": true, "method", "path", "params"}` (write +commands add the exact `body` that would be sent — bracket keys and all), +so you can verify a plan before touching the API. `--help` and `--dry-run` +never require credentials. + +## Pipeline recipes + +### Create, attach audio, publish (the core workflow) + +```bash +export TRANSISTOR_API_KEY="" +SHOW=$(transistor shows --json | jq -r '.shows[0].id') # string id +AUDIO=$(transistor authorize-upload --filename ep12.mp3 --file ./ep12.mp3 --json | jq -r '.audio_url') +EP=$(transistor episode-create --show "$SHOW" --title "Ep 12" \ + --audio-url "$AUDIO" --json | jq -r '.id') # draft id +transistor episode-publish --id "$EP" # dedicated endpoint +transistor episode --id "$EP" --json | jq '{status, media_url, published_at}' +``` + +Each stage's output feeds the next: `shows` → string `id`, +`authorize-upload` → string `audio_url`, `episode-create` → string draft +`id`, `episode-publish` → final `status`. Stage 2 is skippable when the +audio already has a public URL (pass it straight to `episode-create`). + +### Draft triage: what is not out yet? + +```bash +transistor episodes --show --status draft --json \ + | jq -r '.episodes[] | [.id, .title, (if .media_url == "" then "no-audio" else "ready" end)] | @tsv' +# publish the ready ones (rate limit: 10 req / 10 s — add sleep 1 between calls) +``` + +### Weekly downloads report + +```bash +transistor shows --json | jq -r '.shows[].id' | while read -r S; do + transistor analytics --show "$S" --json \ + | jq -r --arg id "$S" '[$id, (.downloads_total|tostring)] | @tsv' + sleep 1 +done +``` + +## JSON and jq + +`--json` keys are stable snake_case wrappers around the JSON:API document: +`shows`/`episodes`/`subscribers`/`webhooks` (arrays with `meta` attached), +flat objects for single resources, `dry_run`/`method`/`path`/`params`/`body` +for plans. Attributes keep Transistor's own names — `status`, `season`, +`number`, `duration` (seconds), `media_url`, `share_url`, `published_at`, +`feed_url` — so jq selectors transfer directly to raw `curl` against +api.transistor.fm. Collection pagination surfaces as +`meta.currentPage`/`meta.totalPages`/`meta.totalCount`. Example: +`transistor episodes --show --json | jq -r '.episodes[] | +[.id, .title, .status] | @tsv'`. For compound documents the CLI resolves +relationships (`show_id`) and prints `included` show summaries in human +mode; with raw curl, match `included[]` by `type` and +`relationships.show.data.id`. ## Known Gotchas -- **API key from Settings → API Keys** — Not from the user profile or account page. Navigate to Settings → API Keys at the bottom of the Transistor.fm dashboard. -- **JSON:API format** — Transistor uses the JSON:API spec. All responses are nested under a `data` key, and attributes are under `data[].attributes`. The CLI unwraps these for display, but raw JSON output shows the full JSON:API structure. -- **Show IDs are required for filtered queries** — Use `transistor-cli shows --json` to get show IDs first, then pass them to `--show` for episodes and analytics. -- **Pagination uses cursor-based pagination** — The `--limit` flag controls the page size. Default is 20 for episodes. The API returns `meta` with pagination info. -- **Analytics are totals only** — The analytics endpoint returns aggregate totals (downloads, plays). Per-episode analytics are not available via this CLI. -- **Transistor API is append-only via this CLI** — The CLI implements GET endpoints for reading. Creating/updating shows or episodes is not covered here. -- **Rate limits** — Transistor.fm has rate limits. The CLI does not auto-retry on 429 responses. +- **Publishing is a separate endpoint, never a side effect** — POST + /episodes and PATCH /episodes/:id cannot change `status`. If an episode + stays draft, the missing step is `PATCH /v1/episodes//publish` with + `episode[status]=published`. (The pre-thickening CLI had no publish path + at all.) +- **The user probe is `GET /v1`** — `/v1/user` and `/v1/authorization` are + 404s, and the user resource has no email attribute (name and time_zone + only). +- **Pagination is `pagination[page]` + `pagination[per]`** (defaults 0 and + 10; docs' examples request page 1). `pagination[limit]` and + `page[number]` are silently ignored — loops using them re-read page 1 + forever. Loop while `meta.currentPage < meta.totalPages`. +- **Show resources carry no counts** — derive episode/subscriber counts + from filtered listings' `meta.totalCount`. +- **Analytics are per-day arrays, not totals** — sum `attributes.downloads` + (the CLI provides `downloads_total`); date bounds are dd-mm-yyyy and + come in pairs; do not parse the row date format (docs' examples are + inconsistent between sections). +- **Show creation is dashboard-only** — there is no POST /v1/shows; + `show-update` is the only show write. +- **Rate limit 10 req / 10 s** — a 429 blocks access for 10 seconds. No + retry headers; back off, batch subscriber imports, cache responses, and + use webhooks for freshness. Transistor explicitly forbids using the API + as a website back end (parse the RSS feed for that). +- **`episode[published_at]` uses the show's time zone** (a show attribute), + not UTC; scheduling and backdating both ride the publish endpoint. +- **Audio processing is asynchronous** — watch `audio_processing` / + `processing_failure` after attaching audio; publishing an unprocessed or + failed file pushes silence to subscribers. +- **Signed upload URLs expire** (~600 s in the docs' example): authorize, + PUT with the returned `content_type`, attach promptly. +- **Error bodies are not formally specified** — the CLI handles JSON:API + `errors[]` arrays and bare `{"message": ...}` objects, flattening either + to one stderr line; 401 (bad key), 403 (role), 404 (bad id/route), 429 + (rate limit) have distinct hints. -## References +## When to use -- [scripts/transistor-cli](scripts/transistor-cli) — The CLI binary. Built following the cli-builder patterns: `--json`, `--dry-run`, `--force`, `--quiet`, `--verbose`, dual-output via `emit()`, lazy auth. -- [Transistor API Docs](https://developers.transistor.fm/) — Official API reference. -- [Transistor.fm Dashboard](https://dashboard.transistor.fm/) — Settings → API Keys for your API key. +Use this skill for anything that reads or drives a Transistor.fm account +through its API: verifying API access, browsing shows/episodes (including +drafts and compound documents), running the episode lifecycle +(create → attach audio → publish/schedule/unpublish), pulling download +analytics windows, importing or revoking private-podcast subscribers, and +registering webhooks. + +## When not to use + +Do not use this skill for other podcast hosts (Buzzsprout, Libsyn, +Megaphone, Spotify for Creators — use their own APIs/tooling); for audio +production or editing (ffmpeg and DAW territory); for generic RSS feed +parsing or website rendering (parse the feed XML directly — Transistor says +the API is not a back-end data source); for creating new shows (the API +cannot — the dashboard does); or for platform-level distribution questions +(Apple/Spotify submission is a dashboard and RSS concern). + +## Reference Files + +| File | Use it for | +| ---- | ---------- | +| [references/auth-and-basics.md](references/auth-and-basics.md) | x-api-key auth, key location and role scoping, the JSON:API envelope (data/attributes/relationships/included[]) with jq patterns, pagination params, error surfaces | +| [references/endpoint-catalog.md](references/endpoint-catalog.md) | Every route's method, path, and parameters (shows, episodes, publish, uploads, analytics, subscribers, webhooks) plus routes that do not exist | +| [references/episode-publish-lifecycle.md](references/episode-publish-lifecycle.md) | The draft/scheduled/published state machine, the exact publish request/response shapes, create→audio→publish recipes in CLI and curl, authorize-upload detour | +| [references/gotchas-and-recipes.md](references/gotchas-and-recipes.md) | Symptom → cause → fix field guide (404 user route, silent pagination, 429 storms...) and multi-step workflows (bulk scheduling, analytics reports, subscriber import, webhooks) | + +## Available Scripts and Prerequisites + +- `scripts/transistor` — the bundled Python CLI (`--json`, `--dry-run`, + `--force`, `--quiet`, `--verbose`, `--help` everywhere). Imports only the + standard library and `requests`; sends write bodies exactly as documented + (bracket-key form fields). +- `scripts/test_transistor.py` — offline test suite (pytest + unittest + compatible); all HTTP mocked with canned JSON:API documents, zero network + egress, no live-call cases (Transistor is a keyed API). +- Requires Python 3.8+, `requests`, and `TRANSISTOR_API_KEY` for live + commands (Account page → API Access). No service is started by this skill. diff --git a/transistor/evals/evals.json b/transistor/evals/evals.json new file mode 100644 index 0000000..0e3abfe --- /dev/null +++ b/transistor/evals/evals.json @@ -0,0 +1,94 @@ +{ + "schema_version": 1, + "skill_name": "transistor", + "evals": [ + { + "id": "browse-shows-and-episodes-json", + "prompt": "List my Transistor shows and then the latest episodes of the first one, as JSON I can pipe to jq.", + "expected_output": "Export TRANSISTOR_API_KEY (Dashboard -> Account -> API Access), run transistor shows --json to get show ids, then transistor episodes --show --json. Collection output is {shows|episodes: [...], meta: {currentPage, totalPages, totalCount}}; page with --page/--per (pagination[page]/pagination[per] on the wire, API default 10 per page) and loop while meta.currentPage < meta.totalPages.", + "assertions": [ + "exports TRANSISTOR_API_KEY and runs transistor shows --json first", + "extracts the show id from the shows output before filtering episodes", + "runs transistor episodes with --show and --json", + "pages with pagination[page]/pagination[per] and meta.currentPage/totalPages, never pagination[limit] or page[number]" + ] + }, + { + "id": "episode-create-then-publish-pipeline", + "prompt": "I have a finished MP3 at https://cdn.example.com/ep12.mp3. Publish it as episode 12 of my Transistor show, season 2, with the title 'Roasting Coffee'.", + "expected_output": "Create the draft: transistor episode-create --show --title 'Roasting Coffee' --season 2 --number 12 --audio-url https://cdn.example.com/ep12.mp3 --json and take .id (creation always yields status draft, published_at null). Then publish on the dedicated endpoint: transistor episode-publish --id , which sends PATCH /v1/episodes//publish with episode[status]=published. Confirm with transistor episode --id --json reading .status and .media_url. Updating episode metadata never publishes; only the /publish endpoint changes status.", + "assertions": [ + "creates the episode as a draft with transistor episode-create including --show, --title, and --audio-url", + "publishes via transistor episode-publish on the dedicated PATCH /v1/episodes/:id/publish endpoint with episode[status]=published", + "does not claim episode-create or episode-update can publish the episode", + "reads the publish result from data.attributes.status / the JSON output .status" + ] + }, + { + "id": "publish-rejected-because-stays-draft", + "prompt": "I keep updating my Transistor episode with PATCH /v1/episodes/:id but it stays in draft and never shows up in the RSS feed. What am I doing wrong?", + "expected_output": "Nothing is broken: metadata updates cannot change publishing state. POST /episodes and PATCH /episodes/:id never publish (the docs say publishing is a separate endpoint). Send PATCH /v1/episodes//publish with episode[status]=published (draft/scheduled/published are the only status values; episode[published_at] in the show's time zone schedules or backdates). The bundled CLI path is transistor episode-publish --id .", + "assertions": [ + "explains that PATCH /episodes/:id can never change publishing state", + "uses the dedicated /publish endpoint with episode[status]=published", + "mentions scheduled/draft states are set on the same publish endpoint", + "does not suggest re-sending episode[audio_url] or metadata fields as the fix" + ] + }, + { + "id": "authorize-upload-attach-audio-workflow", + "prompt": "My episode audio is a local file ep12.mp3 on disk and I don't have a public URL. Walk me through getting it into Transistor and published.", + "expected_output": "Authorize an upload: transistor authorize-upload --filename ep12.mp3 --file ./ep12.mp3 (the CLI GETs /v1/episodes/authorize_upload?filename=..., PUTs the bytes to the signed upload_url with the returned content_type; the URL expires ~600s, max 5GB). Take .audio_url from the output, attach it: transistor episode-update --id --audio-url (or pass it at episode-create), then publish with transistor episode-publish --id . Accepted formats include .mp3, .m4a, .wav.", + "assertions": [ + "starts with transistor authorize-upload and uses the returned audio_url", + "attaches audio via episode-create or episode-update with --audio-url", + "publishes only after attaching audio, via the publish endpoint", + "does not try to PUT the file to api.transistor.fm directly" + ] + }, + { + "id": "rate-limit-429-webhook-guidance", + "prompt": "My script that checks Transistor for new published episodes every few seconds just started failing with 429 errors. Fix it.", + "expected_output": "Transistor rate-limits the API to 10 requests per 10 seconds; a 429 blocks access for 10 seconds and no retry headers are documented. Polling every few seconds will keep tripping it. Fix: slow the loop (sleep between calls), cache responses, and better, register a webhook so Transistor pushes events: transistor webhook-create --show --event episode_published --url https://example.com/hooks (events: episode_created, episode_published, subscriber_created, subscriber_deleted; max 50 per account). Transistor also says the API is not meant as a website back end - parse the RSS feed for display use.", + "assertions": [ + "states the 10 requests per 10 seconds limit and the 10 second 429 block", + "replaces tight polling with a webhook (episode_published) or cached/less frequent calls", + "does not invent retry-after headers or exponential backoff promises from the docs", + "mentions the 50-webhook per account cap or the event names" + ] + }, + { + "id": "private-podcast-subscriber-batch-import", + "prompt": "Import a mailing list of 40 people into my private Transistor podcast without spamming each one manually.", + "expected_output": "Use the batch endpoint: transistor subscriber-batch --show --email --email ... (POST /v1/subscribers/batch with show_id and emails[]), optionally --skip-welcome-email. Verify with transistor subscribers --show --json reading meta.totalCount. Revoke access later with transistor subscriber-delete --show --email or --id. Each subscriber gets a personal feed_url/subscribe_url - never share one person's feed URL.", + "assertions": [ + "uses subscriber-batch (POST /v1/subscribers/batch) instead of 40 single calls", + "lists subscribers and reads meta.totalCount to verify the import", + "revokes with subscriber-delete by email or id when needed", + "does not share or reuse one subscriber's personal feed_url" + ] + }, + { + "id": "create-transistor-show-not-api", + "prompt": "Use the Transistor API to create a brand new podcast called 'Night Shift' on my account.", + "expected_output": "This must not trigger the transistor skill's CLI for creation: show creation is not available via the Transistor API at all (no POST /v1/shows exists; Transistor's support docs say new shows need to be created in the web app). Create the show in the dashboard first; afterwards the skill can manage it (show-update for metadata, episodes, subscribers, analytics).", + "assertions": [ + "must not attempt to create a show through the API", + "states that show creation is dashboard-only because POST /v1/shows does not exist", + "directs the user to create the show in the Transistor dashboard first", + "still offers post-creation management (episode lifecycle, metadata updates) once the show exists" + ] + }, + { + "id": "audio-editing-not-transistor", + "prompt": "Cut the first 30 seconds of silence off my podcast MP3 and normalize the loudness.", + "expected_output": "This must not trigger the transistor skill: audio editing/transcoding is outside a hosting-account API skill (ffmpeg or a DAW does the edit), and Transistor's API manages episodes, subscribers, and analytics - not audio files. After the edited file is hosted somewhere reachable (or via authorize-upload), the Transistor skill can attach and publish it.", + "assertions": [ + "must not trigger transistor for audio editing", + "routes the edit to ffmpeg or a DAW", + "does not invent API endpoints for editing or processing audio", + "may mention re-attaching the edited file afterward as the follow-up step" + ] + } + ] +} diff --git a/transistor/references/auth-and-basics.md b/transistor/references/auth-and-basics.md new file mode 100644 index 0000000..4daa842 --- /dev/null +++ b/transistor/references/auth-and-basics.md @@ -0,0 +1,165 @@ +# Transistor API: Authentication, JSON:API Envelope, and Request Basics + +Everything in this file is from the official API reference +(developers.transistor.fm) and Transistor's own support pages, verified +live at authoring time. Transistor.fm's public API is v1 and speaks +JSON:API on responses; there is exactly one authentication mode. + +## Authentication + +- Every request carries an HTTP header `x-api-key` whose value is the API + key. There is no OAuth, no bearer token, and no signing on the REST API. +- Keys are created, viewed, and reset in the Transistor Dashboard's Account + page, in the section marked **API Access** + (https://dashboard.transistor.fm/account). Transistor's support article + "Does Transistor have an API?" (updated 2026-07) names exactly this + location; the bundled CLI prints it on every auth error. +- A key grants whatever the associated dashboard user can see: access to + podcasts and episodes follows the user's podcast role — **owner**, + **admin**, or **regular team member**. There are no narrower per-key + scopes: a leaked key is as powerful as its user. Treat it like a + password; reset it from the same Account page if it leaks. +- The authorization probe is `GET /v1` — it returns the authenticated + `user` resource and nothing else. There is **no `/v1/user` and no + `/v1/authorization` route**; older tutorials that call `/v1/user` get a + 404. The `user` resource has `name`, `time_zone`, `image_url`, and + timestamps — **it has no email attribute**. + +```sh +curl https://api.transistor.fm/v1 -H "x-api-key: " +``` + +## Rate limits + +- **10 requests per 10 seconds.** Exceeding the limit returns HTTP `429` + and access is blocked for 10 seconds; after that requests flow again. +- No rate-limit headers (`Retry-After` etc.) are documented — don't parse + for them; just back off on 429. +- Transistor explicitly states the API is not meant to be the main data + source for a website or app back end; pull data once, cache it, and parse + the public RSS feed XML when you would otherwise hammer the API. For + push-style updates, webhooks (see the endpoint catalog) exist for + `episode_created`, `episode_published`, `subscriber_created`, and + `subscriber_deleted`. + +## The JSON:API envelope + +Responses are JSON:API documents. Learn four keys and every endpoint is +readable: + +| Key | Shape | Meaning | +| --- | --- | --- | +| `data` | object (single resource) or array (collections) | The primary resource(s) of the response | +| `attributes` | object inside a resource | The resource's fields (title, status, media_url, ...) | +| `relationships` | object of `{"": {"data": {"id", "type"}}}` | Links to related resources by id and type | +| `included` | array (only when requested with `include[]`) | The full related resources — a "compound document" | + +- Resource `type` values: `user`, `show`, `episode`, `subscriber`, + `show_analytics`, `episodes_analytics`, `episode_analytics`, + `audio_upload`, `webhook`. +- Single-resource responses wrap one object: `{"data": {"id": ..., + "type": "episode", "attributes": {...}, "relationships": {...}}}`. +- Collection responses wrap an array plus pagination under `meta`: + `{"data": [...], "meta": {"currentPage", "totalPages", "totalCount"}}`. +- Ids are **strings** even when numeric ("3056098"); analytics ids may be + slugs ("the-caffeine-show"). Keep ids as strings end to end. +- `included[]` appears only when you ask for it. `GET + /v1/episodes/3056098?include[]=show` returns the episode plus the parent + show in `included`, matched via `data.relationships.show.data.id`. + +### jq patterns for the envelope + +```sh +# Single resource: unwrap data.attributes +curl -s https://api.transistor.fm/v1/episodes/ -H "x-api-key: " \ + | jq '.data.attributes | {title, status, published_at}' + +# Collection: titles plus ids, one per line +curl -s 'https://api.transistor.fm/v1/episodes?show_id=' -H "x-api-key: " \ + | jq -r '.data[] | [.id, .attributes.title, .attributes.status] | @tsv' + +# Compound document: pull the parent show's title out of included[] +curl -s 'https://api.transistor.fm/v1/episodes/?include[]=show' -H "x-api-key: " \ + | jq --arg id "$(curl -s ... | jq -r '.data.relationships.show.data.id')" \ + '.included[] | select(.type == "show" and .id == $id) | .attributes.title' + +# Simpler: match included[] by type when only one show was included +... | jq '.included[] | select(.type == "show") | .attributes.title' + +# Pagination loop values live in meta +... | jq '{page: .meta.currentPage, last: .meta.totalPages, total: .meta.totalCount}' +``` + +The bundled CLI does this unwrapping for `--json` output: collections come +back as `{"episodes": [...], "meta": {...}}` with each item flattened to the +fields agents actually need (including `show_id` from relationships), and +single resources as one flat object. + +## Pagination + +- Page-based, two parameters: `pagination[page]` (documented default `0`; + the doc examples explicitly request page `1`) and `pagination[per]` + (default `10`). +- Every collection returns `meta.currentPage`, `meta.totalPages`, and + `meta.totalCount`. Loop while `currentPage < totalPages`, incrementing + the page — do not assume the first page is `0` or `1`, read `meta`. +- There is no cursor, no `page[number]`/`page[size]` JSON:API-style + spelling, and no `pagination[limit]` — unknown params are silently + ignored, which is exactly how scripts that "paginate" with + `pagination[limit]` re-read the first page forever. + +## Sparse fieldsets and compound documents + +Any endpoint accepts JSON:API's standard extras: + +- Sparse fieldsets: `fields[episode][]=title&fields[episode][]=media_url` + returns only those attributes (smaller payloads, faster loops). +- Include related resources: `include[]=show` on an episode, `include[]=show` + on analytics, `include[]=episode` on episode analytics. Combine both: + `include[]=show&fields[show][]=title&fields[show][]=feed_url`. + +## Request bodies: form-encoded bracket keys (documented), JSON accepted + +- The reference intro says endpoints accept **JSON or form-encoded** request + bodies. Every documented mutation example uses form-encoded bracket keys: + `episode[show_id]=...`, `episode[title]=...`, `show[title]=...`, + `subscriber[email]=...`, `episode[status]=published`. +- The docs publish no JSON-body equivalent examples, so the bracket-key + shapes above are the contract to copy. The bundled CLI sends form-encoded + bodies byte-compatible with the documented curl examples. +- Required-vs-optional matters: `episode[show_id]` is the only required + field on episode creation; `episode[status]` is required on the publish + endpoint; `show_id` is required on subscribers/webhooks listings. + +## Error surfaces + +Responses use standard HTTP codes. The reference does not document a formal +error schema, so program defensively: + +- `401` — key missing/invalid → check `x-api-key` and the Account page. +- `403` — key valid, role insufficient (owner/admin needed for some + operations on a shared podcast). +- `404` — id/slug not found (and remember: `/v1/user` is not a route). +- `422` — validation errors (e.g. bad `episode[status]` value). +- `429` — rate limit (10 requests / 10 s window). + +Error bodies seen in practice are JSON; the bundled CLI accepts either a +JSON:API-style `errors[]` array or a bare `{"message": ...}` object and +flattens whichever it gets into one stderr line. + +## Sources + +- https://developers.transistor.fm/ (introduction, JSON:API conformance, + authentication, rate limits, sparse fieldsets/include[] sections; all + endpoint examples) — fetched live 2026-08-29 (HTTP 200) +- https://developers.transistor.fm/#authentication (header name, Account + Area key management, owner/admin/team-member access levels) +- https://developers.transistor.fm/#ratelimits (10 requests / 10 s, 429 + + 10 s block, caching/RSS guidance) +- https://developers.transistor.fm/#get-v1 (GET /v1 user resource example) +- https://developers.transistor.fm/#resources (type list; User resource + fields — no email) +- https://support.transistor.fm/en/article/does-transistor-have-an-api-1b24sjo/ + (API key location: Account page → API Access) — fetched live 2026-08-29 +- https://support.transistor.fm/en/article/what-automations-are-possible-with-transistor-bi27am/ + (supported automations, show-creation limitation) — fetched live 2026-08-29 diff --git a/transistor/references/endpoint-catalog.md b/transistor/references/endpoint-catalog.md new file mode 100644 index 0000000..7d4311e --- /dev/null +++ b/transistor/references/endpoint-catalog.md @@ -0,0 +1,137 @@ +# Transistor API Endpoint Catalog + +Method-by-method reference for Transistor API v1. Every row matches the +official reference at developers.transistor.fm (fetched live 2026-08-29). +Envelope conventions (`data`/`attributes`/`relationships`/`included[]`), +pagination (`pagination[page]`, `pagination[per]`, `meta.currentPage`, +`meta.totalPages`, `meta.totalCount`), and sparse-fieldset/include[] params +apply everywhere — see [auth-and-basics.md](auth-and-basics.md). + +## Root + +| Method | Path | Purpose / params | +| --- | --- | --- | +| GET | `/v1` | Authenticated user probe. No params. Returns one `user` resource (`name`, `time_zone`, `image_url`, timestamps; **no email**). Use as the "does my key work" check. | + +## Shows + +| Method | Path | Purpose / params | +| --- | --- | --- | +| GET | `/v1/shows` | List shows, descending by updated date. Params: `private` (boolean), `query` (title search), `pagination[page]` (default 0), `pagination[per]` (default 10). | +| GET | `/v1/shows/:id` | One show. `:id` accepts the show ID **or slug**. | +| PATCH | `/v1/shows/:id` | Update any of: `show[author]`, `show[category]`, `show[copyright]`, `show[description]`, `show[explicit]`, `show[image_url]`, `show[keywords]`, `show[language]`, `show[owner_email]`, `show[secondary_category]`, `show[show_type]` (`episodic`/`serial`), `show[title]`, `show[time_zone]`, `show[website]`. Category/language/time-zone values are large closed enums — fetch the dashboard values or reuse what GET returns. | + +- Show attributes include `title`, `slug`, `description`, `author`, + `private`, `show_type`, `feed_url`, `time_zone`, `category`, + `secondary_category`, `language`, `owner_email`, `website`, `explicit`, + `keywords`, plus per-directory URLs (`apple_podcasts`, `spotify`, + `overcast`, ...). +- **There are no `episodes_count` or `subscribers_count` attributes** — + count episodes by listing them (`show_id` filter + `meta.totalCount`). +- **No POST /v1/shows exists**: show creation is not available via the API + (Transistor support, updated 2026-08: "Show creation is not currently + available via our API. New shows need to be created in the web app"). + +## Episodes + +| Method | Path | Purpose / params | +| --- | --- | --- | +| GET | `/v1/episodes` | List episodes, ordered by published date. Params: `show_id` (ID or slug), `query`, `status` (`draft`/`scheduled`/`published`), `order` (`asc`/`desc`, default `desc`), `pagination[page]`, `pagination[per]`. | +| GET | `/v1/episodes/:id` | One episode. `include[]=show` supported. `:id` is the Episode ID (slug support not documented here). | +| POST | `/v1/episodes` | Create an episode. Required: `episode[show_id]`. Optional: `episode[title]`, `episode[summary]`, `episode[description]` (HTML allowed), `episode[audio_url]`, `episode[author]`, `episode[season]`, `episode[number]`, `episode[number]` + `episode[increment_number]` (auto next number in season), `episode[type]` (`full`/`trailer`/`bonus`), `episode[image_url]`, `episode[keywords]`, `episode[explicit]`, `episode[alternate_url]`, `episode[video_url]` (video plan), `episode[youtube_url]`, `episode[transcript_text]`, `episode[email_notifications]`. **Always creates a DRAFT** (`status: "draft"`, `published_at: null`) — publishing is a separate endpoint. | +| PATCH | `/v1/episodes/:id` | Update metadata/audio. Accepts the same `episode[...]` fields as create (except `show_id`). **Never changes publishing state** — the docs say so explicitly ("publishing or unpublishing an episode involves a separate endpoint"). | +| PATCH | `/v1/episodes/:id/publish` | Publish / schedule / unpublish. Required: `episode[status]` ∈ `draft`, `scheduled`, `published`. Optional: `episode[published_at]` (show's time zone) to publish in the past, schedule for the future, or backdate. See the publish-lifecycle file for the full recipe. | +| GET | `/v1/episodes/authorize_upload` | Authorize a local audio/video upload (max **5GB**). Required: `filename`. Returns an `audio_upload` resource: signed `upload_url` (HTTP PUT the bytes, header `Content-Type: `), `content_type` (e.g. `audio/mpeg`), `expires_in` (example: 600 s), and the post-upload `audio_url` to attach via create/update. Skip entirely if you already have a public URL. | + +- Episode attributes: `title`, `status`, `season`, `number`, + `published_at`, `duration` (seconds), `duration_in_mmss`, `media_url` + (trackable MP3), `share_url`, `alternate_url`, `slug`, `summary`, + `description` (+ `formatted_*` variants), `author`, `explicit`, + `keywords`, `image_url`, `video_url`, `youtube_url`, `embed_html`(+dark), + `transcript_url`, `transcripts[]`, `audio_processing`, + `video_processing`, `processing_failure`, `hls_manifest_url`, `type`. +- `audio_processing: true` means Transistor is still processing an upload; + `processing_failure` carries the error string when processing failed. +- Vendor-documented upload formats (mcp.transistor.fm): .mp3, .m4a, .wav, + .aif, .aiff, .aifc, .mp4, .mov. + +## Analytics + +| Method | Path | Purpose / params | +| --- | --- | --- | +| GET | `/v1/analytics/:id` | Show downloads per day. `:id` = Show ID or slug. Default window: last 14 days. | +| GET | `/v1/analytics/:id/episodes` | Per-episode download series for a whole show. `:id` = Show ID or slug. Default window: last 7 days. | +| GET | `/v1/analytics/episodes/:id` | Single episode downloads per day. `:id` = Episode ID or slug. Default window: last 14 days. | + +- Date range params on all three: `start_date` and `end_date`, documented + as **dd-mm-yyyy**; if you supply one you must supply both. +- Analytics resources return a per-day `downloads` **array** + (`[{"date": ..., "downloads": N}, ...]`) — not a totals object. Sum the + array yourself (or let the bundled CLI do it: `downloads_total`). +- Doc-format quirk: example responses echo download-row dates + inconsistently (`15-08-2026` in show analytics vs `08-15-2026` in + episodes analytics). Never parse the row date format; aggregate the + numeric `downloads` values keyed by position in your requested window. +- There is no `/v1/shows/:id/analytics` route — analytics paths live under + `/v1/analytics/...`. Downloads are the only analytics exposed by the API + (no countries/apps/video stats). + +## Subscribers (private podcasts) + +| Method | Path | Purpose / params | +| --- | --- | --- | +| GET | `/v1/subscribers` | List a private show's subscribers. Required: `show_id`. Optional: `query`, `activated` (boolean), pagination. | +| GET | `/v1/subscribers/:id` | One subscriber with `email`, `status` (`default`/`subscribed`/`unsubscribed`), per-subscriber `feed_url` and `subscribe_url`, `has_downloads`, `last_notified_at`. | +| POST | `/v1/subscribers` | Add one subscriber. Required: `show_id`, `email`. Optional: `skip_welcome_email` (default false). | +| POST | `/v1/subscribers/batch` | Add many. Required: `show_id`, `emails[]` (repeat the key). Optional: `skip_welcome_email`. Response: array of subscriber resources. | +| PATCH | `/v1/subscribers/:id` | Update. Required: `subscriber[email]`. | +| DELETE | `/v1/subscribers` | Revoke by address. Required: `show_id`, `email`. | +| DELETE | `/v1/subscribers/:id` | Revoke by subscriber ID. | + +Subscriber routes are top-level (`/v1/subscribers...`), not nested under +`/v1/shows/:id/`. Each subscriber gets a unique personal feed URL — that is +how Transistor tracks private-listener downloads. + +## Webhooks + +| Method | Path | Purpose / params | +| --- | --- | --- | +| GET | `/v1/webhooks` | List a show's webhooks. Required: `show_id`. | +| POST | `/v1/webhooks` | Subscribe. Required: `event_name`, `show_id`, `url`. `event_name` ∈ `episode_created`, `episode_published`, `subscriber_created`, `subscriber_deleted`. | +| DELETE | `/v1/webhooks/:id` | Unsubscribe by webhook ID. | + +Maximum **50 webhooks per user account** (Webhook resource doc). Webhooks +are the sanctioned alternative to polling given the 10 req / 10 s rate +limit: register `episode_published` and react, instead of re-reading +episode lists. + +## Routes that do NOT exist (common wrong guesses) + +- `GET /v1/user`, `GET /v1/authorization` — the user probe is `GET /v1`. +- `POST /v1/shows` — show creation is dashboard-only. +- `/v1/shows/:id/analytics`, `/v1/episodes/:id/analytics` (nested) — + analytics lives at `/v1/analytics/...` paths. +- `/v1/shows/:id/subscribers` — subscribers is top-level with `show_id`. +- Any `pagination[limit]`-style param — per-page is `pagination[per]`. + +## Sources + +- https://developers.transistor.fm/ — fetched live 2026-08-29 (HTTP 200); + all endpoint tables above correspond to the reference sections: + #get-v1, #get-v1-analytics-id, #get-v1-analytics-id-episodes, + #get-v1-analytics-episodes-id, #get-v1-shows, #get-v1-shows-id, + #patch-v1-shows-id, #get-v1-episodes, #get-v1-episodes-id, + #get-v1-episodes-authorize_upload, #post-v1-episodes, + #patch-v1-episodes-id, #patch-v1-episodes-id-publish, + #get-v1-subscribers, #get-v1-subscribers-id, #post-v1-subscribers, + #post-v1-subscribers-batch, #patch-v1-subscribers-id, + #delete-v1-subscribers, #delete-v1-subscribers-id, #get-v1-webhooks, + #post-v1-webhooks, #delete-v1-webhooks-id, #Show, #Episode, + #Subscriber, #ShowAnalytics, #EpisodesAnalytics, #EpisodeAnalytics, + #AudioUpload, #Webhook +- https://support.transistor.fm/en/article/what-automations-are-possible-with-transistor-bi27am/ + (show-creation limitation, supported automations) — fetched live 2026-08-29 +- https://mcp.transistor.fm/ (accepted upload formats; draft-then-publish + semantics as implemented by Transistor's own tooling) — fetched live 2026-08-29 +- https://pkg.go.dev/gitlab.com/flimzy/transistor (independent SDK route + inventory corroborating the catalog) — fetched live 2026-08-29 diff --git a/transistor/references/episode-publish-lifecycle.md b/transistor/references/episode-publish-lifecycle.md new file mode 100644 index 0000000..79d10f9 --- /dev/null +++ b/transistor/references/episode-publish-lifecycle.md @@ -0,0 +1,201 @@ +# The Episode Publish Lifecycle (draft → audio → publish) + +The single most important behavioral fact of the Transistor API: +**creating an episode never publishes it, and updating an episode never +publishes it.** Publishing, scheduling, and unpublishing travel on their +own dedicated endpoint. Everything below is from the official reference +(developers.transistor.fm, fetched live 2026-08-29). + +## States + +`attributes.status` is exactly one of: + +| Status | Meaning | +| --- | --- | +| `draft` | Not in the RSS feed. New episodes start here (`published_at: null`). | +| `scheduled` | Will publish at `episode[published_at]` (show's time zone). | +| `published` | Live in the RSS feed; `published_at` records the publish time. | + +Transistor's own tooling describes the same model ("Episodes are always +created as drafts, and publishing is a separate tool call" — the vendor MCP +server), and the REST docs describe the publish endpoint's purpose as +"Publish a single episode now or in the past, schedule for the future, or +revert to a draft." All three states go through the same endpoint: it is a +setter, not a one-way transition — you can pull a published episode back to +draft, or re-publish a draft later. + +## The publish request (exact documented shape) + +The endpoint is `PATCH /v1/episodes/:id/publish` — a metadata PATCH to +`/v1/episodes/:id` will **not** publish (the docs say so on the update +endpoint's own page). The episode ID is the URL path parameter; the body +carries the status: + +```sh +curl https://api.transistor.fm/v1/episodes//publish -X PATCH \ + -H "x-api-key: " \ + -d "episode[status]=published" \ + -d "fields[episode][]=status" +``` + +- Required: `episode[status]` ∈ {`draft`, `scheduled`, `published`}. +- Optional: `episode[published_at]` — the publish date/time **in the + show's time zone**. Combine it with `episode[status]=scheduled` to + schedule for the future, or with `published` to backdate an episode + (e.g. importing an archive with historical dates). +- The documented request body is form-encoded bracket keys. The API intro + says JSON bodies are accepted generally, but the reference publishes no + JSON equivalent for this action — copy the documented shape above. +- Note what the body does **not** contain: no `id`, no `type`, no JSON:API + `data` wrapper. This is not a JSON:API resource-identifier request body; + the resource identity lives in the URL. (The *response*, by contrast, is + a standard JSON:API document — see below.) + +Documented scheduling example (same endpoint): + +```sh +curl https://api.transistor.fm/v1/episodes//publish -X PATCH \ + -H "x-api-key: " \ + -d "episode[status]=scheduled" \ + -d "episode[published_at]=2026-09-03 09:00:00" +``` + +## The publish response + +With `fields[episode][]=status` the documented response is: + +```json +{ + "data": { + "id": "", + "type": "episode", + "attributes": {"status": "published"}, + "relationships": {} + } +} +``` + +Read success off the response: `data.id` matches the episode you patched, +`data.type` is `"episode"`, and `data.attributes.status` carries the new +state. Without the fieldset you also get `published_at`, `media_url`, +`share_url`, `duration`, and the rest of the episode resource. + +## Worked recipe: create → attach audio → publish (CLI) + +```bash +export TRANSISTOR_API_KEY="" + +# 1. Create the episode (always a draft). Audio can ride along now: +transistor episode-create --show \ + --title "Episode 12: Roasting" \ + --summary "A primer on roasting coffee" \ + --season 2 --number 4 \ + --audio-url "https://uploads.example.com/ep12.mp3" +# -> {"id": "", "status": "draft", ...} + +# (skip to 3 if you attached audio above) +# 2. Attach audio later via the metadata PATCH — this does NOT publish: +transistor episode-update --id \ + --audio-url "https://uploads.example.com/ep12.mp3" + +# 3. Publish on the dedicated endpoint: +transistor episode-publish --id +# PATCH /v1/episodes//publish with episode[status]=published +``` + +The bundled CLI guards step 3: if `attributes.media_url` is still empty it +refuses to publish (an audio-less item would go out to every feed reader) +and prints the attach-audio recipe; `--force` overrides. + +## Worked recipe: raw curl + +```bash +# 1. Create a draft +curl https://api.transistor.fm/v1/episodes -X POST \ + -H "x-api-key: " \ + -d "episode[show_id]=" \ + -d "episode[title]=Example episode" \ + -d "episode[audio_url]=https://example.com/audio/episode.mp3" +# -> {"data": {"id": "", "attributes": {"status": "draft", "published_at": null, ...}}} + +# 2. (only if audio was omitted) attach through the ordinary metadata PATCH +curl https://api.transistor.fm/v1/episodes/ -X PATCH \ + -H "x-api-key: " \ + -d "episode[audio_url]=https://example.com/audio/episode.mp3" + +# 3. Publish through the dedicated endpoint +curl https://api.transistor.fm/v1/episodes//publish -X PATCH \ + -H "x-api-key: " \ + -d "episode[status]=published" +# -> {"data": {"id": "", "type": "episode", +# "attributes": {"status": "published"}, "relationships": {}}} +``` + +## Local files: the authorize-upload detour + +If the audio exists only on disk (no public URL), insert this before step 1 +or 2: + +```bash +# 1. Authorize: get a signed upload URL (max 5GB) +transistor authorize-upload --filename Episode1.mp3 +# -> {"audio_url": "https://uploads.example.com/ep1.mp3", +# "upload_url": "https://...r2.cloudflarestorage.com/...", +# "content_type": "audio/mpeg", "expires_in": 600} + +# 2. PUT the file bytes to attributes.upload_url with the returned +# Content-Type (the CLI does this when you pass --file): +curl -X PUT -H "Content-Type: audio/mpeg" -T /path/to/Episode1.mp3 "" + +# 3. Attach attributes.audio_url via episode-create/episode-update, then publish. +``` + +The signed URL expires (`expires_in`, example 600 s) — upload promptly and +only then attach. Accepted formats (vendor-documented): .mp3, .m4a, .wav, +.aif, .aiff, .aifc, .mp4, .mov. + +## Gotchas in the lifecycle + +- **Publishing is never a side effect.** POST /episodes and PATCH + /episodes/:id both return `status: "draft"` (or leave it untouched) no + matter what fields you send. If your episode stays stubbornly draft, + you are missing the `/publish` endpoint call — that is the bug, not a + permissions problem. +- **`published_at` vs status.** Setting `episode[published_at]` alone on + the metadata PATCH does nothing to feed visibility; the publish + endpoint's `episode[status]` field is what changes state, and + `published_at` only qualifies *when*. +- **Time zone.** `episode[published_at]` is interpreted in the show's + configured `time_zone` (a show attribute), not in UTC and not in your + machine's zone. Check `transistor show --id --json`. +- **Watch processing.** After attaching audio, `audio_processing` is true + until Transistor finishes; `processing_failure` explains failures. + Publishing with an unprocessed/failed file pushes silence to subscribers. +- **Unpublish = status draft.** Same endpoint, `episode[status]=draft`; + the episode drops out of the RSS feed but keeps its id, audio, and + metadata. +- **Rate budget.** The 10 req / 10 s limit applies to the whole lifecycle; + a create + audio-attach + publish burst is fine, a loop re-publishing + 50 episodes needs sleeps or webhooks. + +## Sources + +- https://developers.transistor.fm/#patch-v1-episodes-id-publish + ("Publish, schedule, or unpublish an episode": required `episode[status]` + ∈ draft/scheduled/published, optional `episode[published_at]`, publish + request + response examples) — fetched live 2026-08-29 +- https://developers.transistor.fm/#post-v1-episodes ("Create a new draft + episode... publishing an episode involves a separate endpoint"; response + with `status: "draft"`, `published_at: null`) — fetched live 2026-08-29 +- https://developers.transistor.fm/#patch-v1-episodes-id ("publishing or + unpublishing an episode involves a separate endpoint") — fetched live 2026-08-29 +- https://developers.transistor.fm/#get-v1-episodes-authorize_upload + (authorize_upload flow, upload_url/content_type/expires_in/audio_url, + 5GB max, PUT requirement) — fetched live 2026-08-29 +- https://developers.transistor.fm/#Episode (status values, audio/video + processing attributes) — fetched live 2026-08-29 +- https://mcp.transistor.fm/ ("Episodes are always created as drafts, and + publishing is a separate tool call"; accepted upload formats) — fetched live 2026-08-29 +- https://pkg.go.dev/gitlab.com/flimzy/transistor (PublishEpisode against + PATCH /v1/episodes/:id/publish — independent implementation corroborating + the dedicated endpoint) — fetched live 2026-08-29 diff --git a/transistor/references/gotchas-and-recipes.md b/transistor/references/gotchas-and-recipes.md new file mode 100644 index 0000000..d95285e --- /dev/null +++ b/transistor/references/gotchas-and-recipes.md @@ -0,0 +1,256 @@ +# Gotchas Field Guide and Worked Recipes + +Symptom-first troubleshooting for the Transistor API, followed by end-to-end +recipes. Everything traces to developers.transistor.fm or Transistor's +support pages (fetched live 2026-08-29); independent-implementation +corroboration (the flimzy/transistor Go SDK) is cited where noted. + +## Symptom → cause → fix + +### `404` on the "current user" call + +- **Symptom:** `GET /v1/user` (or `/v1/authorization`) returns 404. +- **Cause:** those routes do not exist. The authenticated-user probe is + `GET /v1` — root, no suffix. +- **Fix:** `transistor user` (sends `GET /v1`). Older tutorials showing + `/v1/user` predate the current API surface. + +### Writes "succeed" but the episode never appears in the feed + +- **Symptom:** episode exists via `GET /v1/episodes/:id`, `status` stays + `"draft"` even after updates. +- **Cause:** POST /episodes and PATCH /episodes/:id never publish; only + `PATCH /v1/episodes/:id/publish` changes status. (The docs state this on + both the create and update pages.) +- **Fix:** `transistor episode-publish --id `, or the raw call: + `curl .../v1/episodes//publish -X PATCH -d "episode[status]=published"` + with `x-api-key`. + +### Pagination loop re-reads page 1 forever + +- **Symptom:** every page of your loop returns the same items. +- **Cause:** wrong param name. There is no `pagination[limit]`, no + `page[number]`, no `page` — unknown params are ignored. Per-page is + `pagination[per]` (default 10) and the page number is `pagination[page]` + (docs' default 0, examples request 1). +- **Fix:** loop on `meta.currentPage < meta.totalPages`, sending + `pagination[page]=N`; verify with `meta.totalCount` that you captured + everything. In the bundled CLI: `--page N --per M`. + +### `meta.totalCount` disagrees with the number of items + +- **Symptom:** `totalCount: 25` but only 10 objects in `data`. +- **Cause:** nothing is wrong — `per` defaults to 10 and the rest are on + later pages. +- **Fix:** raise `pagination[per]` or walk pages. Compare your accumulated + item count to `meta.totalCount`, not to `len(data)` of one page. + +### Show "counts" fields are missing + +- **Symptom:** your script reads `attributes.episodes_count` / + `subscribers_count` and gets `null`. +- **Cause:** show resources do not carry those fields (an older wrapper's + display invented them). +- **Fix:** list episodes with `show_id` and read `meta.totalCount`; + subscribers likewise (`GET /v1/subscribers?show_id=...`). + +### The user object has no email + +- **Symptom:** you expected `data.attributes.email` from the user probe. +- **Cause:** the `user` resource has `name`, `time_zone`, `image_url`, + timestamps — no email. +- **Fix:** use `name`/`time_zone`; identify accounts by the dashboard, not + the API. + +### Analytics numbers look "empty" + +- **Symptom:** you expected `attributes.totals.downloads.total`; you got an + array. +- **Cause:** analytics resources return per-day arrays: + `attributes.downloads = [{"date": ..., "downloads": N}, ...]`. +- **Fix:** sum the array. The bundled CLI exposes `downloads_total` and + keeps the raw `downloads` array in `--json`. +- **Related:** don't parse the row `date` format — the docs' examples echo + dates inconsistently (`15-08-2026` vs `08-15-2026`); your requested + `start_date`/`end_date` (dd-mm-yyyy) define the window, and both are + required if either is given. + +### `429` mid-loop + +- **Symptom:** bulk operations fail after ~10 quick calls. +- **Cause:** rate limit is 10 requests / 10 seconds, and the 429 blocks + access for 10 seconds. No retry headers are documented. +- **Fix:** sleep ≥10 s on 429 and retry; batch what you can + (`/v1/subscribers/batch` for imports); prefer webhooks + (`episode_published`) over polling; cache — Transistor explicitly says + the API is not a website back end. + +### `403` on something you can see in the dashboard + +- **Symptom:** the key is valid (other calls pass) but one resource 403s. +- **Cause:** role scoping. Keys inherit the user's per-podcast role + (owner/admin/team member); some operations require owner/admin. +- **Fix:** have a podcast owner/admin run it, or adjust roles in the + dashboard. + +### Signed upload URL suddenly 403s + +- **Symptom:** your PUT to the `upload_url` worked in testing, fails now. +- **Cause:** `expires_in` (example: 600 s) elapsed. +- **Fix:** re-run `authorize-upload`, PUT promptly, then attach. The PUT + must carry `Content-Type` equal to the returned `content_type`. + +### Audio attached but `duration` is null / `media_url` empty in feeds + +- **Symptom:** episode created with `episode[audio_url]` but processing + fields look stuck. +- **Cause:** `audio_processing` is true while Transistor processes; + `processing_failure` carries an error string on failure. +- **Fix:** poll `transistor episode --id --json` until + `audio_processing` is false (respecting the rate limit) before + publishing. + +## Recipe: full publish pipeline (CLI) + +```bash +export TRANSISTOR_API_KEY="" + +# 1. Verify the key and find the show +transistor shows --json | jq -r '.shows[] | [.id, .title, .slug] | @tsv' + +# 2. Local audio? authorize + upload (skippable if you have a URL) +transistor authorize-upload --filename ep12.mp3 --file ./ep12.mp3 --json | jq -r '.audio_url' + +# 3. Create the draft with audio attached +EP=$(transistor episode-create --show --title "Ep 12" \ + --audio-url "$(cat /tmp/audio_url)" --json | jq -r '.id') +echo "$EP" # draft id, type string + +# 4. Publish when ready +transistor episode-publish --id "$EP" + +# 5. Confirm state and the trackable media URL +transistor episode --id "$EP" --json | jq '{status, media_url, published_at}' +``` + +Every stage's JSON output feeds the next: `shows` → string `id`; +`authorize-upload` → string `audio_url`; `episode-create` → string `id`; +`episode-publish` → new `status`. Same pipeline raw: + +```bash +AUDIO=$(curl -s https://api.transistor.fm/v1/episodes/authorize_upload?filename=ep12.mp3 \ + -H "x-api-key: " | jq -r '.data.attributes.audio_url') +EP=$(curl -s https://api.transistor.fm/v1/episodes -X POST \ + -H "x-api-key: " -d "episode[show_id]=" \ + -d "episode[title]=Ep 12" -d "episode[audio_url]=$AUDIO" | jq -r '.data.id') +curl -s "https://api.transistor.fm/v1/episodes/$EP/publish" -X PATCH \ + -H "x-api-key: " -d "episode[status]=published" | jq '.data.attributes.status' +``` + +## Recipe: schedule a season in bulk (respect the rate limit) + +```bash +# Renumber + schedule episodes for weekly drops; 1 write call each, +# ≥1 s spacing keeps you far under 10 req / 10 s. +i=0 +for AUDIO in /media/season3/*.mp3; do + i=$((i+1)) + URL=$(transistor authorize-upload --filename "$(basename "$AUDIO")" \ + --file "$AUDIO" --json | jq -r '.audio_url') + EP=$(transistor episode-create --show \ + --title "S3E$i" --season 3 --number "$i" --audio-url "$URL" \ + --json | jq -r '.id') + transistor episode-publish --id "$EP" --status scheduled \ + --published-at "2026-09-$((7*i)) 09:00:00" + sleep 1 +done +``` + +`--published-at` is interpreted in the show's time zone; backdating +(publishing "in the past") uses the same fields with status `published`. + +## Recipe: weekly downloads report (analytics) + +```bash +# Per-show totals for a window (dd-mm-yyyy, both bounds required) +for S in $(transistor shows --json | jq -r '.shows[].id'); do + transistor analytics --show "$S" \ + --start-date 01-08-2026 --end-date 28-08-2026 --json \ + | jq -r --arg id "$S" '[$id, (.downloads_total|tostring)] | @tsv' + sleep 1 +done + +# Per-episode series for a show's recent drops +transistor episodes --show --status published --per 5 --json \ + | jq -r '.episodes[].id' \ + | while read -r EP; do + transistor episode-analytics --id "$EP" --json \ + | jq -r '[(.episode_id|tostring), (.downloads_total|tostring)] | @tsv' + sleep 1 + done +``` + +## Recipe: private-podcast subscriber import + +```bash +# Batch import (single call), then verify with a filtered listing +transistor subscriber-batch --show \ + --email "one@example.com" --email "two@example.com" \ + --skip-welcome-email --json | jq '.subscribers' + +transistor subscribers --show --json | jq -r '.meta.totalCount' +# Revoke someone: by email... +transistor subscriber-delete --show --email "two@example.com" +# ...or by id +transistor subscriber-delete --id +``` + +Private subscribers each get personal `feed_url`/`subscribe_url` values — +never share one subscriber's feed URL; it identifies them. + +## Recipe: webhook instead of polling + +```bash +transistor webhook-create --show \ + --event episode_published --url "https://example.com/hooks/transistor" +transistor webhooks --show --json | jq '.webhooks' +transistor webhook-delete --id +``` + +Events: `episode_created`, `episode_published`, `subscriber_created`, +`subscriber_deleted`. Account-wide cap: 50 webhooks. This is the sanctioned +way to stay current without spending the 10 req / 10 s budget on polling. + +## Automation boundaries + +- Show creation is not available via the API (dashboard-only). Everything + else in this guide is API-land: show updates, episode lifecycle, + subscribers, webhooks, analytics. +- The API is not meant to power a website's back end: pull once, cache, + and parse the public RSS feed for display pages. + +## Sources + +- https://developers.transistor.fm/ (authentication, rate limits, all + endpoint examples) — fetched live 2026-08-29 +- https://developers.transistor.fm/#ratelimits (10 req / 10 s, 429 + 10 s + block, caching/RSS guidance) — fetched live 2026-08-29 +- https://developers.transistor.fm/#patch-v1-episodes-id-publish, + #post-v1-episodes, #patch-v1-episodes-id (lifecycle facts: draft on + create, publish via separate endpoint, episode[status] enum, + episode[published_at]) — fetched live 2026-08-29 +- https://developers.transistor.fm/#get-v1-episodes-authorize_upload + (signed upload flow, expires_in example 600, 5GB max) — fetched live 2026-08-29 +- https://developers.transistor.fm/#get-v1-analytics-id, + #get-v1-analytics-id-episodes, #get-v1-analytics-episodes-id + (dd-mm-yyyy date pair rule; per-day downloads arrays) — fetched live 2026-08-29 +- https://developers.transistor.fm/#Show, #Episode, #Subscriber (attribute + inventory: no counts on shows, no email on users, per-subscriber feed + URLs) — fetched live 2026-08-29 +- https://developers.transistor.fm/#Webhook (event names, 50-webhook cap) — + fetched live 2026-08-29 +- https://support.transistor.fm/en/article/what-automations-are-possible-with-transistor-bi27am/ + (show creation limitation; automation guidance) — fetched live 2026-08-29 +- https://mcp.transistor.fm/ (accepted upload formats) — fetched live 2026-08-29 +- https://pkg.go.dev/gitlab.com/flimzy/transistor (route/param + corroboration) — fetched live 2026-08-29 diff --git a/transistor/scripts/test_transistor.py b/transistor/scripts/test_transistor.py new file mode 100644 index 0000000..c643e33 --- /dev/null +++ b/transistor/scripts/test_transistor.py @@ -0,0 +1,912 @@ +"""Offline test suite for the bundled transistor CLI. + +All HTTP is mocked at the client seam (TransistorClient._request is replaced +by a FakeTransport that records method/path/params/body and returns canned +JSON:API documents) — the suite is fully offline and passes the proxy-trap +rerun. Transistor is a keyed API, so there are deliberately NO live-call test +cases (the AGENTS.md network policy is mock-everything for keyed APIs). + +Covers the four contract behavior classes: --help output, argument-error +paths, --dry-run plans, and mocked parsing of canned JSON:API compound +documents (data/attributes/relationships/included[]), plus the documented +multi-step pipelines (each stage's output fields AND JSON types feed the +next: shows -> episodes, episode-create -> episode-update(audio) -> +episode-publish, analytics -> summed downloads). +""" + +import contextlib +import importlib.machinery +import importlib.util +import io +import json +import os +import pathlib +import stat +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +SCRIPT = pathlib.Path(__file__).resolve().parent / "transistor" +LOADER = importlib.machinery.SourceFileLoader("transistor_cli", str(SCRIPT)) +SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER) +ts = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = ts # so unittest.mock.patch("transistor_cli....") resolves +LOADER.exec_module(ts) + +SHOW_ID = "132543" +EPISODE_ID = "3056098" +DRAFT_ID = "3056099" + +# Canned JSON:API compound documents mirroring the documented response shapes +# (developers.transistor.fm): top-level `data`, resource objects with +# `attributes` + `relationships`, compound documents with `included[]`, and +# collection pagination under `meta` (currentPage/totalPages/totalCount). +SHOW_RESOURCE = { + "id": SHOW_ID, + "type": "show", + "attributes": { + "title": "The Caffeine Show", + "slug": "the-caffeine-show", + "description": "A podcast covering all things coffee and caffeine", + "show_type": "episodic", + "private": False, + "feed_url": "https://feeds.transistor.fm/the-caffeine-show", + "time_zone": "UTC", + "author": "Jimmy Podcaster", + "website": "https://example.com/caffeine", + }, + "relationships": {"episodes": {"data": []}}, +} + +USER_DOC = { + "data": { + "id": "173455", + "type": "user", + "attributes": {"name": "Jimmy Podcaster", "time_zone": "UTC", "image_url": None}, + } +} + +SHOWS_DOC = {"data": [SHOW_RESOURCE], "meta": {"currentPage": 0, "totalPages": 1, "totalCount": 1}} +SHOW_DOC = {"data": SHOW_RESOURCE} + +EPISODE_RESOURCE = { + "id": EPISODE_ID, + "type": "episode", + "attributes": { + "title": "How To Roast Coffee", + "number": 1, + "season": 1, + "status": "published", + "published_at": "2020-07-01 00:00:00 UTC", + "duration": 568, + "duration_in_mmss": "09:28", + "media_url": "https://media.transistor.fm/ba1d5241/c1ae0a3a.mp3", + "share_url": "https://share.transistor.fm/s/ba1d5241", + "audio_processing": False, + "processing_failure": None, + }, + "relationships": {"show": {"data": {"id": SHOW_ID, "type": "show"}}}, +} + +DRAFT_RESOURCE = dict( + EPISODE_RESOURCE, + id=DRAFT_ID, + attributes=dict( + EPISODE_RESOURCE["attributes"], + title="Unfinished Episode", + number=2, + status="draft", + published_at=None, + media_url="", + ), +) + +# Compound document: episode collection with the parent show in included[]. +EPISODES_DOC = { + "data": [EPISODE_RESOURCE, DRAFT_RESOURCE], + "included": [SHOW_RESOURCE], + "meta": {"currentPage": 1, "totalPages": 3, "totalCount": 25}, +} +EPISODE_DOC = {"data": EPISODE_RESOURCE} +DRAFT_DOC = {"data": DRAFT_RESOURCE} +NO_AUDIO_DOC = { + "data": dict(EPISODE_RESOURCE, attributes=dict(EPISODE_RESOURCE["attributes"], media_url="")) +} + +CREATED_DRAFT_RESOURCE = dict( + DRAFT_RESOURCE, + id="3056100", + attributes=dict(DRAFT_RESOURCE["attributes"], title="Fresh Draft", number=None, season=2), +) +CREATE_DOC = {"data": CREATED_DRAFT_RESOURCE} + +PUBLISH_DOC = { + "data": { + "id": EPISODE_ID, + "type": "episode", + "attributes": { + "status": "published", + "published_at": "2026-08-29 12:00:00 UTC", + "media_url": EPISODE_RESOURCE["attributes"]["media_url"], + }, + "relationships": {}, + } +} + +SHOW_ANALYTICS_DOC = { + "data": { + "id": "the-caffeine-show", + "type": "show_analytics", + "attributes": { + "downloads": [ + {"date": "15-08-2026", "downloads": 4}, + {"date": "16-08-2026", "downloads": 6}, + ], + "start_date": "08-15-2026", + "end_date": "08-16-2026", + }, + "relationships": {"show": {"data": {"id": SHOW_ID, "type": "show"}}}, + }, + "included": [dict(SHOW_RESOURCE, attributes={"title": "The Caffeine Show"})], +} + +EPISODE_ANALYTICS_DOC = { + "data": dict( + SHOW_ANALYTICS_DOC["data"], + id=EPISODE_ID, + type="episode_analytics", + relationships={"episode": {"data": {"id": EPISODE_ID, "type": "episode"}}}, + ) +} + +AUDIO_UPLOAD_DOC = { + "data": { + "id": "upload-1", + "type": "audio_upload", + "attributes": { + "upload_url": "https://storage.example.com/uploads/episode1.mp3?sig=stub", + "content_type": "audio/mpeg", + "expires_in": 600, + "audio_url": "https://uploads.example.com/episode1.mp3", + }, + } +} + +SUBSCRIBER_RESOURCE = { + "id": "709423", + "type": "subscriber", + "attributes": { + "email": "arthur@example.com", + "status": "default", + "feed_url": "https://subscribers.example.com/a52a98c03f28eb", + "subscribe_url": "https://subscribe.example.com/a52a98c03f28eb", + "has_downloads": False, + }, + "relationships": {"show": {"data": {"id": SHOW_ID, "type": "show"}}}, +} +SUBSCRIBERS_DOC = { + "data": [SUBSCRIBER_RESOURCE], + "meta": {"currentPage": 0, "totalPages": 1, "totalCount": 1}, +} +SUBSCRIBER_BATCH_DOC = { + "data": [ + SUBSCRIBER_RESOURCE, + dict( + SUBSCRIBER_RESOURCE, + id="709424", + attributes=dict(SUBSCRIBER_RESOURCE["attributes"], email="beatrice@example.com"), + ), + ] +} + +WEBHOOK_RESOURCE = { + "id": "104325", + "type": "webhook", + "attributes": {"event_name": "episode_published", "url": "https://example.com/hook"}, + "relationships": {"show": {"data": {"id": SHOW_ID, "type": "show"}}}, +} +WEBHOOKS_DOC = {"data": [WEBHOOK_RESOURCE]} + + +class FakeResponse: + def __init__(self, status_code=200, json_body=None, text=""): + self.status_code = status_code + self._json = json_body + self.text = text if text else (json.dumps(json_body) if json_body is not None else "") + + def json(self): + if self._json is None: + raise ValueError("no json body") + return self._json + + +class FakeTransport: + """Stands in for TransistorClient._request: records every call and serves + canned JSON:API documents by (method, path).""" + + def __init__(self, routes=None): + self.routes = routes or {} + self.calls = [] + + def __call__(self, method, path, params=None, body=None): + self.calls.append({"method": method, "path": path, "params": params, "body": body}) + for (m, p), doc in self.routes.items(): + if m == method and path == p: + return doc + for (m, p), doc in self.routes.items(): + if m == method and path.startswith(p): + return doc + raise AssertionError(f"unexpected request: {method} {path} {params} {body}") + + +def make_client(routes=None): + client = ts.TransistorClient(key="test-key", dry_run=False) + client._request = FakeTransport(routes) + return client + + +def run_handler(client, handler, *argv): + out = io.StringIO() + with contextlib.redirect_stdout(out): + handler(client, list(argv)) + return out.getvalue() + + +def run_cli(*args): + env = os.environ.copy() + env.pop("TRANSISTOR_API_KEY", None) + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + env=env, + ) + + +class ModuleStateTestCase(unittest.TestCase): + """Base that restores module globals mutated by in-process tests.""" + + def setUp(self): + self._flags = dict(ts.GLOBAL_FLAGS) + self._quiet = ts.QUIET + ts.GLOBAL_FLAGS = {"json": True, "dry_run": False, "quiet": False, "verbose": False} + + def tearDown(self): + ts.GLOBAL_FLAGS = self._flags + ts.QUIET = self._quiet + + +class HelpOutputTests(unittest.TestCase): + """Class 1: --help output.""" + + def test_help_lists_every_subcommand(self): + result = run_cli("--help") + self.assertEqual(result.returncode, 0, result.stderr) + for noun in ( + "user", + "shows", + "show-update", + "episodes", + "episode-create", + "episode-update", + "episode-publish", + "authorize-upload", + "analytics", + "episode-analytics", + "subscribers", + "subscriber-create", + "subscriber-batch", + "subscriber-delete", + "webhooks", + "webhook-create", + "webhook-delete", + ): + self.assertIn(noun, result.stdout) + + def test_help_names_the_env_var_and_docs_url(self): + result = run_cli("--help") + self.assertIn("TRANSISTOR_API_KEY", result.stdout) + self.assertIn("dashboard.transistor.fm/account", result.stdout) + self.assertIn("developers.transistor.fm", result.stdout) + + def test_leaf_help_carries_examples_and_flags(self): + for leaf in ("episodes", "episode-publish", "authorize-upload", "analytics"): + result = run_cli(leaf, "--help") + with self.subTest(leaf=leaf): + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("--", result.stdout) + + def test_no_command_prints_help_and_exits_one(self): + result = run_cli() + self.assertEqual(result.returncode, 1) + self.assertIn("usage", result.stdout) + + +class ArgumentErrorTests(unittest.TestCase): + """Class 2: argument-error paths fail cleanly before any network call.""" + + def assertCleanError(self, result, needle): + self.assertNotEqual(result.returncode, 0) + self.assertIn(needle, result.stderr) + self.assertNotIn("Traceback", result.stderr) + + def test_episode_requires_id(self): + self.assertCleanError(run_cli("episode"), "--id") + + def test_episode_create_requires_show_and_title(self): + self.assertCleanError(run_cli("episode-create", "--show", SHOW_ID), "--title") + + def test_analytics_requires_show(self): + self.assertCleanError(run_cli("analytics"), "--show") + + def test_analytics_dates_require_each_other(self): + self.assertCleanError( + run_cli("analytics", "--show", SHOW_ID, "--start-date", "01-09-2026"), + "end_date must be used together", + ) + + def test_analytics_dates_reject_wrong_format(self): + self.assertCleanError( + run_cli( + "analytics", + "--show", + SHOW_ID, + "--start-date", + "2026-09-01", + "--end-date", + "2026-09-07", + ), + "dd-mm-yyyy", + ) + + def test_episode_update_with_no_fields_dies(self): + self.assertCleanError(run_cli("episode-update", "--id", EPISODE_ID), "Nothing to update") + + def test_subscriber_delete_requires_arguments(self): + self.assertCleanError(run_cli("subscriber-delete"), "--id") + + def test_missing_api_key_dies_before_network(self): + result = run_cli("shows") + self.assertCleanError(result, "TRANSISTOR_API_KEY") + + +class DryRunPlanTests(ModuleStateTestCase): + """Class 3: --dry-run emits valid JSON plans with zero network activity.""" + + def run_json(self, handler, client, *argv): + out = run_handler(client, handler, *argv) + return json.loads(out) + + def test_plan_shape_covers_method_path_params_body(self): + client = ts.TransistorClient(dry_run=True) + plan = self.run_json(ts.cmd_user, client) + self.assertTrue(plan["dry_run"]) + self.assertEqual(plan["method"], "GET") + self.assertEqual(plan["path"], "") + + def test_episodes_plan_carries_documented_params(self): + client = ts.TransistorClient(dry_run=True) + plan = self.run_json( + ts.cmd_episodes, client, "--show", SHOW_ID, "--status", "draft", "--per", "5" + ) + self.assertEqual(plan["method"], "GET") + self.assertEqual(plan["path"], "/episodes") + self.assertEqual(plan["params"]["show_id"], SHOW_ID) + self.assertEqual(plan["params"]["status"], "draft") + self.assertEqual(plan["params"]["pagination[per]"], 5) + # pagination[page] only appears when requested (API default is page 0, + # but the CLI does not send params the user did not ask for). + self.assertNotIn("pagination[page]", plan["params"]) + + def test_limit_alias_feeds_per_param(self): + client = ts.TransistorClient(dry_run=True) + plan = self.run_json(ts.cmd_episodes, client, "--limit", "7") + self.assertEqual(plan["params"]["pagination[per]"], 7) + + def test_create_plan_sends_bracket_keys(self): + client = ts.TransistorClient(dry_run=True) + plan = self.run_json( + ts.cmd_episode_create, client, "--show", SHOW_ID, "--title", "Fresh Draft" + ) + self.assertEqual(plan["method"], "POST") + self.assertEqual(plan["path"], "/episodes") + self.assertEqual(plan["body"]["episode[show_id]"], SHOW_ID) + self.assertEqual(plan["body"]["episode[title]"], "Fresh Draft") + + def test_publish_plan_is_the_dedicated_publish_endpoint(self): + client = ts.TransistorClient(dry_run=True) + plan = self.run_json(ts.cmd_episode_publish, client, "--id", EPISODE_ID) + self.assertEqual(plan["method"], "PATCH") + self.assertEqual(plan["path"], f"/episodes/{EPISODE_ID}/publish") + self.assertEqual(plan["body"], {"episode[status]": "published"}) + + def test_schedule_plan_carries_published_at(self): + client = ts.TransistorClient(dry_run=True) + plan = self.run_json( + ts.cmd_episode_publish, + client, + "--id", + EPISODE_ID, + "--status", + "scheduled", + "--published-at", + "2026-09-03 09:00:00", + ) + self.assertEqual(plan["body"]["episode[status]"], "scheduled") + self.assertEqual(plan["body"]["episode[published_at]"], "2026-09-03 09:00:00") + + def test_update_plan_attaches_audio_without_publishing(self): + client = ts.TransistorClient(dry_run=True) + plan = self.run_json( + ts.cmd_episode_update, + client, + "--id", + EPISODE_ID, + "--audio-url", + "https://uploads.example.com/episode1.mp3", + ) + self.assertEqual(plan["method"], "PATCH") + self.assertEqual(plan["path"], f"/episodes/{EPISODE_ID}") + self.assertEqual( + plan["body"], {"episode[audio_url]": "https://uploads.example.com/episode1.mp3"} + ) + + def test_authorize_upload_plan_does_not_leak_urls(self): + client = ts.TransistorClient(dry_run=True) + out = run_handler(client, ts.cmd_authorize_upload, "--filename", "Episode1.mp3") + plan = json.loads(out) + self.assertEqual(plan["method"], "GET") + self.assertEqual(plan["params"], {"filename": "Episode1.mp3"}) + self.assertIn("then_put", plan) + self.assertIn("HTTP PUT", plan["then_put"]["how"]) + + def test_batch_plan_sends_email_array(self): + client = ts.TransistorClient(dry_run=True) + plan = self.run_json( + ts.cmd_subscriber_batch, + client, + "--show", + SHOW_ID, + "--email", + "one@example.com", + "--email", + "two@example.com", + ) + self.assertEqual(plan["path"], "/subscribers/batch") + self.assertEqual(plan["body"]["emails[]"], ["one@example.com", "two@example.com"]) + self.assertEqual(plan["body"]["show_id"], SHOW_ID) + + def test_cli_json_dry_run_subprocess_is_valid_json(self): + result = run_cli( + "--json", "--dry-run", "episodes", "--show", SHOW_ID, "--status", "published" + ) + self.assertEqual(result.returncode, 0, result.stderr) + plan = json.loads(result.stdout) + self.assertTrue(plan["dry_run"]) + self.assertEqual(plan["path"], "/episodes") + self.assertEqual(plan["params"]["status"], "published") + + +class JSONAPIDocumentTests(ModuleStateTestCase): + """Class 4: mocked parsing of canned JSON:API compound documents.""" + + def test_user_parses_data_attributes(self): + client = make_client({("GET", ""): USER_DOC}) + out = run_handler(client, ts.cmd_user) + payload = json.loads(out) + self.assertEqual(payload["id"], "173455") + self.assertEqual(payload["name"], "Jimmy Podcaster") + self.assertEqual(payload["time_zone"], "UTC") + + def test_shows_parses_collection_and_meta(self): + client = make_client({("GET", "/shows"): SHOWS_DOC}) + payload = json.loads(run_handler(client, ts.cmd_shows)) + self.assertEqual(payload["meta"]["totalPages"], 1) + show = payload["shows"][0] + self.assertEqual(show["id"], SHOW_ID) + self.assertEqual(show["title"], "The Caffeine Show") + self.assertEqual(show["feed_url"], "https://feeds.transistor.fm/the-caffeine-show") + + def test_episodes_compound_document_includes_show(self): + client = make_client({("GET", "/episodes"): EPISODES_DOC}) + payload = json.loads(run_handler(client, ts.cmd_episodes, "--include", "show")) + self.assertEqual(payload["meta"]["totalCount"], 25) + first = payload["episodes"][0] + self.assertIsInstance(first["id"], str) + self.assertEqual(first["title"], "How To Roast Coffee") + self.assertEqual(first["status"], "published") + self.assertIsInstance(first["season"], int) + self.assertIsInstance(first["duration"], int) + self.assertEqual(first["show_id"], SHOW_ID) + draft = payload["episodes"][1] + self.assertEqual(draft["status"], "draft") + self.assertEqual(draft["published_at"], "") + # Human mode also prints the included[] show summary via log(). + ts.GLOBAL_FLAGS = {"json": False, "dry_run": False, "quiet": False, "verbose": False} + human = run_handler(client, ts.cmd_episodes, "--include", "show") + ts.GLOBAL_FLAGS = {"json": True, "dry_run": False, "quiet": False, "verbose": False} + self.assertIn("included show: The Caffeine Show", human) + + def test_episode_relationships_expose_show_id(self): + client = make_client({("GET", f"/episodes/{EPISODE_ID}"): EPISODE_DOC}) + payload = json.loads( + run_handler(client, ts.cmd_episode, "--id", EPISODE_ID, "--include", "show") + ) + self.assertEqual(payload["show_id"], SHOW_ID) + self.assertEqual(payload["media_url"], EPISODE_RESOURCE["attributes"]["media_url"]) + + def test_analytics_sums_downloads_array(self): + client = make_client({("GET", f"/analytics/{SHOW_ID}"): SHOW_ANALYTICS_DOC}) + payload = json.loads(run_handler(client, ts.cmd_analytics, "--show", SHOW_ID)) + self.assertEqual(payload["downloads_total"], 10) + self.assertEqual(payload["days"], 2) + self.assertIsInstance(payload["downloads"], list) + row = payload["downloads"][0] + self.assertIsInstance(row["downloads"], int) + + def test_subscribers_list_parses_envelope(self): + client = make_client({("GET", "/subscribers"): SUBSCRIBERS_DOC}) + payload = json.loads(run_handler(client, ts.cmd_subscribers, "--show", SHOW_ID)) + sub = payload["subscribers"][0] + self.assertEqual(sub["email"], "arthur@example.com") + self.assertEqual(sub["subscribe_url"], SUBSCRIBER_RESOURCE["attributes"]["subscribe_url"]) + + def test_webhooks_list_parses_event_names(self): + client = make_client({("GET", "/webhooks"): WEBHOOKS_DOC}) + payload = json.loads(run_handler(client, ts.cmd_webhooks, "--show", SHOW_ID)) + self.assertEqual(payload["webhooks"][0]["event_name"], "episode_published") + + +class WritePathTests(ModuleStateTestCase): + """Mocked write commands must send the documented bracket-key bodies and + the dedicated publish endpoint.""" + + def test_episode_create_posts_show_id_and_title(self): + client = make_client({("POST", "/episodes"): CREATE_DOC}) + payload = json.loads( + run_handler( + client, + ts.cmd_episode_create, + "--show", + SHOW_ID, + "--title", + "Fresh Draft", + "--season", + "2", + "--audio-url", + "https://uploads.example.com/x.mp3", + ) + ) + call = client._request.calls[0] + self.assertEqual(call["method"], "POST") + self.assertEqual(call["body"]["episode[show_id]"], SHOW_ID) + self.assertEqual(call["body"]["episode[title]"], "Fresh Draft") + self.assertEqual(call["body"]["episode[season]"], 2) + self.assertEqual(call["body"]["episode[audio_url]"], "https://uploads.example.com/x.mp3") + self.assertEqual(payload["status"], "draft") + + def test_created_draft_points_at_publish_recipe(self): + client = make_client({("POST", "/episodes"): CREATE_DOC}) + ts.GLOBAL_FLAGS = {"json": False, "dry_run": False, "quiet": False, "verbose": False} + out = run_handler( + client, ts.cmd_episode_create, "--show", SHOW_ID, "--title", "Fresh Draft" + ) + self.assertIn(f"episode-publish --id {CREATED_DRAFT_RESOURCE['id']}", out) + + def test_publish_sends_status_to_publish_endpoint(self): + client = make_client( + { + ("GET", f"/episodes/{EPISODE_ID}"): EPISODE_DOC, + ("PATCH", f"/episodes/{EPISODE_ID}/publish"): PUBLISH_DOC, + } + ) + payload = json.loads(run_handler(client, ts.cmd_episode_publish, "--id", EPISODE_ID)) + call = client._request.calls[-1] + self.assertEqual(call["method"], "PATCH") + self.assertEqual(call["path"], f"/episodes/{EPISODE_ID}/publish") + self.assertEqual(call["body"], {"episode[status]": "published"}) + self.assertEqual(payload["status"], "published") + + def test_unpublish_sends_draft_status(self): + client = make_client( + { + ("GET", f"/episodes/{EPISODE_ID}"): EPISODE_DOC, + ("PATCH", f"/episodes/{EPISODE_ID}/publish"): PUBLISH_DOC, + } + ) + run_handler(client, ts.cmd_episode_publish, "--id", EPISODE_ID, "--status", "draft") + self.assertEqual(client._request.calls[-1]["body"], {"episode[status]": "draft"}) + + def test_show_update_sends_show_bracket_keys(self): + client = make_client({("PATCH", f"/shows/{SHOW_ID}"): SHOW_DOC}) + run_handler( + client, + ts.cmd_show_update, + "--id", + SHOW_ID, + "--title", + "New Title", + "--author", + "New Author", + ) + call = client._request.calls[0] + self.assertEqual(call["method"], "PATCH") + self.assertEqual(call["body"], {"show[title]": "New Title", "show[author]": "New Author"}) + + +class PipelineChainTests(ModuleStateTestCase): + """Documented multi-step recipes must execute stage by stage, each stage's + output field names AND JSON types consumable by the next.""" + + def test_shows_then_filtered_episodes_pipeline(self): + client = make_client({("GET", "/shows"): SHOWS_DOC, ("GET", "/episodes"): EPISODES_DOC}) + first = json.loads(run_handler(client, ts.cmd_shows)) + consumed_show_id = first["shows"][0]["id"] + self.assertIsInstance(consumed_show_id, str) + second = json.loads(run_handler(client, ts.cmd_episodes, "--show", consumed_show_id)) + self.assertEqual(client._request.calls[1]["params"]["show_id"], consumed_show_id) + self.assertEqual(second["episodes"][0]["show_id"], consumed_show_id) + + def test_analytics_date_pair_pipeline(self): + client = make_client({("GET", f"/analytics/{SHOW_ID}"): SHOW_ANALYTICS_DOC}) + payload = json.loads( + run_handler( + client, + ts.cmd_analytics, + "--show", + SHOW_ID, + "--start-date", + "15-08-2026", + "--end-date", + "16-08-2026", + ) + ) + call = client._request.calls[0] + self.assertEqual(call["params"], {"start_date": "15-08-2026", "end_date": "16-08-2026"}) + self.assertEqual(payload["downloads_total"], 10) + + def test_create_then_attach_audio_then_publish_pipeline(self): + client = make_client( + { + ("POST", "/episodes"): CREATE_DOC, + ("PATCH", f"/episodes/{CREATED_DRAFT_RESOURCE['id']}"): { + "data": CREATED_DRAFT_RESOURCE + }, + ("GET", f"/episodes/{CREATED_DRAFT_RESOURCE['id']}"): { + "data": dict( + CREATED_DRAFT_RESOURCE, + attributes=dict( + CREATED_DRAFT_RESOURCE["attributes"], + media_url="https://uploads.example.com/final.mp3", + ), + ) + }, + ("PATCH", f"/episodes/{CREATED_DRAFT_RESOURCE['id']}/publish"): { + "data": dict( + CREATED_DRAFT_RESOURCE, + attributes=dict( + CREATED_DRAFT_RESOURCE["attributes"], + status="published", + published_at="2026-08-29 12:00:00 UTC", + ), + ) + }, + } + ) + # Stage 1: create -> returns the draft id (str) consumed downstream. + created = json.loads( + run_handler(client, ts.cmd_episode_create, "--show", SHOW_ID, "--title", "Fresh Draft") + ) + self.assertEqual(created["status"], "draft") + self.assertIsInstance(created["id"], str) + draft_id = created["id"] + # Stage 2: attach audio via the metadata PATCH (id + audio_url feed in). + run_handler( + client, + ts.cmd_episode_update, + "--id", + draft_id, + "--audio-url", + "https://uploads.example.com/final.mp3", + ) + attach_call = client._request.calls[1] + self.assertEqual(attach_call["path"], f"/episodes/{draft_id}") + self.assertEqual( + attach_call["body"], {"episode[audio_url]": "https://uploads.example.com/final.mp3"} + ) + # Stage 3: publish on the dedicated endpoint reuses the same id (str). + published = json.loads(run_handler(client, ts.cmd_episode_publish, "--id", draft_id)) + publish_call = client._request.calls[-1] + self.assertEqual(publish_call["path"], f"/episodes/{draft_id}/publish") + self.assertEqual(publish_call["body"], {"episode[status]": "published"}) + self.assertEqual(published["status"], "published") + self.assertIsInstance(published["published_at"], str) + + def test_episodes_then_episode_analytics_pipeline(self): + client = make_client( + { + ("GET", "/episodes"): EPISODES_DOC, + ("GET", f"/analytics/episodes/{EPISODE_ID}"): EPISODE_ANALYTICS_DOC, + } + ) + listing = json.loads( + run_handler(client, ts.cmd_episodes, "--show", SHOW_ID, "--status", "published") + ) + consumed_episode_id = listing["episodes"][0]["id"] + self.assertIsInstance(consumed_episode_id, str) + stats = json.loads( + run_handler(client, ts.cmd_episode_analytics, "--id", consumed_episode_id) + ) + self.assertEqual( + client._request.calls[1]["path"], f"/analytics/episodes/{consumed_episode_id}" + ) + self.assertEqual(stats["episode_id"], consumed_episode_id) + self.assertEqual(stats["downloads_total"], 10) + + def test_authorize_upload_then_attach_then_publish(self): + client = make_client( + { + ("GET", "/episodes/authorize_upload"): AUDIO_UPLOAD_DOC, + } + ) + with ( + patch("transistor_cli.requests.request") as req_mock, + patch("transistor_cli.requests.put") as put_mock, + tempfile.TemporaryDirectory(prefix="ts-upload-") as tmpdir, + ): + fake_audio = pathlib.Path(tmpdir) / "episode1.mp3" + fake_audio.write_bytes(b"ID3") + req_mock.return_value = FakeResponse(200, AUDIO_UPLOAD_DOC) + put_mock.return_value = FakeResponse(200, {}) + authorized = json.loads( + run_handler( + client, + ts.cmd_authorize_upload, + "--filename", + "Episode1.mp3", + "--file", + str(fake_audio), + ) + ) + self.assertEqual(authorized["content_type"], "audio/mpeg") + self.assertIsInstance(authorized["expires_in"], int) + audio_url = authorized["audio_url"] + self.assertIsInstance(audio_url, str) + # The documented flow: attach audio via episode[audio_url], then publish. + client2 = make_client( + { + ("PATCH", f"/episodes/{EPISODE_ID}"): EPISODE_DOC, + ("GET", f"/episodes/{EPISODE_ID}"): EPISODE_DOC, + ("PATCH", f"/episodes/{EPISODE_ID}/publish"): PUBLISH_DOC, + } + ) + json.loads( + run_handler( + client2, ts.cmd_episode_update, "--id", EPISODE_ID, "--audio-url", audio_url + ) + ) + self.assertEqual(client2._request.calls[0]["body"], {"episode[audio_url]": audio_url}) + json.loads(run_handler(client2, ts.cmd_episode_publish, "--id", EPISODE_ID)) + self.assertEqual(client2._request.calls[-1]["path"], f"/episodes/{EPISODE_ID}/publish") + + +class PublishGuardTests(ModuleStateTestCase): + """The publish guard: refuse to publish episodes with no audio attached + (unless --force), since publishing pushes an unplayable item to feeds.""" + + def test_publish_without_audio_dies_with_recipe(self): + client = make_client({("GET", f"/episodes/{EPISODE_ID}"): NO_AUDIO_DOC}) + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr), self.assertRaises(SystemExit) as ctx: + run_handler(client, ts.cmd_episode_publish, "--id", EPISODE_ID) + self.assertEqual(ctx.exception.code, 1) + self.assertIn("no audio yet", stderr.getvalue()) + self.assertIn("episode-update", stderr.getvalue()) + + def test_force_publishes_without_audio(self): + client = make_client( + { + ("GET", f"/episodes/{EPISODE_ID}"): NO_AUDIO_DOC, + ("PATCH", f"/episodes/{EPISODE_ID}/publish"): PUBLISH_DOC, + } + ) + payload = json.loads( + run_handler(client, ts.cmd_episode_publish, "--id", EPISODE_ID, "--force") + ) + self.assertEqual(payload["status"], "published") + + def test_publish_with_audio_skips_guard(self): + client = make_client( + { + ("GET", f"/episodes/{EPISODE_ID}"): EPISODE_DOC, + ("PATCH", f"/episodes/{EPISODE_ID}/publish"): PUBLISH_DOC, + } + ) + payload = json.loads(run_handler(client, ts.cmd_episode_publish, "--id", EPISODE_ID)) + self.assertEqual(payload["status"], "published") + + +class ErrorSignatureTests(unittest.TestCase): + """HTTP error signatures: 401/403/404/429 and JSON:API errors[] bodies.""" + + def run_status(self, status_code, body=None): + client = ts.TransistorClient(key="test-key", dry_run=False) + stderr = io.StringIO() + with patch("transistor_cli.requests.request") as req_mock: + req_mock.return_value = FakeResponse(status_code, body) + with contextlib.redirect_stderr(stderr), self.assertRaises(SystemExit) as ctx: + client.list_shows() + return ctx.exception.code, stderr.getvalue() + + def test_401_names_the_env_var(self): + code, err = self.run_status(401, {"message": "Unauthorized"}) + self.assertEqual(code, 1) + self.assertIn("401", err) + self.assertIn("TRANSISTOR_API_KEY", err) + + def test_403_explains_role_access(self): + _, err = self.run_status(403, {"message": "Forbidden"}) + self.assertIn("403", err) + self.assertIn("role", err) + + def test_404_suggests_id_or_slug(self): + _, err = self.run_status(404, {"message": "Not Found"}) + self.assertIn("404", err) + self.assertIn("slug", err) + + def test_429_states_the_rate_limit_window(self): + _, err = self.run_status(429, {"message": "Too Many Requests"}) + self.assertIn("429", err) + self.assertIn("10 requests per 10 seconds", err) + + def test_errors_envelope_is_flattened(self): + body = { + "errors": [{"title": "Unprocessable Entity", "detail": "Status can't be published"}] + } + _, err = self.run_status(422, body) + self.assertIn("422", err) + self.assertIn("Unprocessable Entity", err) + self.assertIn("Status can't be published", err) + + +class ScriptConventionsTests(unittest.TestCase): + """SCRIPT-GATES-adjacent invariants: imports whitelist, no tech-debt + markers, executable bit, stdlib+requests only.""" + + def test_executable_bit(self): + mode = stat.S_IMODE(os.stat(SCRIPT).st_mode) + self.assertTrue(mode & stat.S_IXUSR, "scripts/transistor must stay executable") + + def test_imports_are_stdlib_plus_requests(self): + text = SCRIPT.read_text() + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("import ") or stripped.startswith("from "): + module = stripped.split()[1].split(".")[0].rstrip(",") + self.assertIn( + module, + { + "argparse", + "json", + "os", + "re", + "sys", + "warnings", + "typing", + "requests", + }, + f"unexpected import: {stripped}", + ) + + def test_no_tech_debt_markers(self): + for i, line in enumerate(SCRIPT.read_text().splitlines(), start=1): + if "#" in line: + comment = line.split("#", 1)[1] + for marker in ("TODO", "FIXME", "HACK", "XXX"): + self.assertNotIn(marker, comment, f"line {i}: {marker} marker") + + +if __name__ == "__main__": + unittest.main() diff --git a/transistor/scripts/transistor b/transistor/scripts/transistor new file mode 100755 index 0000000..f6e4aa3 --- /dev/null +++ b/transistor/scripts/transistor @@ -0,0 +1,757 @@ +#!/usr/bin/env python3 +"""transistor — Transistor.fm podcast hosting from the terminal. + +Manage shows, episodes, private-podcast subscribers, webhooks, and download +analytics on Transistor.fm. Read commands: user, shows, show, episodes, +episode, analytics, episode-analytics, subscribers, webhooks. Write commands: +show-update, episode-create, episode-update, episode-publish, +authorize-upload, subscriber-create, subscriber-batch, subscriber-delete, +webhook-create, webhook-delete. Show creation is dashboard-only (the API +cannot create shows). + +The API is JSON:API on responses (data/attributes/relationships/included); +write bodies are form-encoded with the documented bracket keys +(episode[title]=..., show[title]=..., subscriber[email]=...). Publishing +travels on the dedicated PATCH /v1/episodes/:id/publish endpoint with +episode[status]=draft|scheduled|published. + +API key from the environment variable TRANSISTOR_API_KEY (Account page -> +API Access: https://dashboard.transistor.fm/account). --help and --dry-run +never need it. +""" + +import argparse, json, os, re, sys, warnings +from typing import Any, Dict +warnings.simplefilter("ignore") +import requests + +ENV_KEY = "TRANSISTOR_API_KEY" +API_BASE = "https://api.transistor.fm/v1" +EPISODE_STATUSES = ("draft", "scheduled", "published") +WEBHOOK_EVENTS = ("episode_created", "episode_published", "subscriber_created", "subscriber_deleted") +DATE_RE = re.compile(r"^\d{2}-\d{2}-\d{4}$") + +QUIET = False +GLOBAL_FLAGS: Dict[str, Any] = {"json": False, "dry_run": False, "quiet": False, "verbose": False} + +def log(m): global QUIET; (not QUIET and not GLOBAL_FLAGS.get("json")) and print(m) +def warn(m): print(f"Warning: {m}", file=sys.stderr) +def die(m, c=1): print(f"Error: {m}", file=sys.stderr); sys.exit(c) +def emit(h, d): + if GLOBAL_FLAGS.get("json"): print(json.dumps(d, default=str)) + else: print(h) + +def _preparse(argv): + """Hoist global boolean flags so they work in any position.""" + BOOLS = {"--json","--dry-run","--force","--quiet","--verbose"} + f, fl = {}, [argv[0]] + i = 1 + while i < len(argv): + a = argv[i] + if a in BOOLS: f[a.lstrip("-").replace("-","_")] = True; i += 1 + elif a in ("--help","-h"): return f, argv + elif a == "--": fl.extend(argv[i:]); break + else: fl.append(a); i += 1 + return f, fl + +def _pagination(page, per): + params = {} + if page is not None: params["pagination[page]"] = page + if per is not None: params["pagination[per]"] = per + return params + +class TransistorClient: + def __init__(self, key="", dry_run=False): + self.key = key or os.getenv(ENV_KEY, ""); self.dry_run = dry_run + def _headers(self): + return {"x-api-key": self.key, "Accept": "application/json"} + def _error(self, r): + try: d = r.json() + except ValueError: die(f"Transistor API error ({r.status_code}): {r.text[:200]}") + errs = d.get("errors") if isinstance(d, dict) else None + if isinstance(errs, list) and errs: + parts = [] + for e in errs: + if isinstance(e, dict): + bits = [str(e[k]) for k in ("title", "detail", "code") if e.get(k)] + parts.append(": ".join(bits) if bits else json.dumps(e)) + else: + parts.append(str(e)) + die(f"Transistor API error ({r.status_code}): " + "; ".join(parts)) + if isinstance(d, dict) and d.get("message"): + die(f"Transistor API error ({r.status_code}): {d['message']}") + die(f"Transistor API error ({r.status_code}): {json.dumps(d, default=str)[:200]}") + def _request(self, method, path, params=None, body=None): + url = f"{API_BASE}{path}" + if self.dry_run: + plan = {"dry_run": True, "method": method.upper(), "path": path, "params": params or {}} + if body is not None: plan["body"] = body + return plan + if not self.key: + die(f"{ENV_KEY} not set. Find your API key under API Access on the Account page: " + "https://dashboard.transistor.fm/account") + try: + r = requests.request(method, url, params=params, data=body, headers=self._headers(), timeout=30) + except requests.RequestException as e: + die(f"Cannot reach api.transistor.fm: {e.__class__.__name__}: {e}") + if r.status_code == 401: + die(f"Transistor API rejected the API key (401 Unauthorized). Check {ENV_KEY} at https://dashboard.transistor.fm/account.") + if r.status_code == 403: + die("Transistor API denied access (403 Forbidden). The key is valid but your dashboard role " + "(owner/admin/team member) lacks access to this resource.") + if r.status_code == 404: + die(f"Transistor API resource not found (404): {path}. Verify the id or slug — " + "shows accept id or slug, episode ids come from `transistor episodes --json`.") + if r.status_code == 429: + die("Transistor API rate limit reached (429). The limit is 10 requests per 10 seconds and access " + "blocks for 10 seconds; wait and retry, cache responses, or batch your calls.") + if r.status_code >= 400: + self._error(r) + return r.json() + # ---- reads ---- + def get_user(self): + """Authorization check: GET /v1 returns the authenticated user resource.""" + return self._request("GET", "") + def list_shows(self, private=None, query="", page=None, per=None): + params = _pagination(page, per) + if private is not None: params["private"] = "true" if private else "false" + if query: params["query"] = query + return self._request("GET", "/shows", params) + def get_show(self, show_id): + return self._request("GET", f"/shows/{show_id}") + def list_episodes(self, show_id="", query="", status="", order="", page=None, per=None, include=""): + params = _pagination(page, per) + if show_id: params["show_id"] = show_id + if query: params["query"] = query + if status: params["status"] = status + if order: params["order"] = order + if include: params["include[]"] = include + return self._request("GET", "/episodes", params) + def get_episode(self, episode_id, include=""): + params = {"include[]": include} if include else None + return self._request("GET", f"/episodes/{episode_id}", params) + def show_analytics(self, show_id, start_date="", end_date=""): + params = {} + if start_date: params["start_date"] = start_date + if end_date: params["end_date"] = end_date + return self._request("GET", f"/analytics/{show_id}", params) + def episode_analytics(self, episode_id, start_date="", end_date=""): + params = {} + if start_date: params["start_date"] = start_date + if end_date: params["end_date"] = end_date + return self._request("GET", f"/analytics/episodes/{episode_id}", params) + def list_subscribers(self, show_id, query="", activated=None, page=None, per=None): + params = _pagination(page, per) + params["show_id"] = show_id + if query: params["query"] = query + if activated is not None: params["activated"] = "true" if activated else "false" + return self._request("GET", "/subscribers", params) + def list_webhooks(self, show_id): + return self._request("GET", "/webhooks", {"show_id": show_id}) + # ---- writes: bracket-key form bodies, exactly the documented curl shapes ---- + def update_show(self, show_id, fields): + """PATCH /shows/:id with show[...] keys (id may be a show id or slug).""" + body = {f"show[{k}]": v for k, v in fields.items()} + return self._request("PATCH", f"/shows/{show_id}", body=body) + def create_episode(self, show_id, title, **kw): + body = {"episode[show_id]": show_id} + if title: body["episode[title]"] = title + for k, v in kw.items(): + if v is not None: body[f"episode[{k}]"] = v + return self._request("POST", "/episodes", body=body) + def update_episode(self, episode_id, fields): + """PATCH /episodes/:id updates metadata or attaches audio. It never + changes publishing state; publishing has its own endpoint.""" + body = {f"episode[{k}]": v for k, v in fields.items()} + return self._request("PATCH", f"/episodes/{episode_id}", body=body) + def publish_episode(self, episode_id, status, published_at=""): + """PATCH /episodes/:id/publish — publish, schedule, or revert to draft.""" + body = {"episode[status]": status} + if published_at: body["episode[published_at]"] = published_at + return self._request("PATCH", f"/episodes/{episode_id}/publish", body=body) + def authorize_upload(self, filename): + return self._request("GET", "/episodes/authorize_upload", {"filename": filename}) + def upload_audio(self, upload_url, content_type, path): + with open(path, "rb") as fh: + r = requests.put(upload_url, data=fh, headers={"Content-Type": content_type}, timeout=600) + if r.status_code >= 400: + die(f"Audio upload failed ({r.status_code}): {r.text[:200]}") + return r + def create_subscriber(self, show_id, email, skip_welcome=False): + body = {"show_id": show_id, "email": email, "skip_welcome_email": "true" if skip_welcome else "false"} + return self._request("POST", "/subscribers", body=body) + def create_subscribers_batch(self, show_id, emails, skip_welcome=False): + body = {"show_id": show_id, "emails[]": list(emails), "skip_welcome_email": "true" if skip_welcome else "false"} + return self._request("POST", "/subscribers/batch", body=body) + def delete_subscriber(self, subscriber_id="", show_id="", email=""): + if subscriber_id: + return self._request("DELETE", f"/subscribers/{subscriber_id}") + return self._request("DELETE", "/subscribers", body={"show_id": show_id, "email": email}) + def create_webhook(self, show_id, event_name, url): + body = {"show_id": show_id, "event_name": event_name, "url": url} + return self._request("POST", "/webhooks", body=body) + def delete_webhook(self, webhook_id): + return self._request("DELETE", f"/webhooks/{webhook_id}") + +# ---- JSON:API unwrapping helpers (envelope: data / attributes / included) ---- +def _single(doc): + if not isinstance(doc, dict): return {} + data = doc.get("data", {}) + return data if isinstance(data, dict) else {} +def _items(doc): + if not isinstance(doc, dict): return [] + data = doc.get("data", []) + return data if isinstance(data, list) else [] +def _meta(doc): + if not isinstance(doc, dict): return {} + meta = doc.get("meta", {}) + return meta if isinstance(meta, dict) else {} +def _included_of_type(doc, rtype): + if not isinstance(doc, dict): return [] + inc = doc.get("included", []) + if not isinstance(inc, list): return [] + return [i for i in inc if isinstance(i, dict) and i.get("type") == rtype] +def _attrs(data_obj): + a = data_obj.get("attributes", {}) if isinstance(data_obj, dict) else {} + return a if isinstance(a, dict) else {} +def _sum_downloads(analytics_attrs): + rows = analytics_attrs.get("downloads") + total, days = 0, 0 + if isinstance(rows, list): + for row in rows: + if isinstance(row, dict): + days += 1 + try: total += int(row.get("downloads", 0)) + except (TypeError, ValueError): pass + return total, days + +# ---- single-sourced flag definitions ---- +# Handlers own their flags via these adders; main() builds the parser from the +# same functions and each handler parses its own argv slice with a local +# ArgumentParser (prog=) so tests can call handlers with raw argv lists. +def _add_shows_flags(p): + p.add_argument("--private", action="store_true", help="Only private shows") + p.add_argument("--query", help="Search shows by title") + p.add_argument("--page", type=int, default=None) + p.add_argument("--per", type=int, default=None, help="Page size (API default 10)") + +def _add_episodes_flags(p): + p.add_argument("--show", help="Show ID or slug filter") + p.add_argument("--status", choices=list(EPISODE_STATUSES), default=None) + p.add_argument("--query", help="Search episodes") + p.add_argument("--order", choices=["asc", "desc"], default=None, help="Default: desc (newest first)") + p.add_argument("--page", type=int, default=None) + p.add_argument("--per", type=int, default=None, help="Page size (API default 10)") + p.add_argument("--limit", dest="per", type=int, default=None, + help="Alias for --per (the old CLI's --limit)") + p.add_argument("--include", default=None, help="Compound document, e.g. 'show'") + +def _add_episode_id_flags(p, with_include=False): + p.add_argument("--id", required=True, help="Episode ID") + if with_include: + p.add_argument("--include", default=None, help="e.g. 'show' adds the parent show to included[]") + +def _add_episode_metadata_flags(p, create=False): + p.add_argument("--title", required=create, default=None) + p.add_argument("--season", type=int, default=None) + p.add_argument("--number", type=int, default=None) + p.add_argument("--summary", default=None, help="Short summary") + p.add_argument("--description", default=None, help="Long description (HTML allowed)") + p.add_argument("--audio-url", dest="audio_url", default=None, + help="http(s) URL of finished audio; attaches audio at creation") + p.add_argument("--author", default=None) + if create: + p.add_argument("--type", dest="episode_type", choices=["full", "trailer", "bonus"], default=None) + p.add_argument("--increment-number", dest="increment_number", action="store_true", + help="Auto-set number to the next episode of the current season") + +def _add_analytics_flags(p): + p.add_argument("--start-date", dest="start_date", default=None, help="dd-mm-yyyy (requires --end-date)") + p.add_argument("--end-date", dest="end_date", default=None, help="dd-mm-yyyy (requires --start-date)") + +def _add_subscriber_list_flags(p): + p.add_argument("--query", help="Search subscribers") + p.add_argument("--activated", dest="activated", action="store_true", + help="Only subscribers who activated") + p.add_argument("--page", type=int, default=None) + p.add_argument("--per", type=int, default=None) + p.add_argument("--limit", dest="per", type=int, default=None, + help="Alias for --per (the old CLI's --limit)") + +def cmd_user(client, raw_args): + p = argparse.ArgumentParser(prog="transistor user", description="Who am I? (GET /v1 authorization check)") + args = p.parse_args(raw_args) + if client.dry_run: return emit("[dry-run] GET /v1 (authorization check)", client.get_user()) + d = client.get_user() or {} + data = _single(d) + u = _attrs(data) + emit(f"{u.get('name', '?')} (time zone: {u.get('time_zone', '?')})", + {"id": data.get("id"), "type": data.get("type", "user"), + "name": u.get("name"), "time_zone": u.get("time_zone"), "image_url": u.get("image_url")}) + +def cmd_shows(client, raw_args): + p = argparse.ArgumentParser(prog="transistor shows", description="List your shows (newest-updated first)") + _add_shows_flags(p) + args = p.parse_args(raw_args) + if client.dry_run: return emit("[dry-run] GET /shows", client.list_shows( + private=args.private, query=args.query or "", page=args.page, per=args.per)) + d = client.list_shows(private=args.private, query=args.query or "", page=args.page, per=args.per) or {} + shows = _items(d) + if not shows: return emit("No shows.", {"shows": [], "meta": _meta(d)}) + lines, out = [], [] + for s in shows: + a = _attrs(s) + t = a.get("title", "?") + lines.append(f" {t} slug={a.get('slug', '?')} id={s.get('id', '?')}" + + (" [private]" if a.get("private") else "")) + out.append({"id": s.get("id"), "type": s.get("type", "show"), "title": t, "slug": a.get("slug", ""), + "private": a.get("private"), "show_type": a.get("show_type", ""), + "feed_url": a.get("feed_url", "")}) + emit(f"{len(shows)} show(s):\n" + "\n".join(lines), {"shows": out, "meta": _meta(d)}) + +def cmd_show(client, raw_args): + p = argparse.ArgumentParser(prog="transistor show", description="One show's full attributes") + p.add_argument("--id", required=True, help="Show ID or slug") + args = p.parse_args(raw_args) + if client.dry_run: return emit(f"[dry-run] GET /shows/{args.id}", client.get_show(args.id)) + d = client.get_show(args.id) or {} + data = _single(d) + a = _attrs(data) + emit(f"{a.get('title', '?')} (id={data.get('id', '?')}, slug={a.get('slug', '?')})", + {"id": data.get("id"), "type": data.get("type", "show"), "title": a.get("title"), + "slug": a.get("slug", ""), "description": a.get("description", ""), + "show_type": a.get("show_type", ""), "private": a.get("private"), + "feed_url": a.get("feed_url", ""), "time_zone": a.get("time_zone", ""), + "author": a.get("author", ""), "website": a.get("website", "")}) + +_SHOW_UPDATE_FIELDS = (("--title", "title", None), ("--description", "description", None), + ("--author", "author", None), ("--website", "website", None), + ("--keywords", "keywords", None), ("--copyright", "copyright", None), + ("--owner-email", "owner_email", None), ("--time-zone", "time_zone", None), + ("--show-type", "show_type", ("episodic", "serial"))) + +def cmd_show_update(client, raw_args): + p = argparse.ArgumentParser(prog="transistor show-update", description="Update show metadata (PATCH /shows/:id)") + p.add_argument("--id", required=True, help="Show ID or slug") + for flag, dest, choices in _SHOW_UPDATE_FIELDS: + p.add_argument(flag, dest=dest, choices=choices, default=None) + args = p.parse_args(raw_args) + fields = {} + for flag, dest, choices in _SHOW_UPDATE_FIELDS: + v = getattr(args, dest, None) + if v is not None: fields[dest] = v + if not fields: + die("Nothing to update: pass at least one of --title, --description, --author, --website, " + "--keywords, --copyright, --owner-email, --time-zone, --show-type.") + if client.dry_run: return emit(f"[dry-run] PATCH /shows/{args.id}", client.update_show(args.id, fields)) + d = client.update_show(args.id, fields) or {} + a = _attrs(_single(d)) + emit(f"Updated show {args.id}.", {"id": args.id, "title": a.get("title"), + "updated_fields": sorted(fields.keys())}) + +def cmd_episodes(client, raw_args): + p = argparse.ArgumentParser(prog="transistor episodes", description="List episodes (ordered by publish date)") + _add_episodes_flags(p) + args = p.parse_args(raw_args) + if client.dry_run: return emit("[dry-run] GET /episodes", client.list_episodes( + show_id=args.show or "", query=args.query or "", status=args.status or "", + order=args.order or "", page=args.page, per=args.per, include=args.include or "")) + d = client.list_episodes(show_id=args.show or "", query=args.query or "", status=args.status or "", + order=args.order or "", page=args.page, per=args.per, + include=args.include or "") or {} + eps = _items(d) + if not eps: return emit("No episodes.", {"episodes": [], "meta": _meta(d)}) + lines, out = [], [] + for e in eps: + a = _attrs(e) + eid = e.get("id", "?") + t = a.get("title", "?") + status = a.get("status", "?") + pub = a.get("published_at") or "" + seas = a.get("season", "") + num = a.get("number", "") + lines.append(f" S{seas}E{num} [{status}] {t} (id={eid})") + out.append({"id": eid, "type": e.get("type", "episode"), "title": t, "status": status, + "season": seas, "number": num, "duration": a.get("duration"), + "published_at": pub, "media_url": a.get("media_url", ""), + "share_url": a.get("share_url", ""), + "show_id": ((e.get("relationships", {}) or {}).get("show", {}) or {}).get("data", {}).get("id", "")}) + emit(f"{len(eps)} episode(s):\n" + "\n".join(lines), {"episodes": out, "meta": _meta(d)}) + if args.include and "show" in args.include: + for s in _included_of_type(d, "show"): + log(f" included show: {_attrs(s).get('title', '?')} (id={s.get('id', '?')})") + +def cmd_episode(client, raw_args): + p = argparse.ArgumentParser(prog="transistor episode", description="One episode's full attributes") + _add_episode_id_flags(p, with_include=True) + args = p.parse_args(raw_args) + if client.dry_run: return emit(f"[dry-run] GET /episodes/{args.id}", client.get_episode(args.id, include=args.include or "")) + d = client.get_episode(args.id, include=args.include or "") or {} + data = _single(d) + a = _attrs(data) + media = a.get("media_url") or "" + emit(f"{a.get('title', '?')} [{a.get('status', '?')}] (id={data.get('id', '?')})" + + (f" media: {media}" if media else " (no audio attached yet)"), + {"id": data.get("id"), "type": data.get("type", "episode"), "title": a.get("title"), + "status": a.get("status"), "season": a.get("season"), "number": a.get("number"), + "duration": a.get("duration"), "duration_in_mmss": a.get("duration_in_mmss", ""), + "media_url": media, "share_url": a.get("share_url", ""), + "published_at": a.get("published_at"), "audio_processing": a.get("audio_processing"), + "processing_failure": a.get("processing_failure"), + "show_id": ((data.get("relationships", {}) or {}).get("show", {}) or {}).get("data", {}).get("id", "")}) + if args.include and "show" in args.include: + for s in _included_of_type(d, "show"): + log(f" included show: {_attrs(s).get('title', '?')} (id={s.get('id', '?')})") + +def cmd_episode_create(client, raw_args): + p = argparse.ArgumentParser(prog="transistor episode-create", + description="Create an episode (always created as DRAFT)") + p.add_argument("--show", required=True, help="Show ID or slug to attach the episode to") + _add_episode_metadata_flags(p, create=True) + args = p.parse_args(raw_args) + fields: Dict[str, Any] = {} + if args.season is not None: fields["season"] = args.season + if args.number is not None: fields["number"] = args.number + if args.summary: fields["summary"] = args.summary + if args.description: fields["description"] = args.description + if args.audio_url: fields["audio_url"] = args.audio_url + if args.author: fields["author"] = args.author + if args.episode_type: fields["type"] = args.episode_type + if args.increment_number: fields["increment_number"] = "true" + if client.dry_run: + plan = client.create_episode(args.show, args.title, **fields) + return emit("[dry-run] POST /episodes (creates a DRAFT; publishing is a separate endpoint)", plan) + d = client.create_episode(args.show, args.title, **fields) or {} + data = _single(d) + a = _attrs(data) + eid = data.get("id", "?") + hint = "" if a.get("status") == "published" else f" Publish it with: transistor episode-publish --id {eid}" + emit(f"Created episode '{a.get('title', '?')}' (id={eid}, status={a.get('status', '?')}).{hint}", + {"id": eid, "title": a.get("title"), "status": a.get("status"), + "media_url": a.get("media_url", ""), "published_at": a.get("published_at")}) + +def cmd_episode_update(client, raw_args): + p = argparse.ArgumentParser(prog="transistor episode-update", + description="Update episode metadata or attach audio (never publishes)") + _add_episode_id_flags(p) + _add_episode_metadata_flags(p, create=False) + args = p.parse_args(raw_args) + fields = {} + if args.title is not None: fields["title"] = args.title + if args.summary is not None: fields["summary"] = args.summary + if args.description is not None: fields["description"] = args.description + if args.season is not None: fields["season"] = args.season + if args.number is not None: fields["number"] = args.number + if args.audio_url: fields["audio_url"] = args.audio_url + if args.author is not None: fields["author"] = args.author + if not fields: + die("Nothing to update: pass at least one of --title, --summary, --description, --season, " + "--number, --audio-url, --author. (Publishing state is changed by episode-publish, not here.)") + if client.dry_run: return emit(f"[dry-run] PATCH /episodes/{args.id}", client.update_episode(args.id, fields)) + d = client.update_episode(args.id, fields) or {} + a = _attrs(_single(d)) + emit(f"Updated episode {args.id} ({', '.join(sorted(fields.keys()))}).", + {"id": args.id, "title": a.get("title"), "status": a.get("status"), + "media_url": a.get("media_url", ""), "published_at": a.get("published_at")}) + +def cmd_episode_publish(client, raw_args): + p = argparse.ArgumentParser(prog="transistor episode-publish", + description="Publish, schedule, or unpublish (PATCH /episodes/:id/publish)") + _add_episode_id_flags(p) + p.add_argument("--status", choices=list(EPISODE_STATUSES), default="published", + help="Target state (default: published; scheduled needs --published-at; draft unpublishes)") + p.add_argument("--published-at", dest="published_at", default=None, + help="Publish datetime in the show's time zone (with --status published or scheduled)") + p.add_argument("--force", action="store_true", help="Publish even if no audio is attached") + args = p.parse_args(raw_args) + if not client.dry_run and args.status == "published" and not args.force: + current = client.get_episode(args.id) + a = _attrs(_single(current)) + if not (a.get("media_url") or "").strip(): + die(f"Episode {args.id} has no audio yet (attributes.media_url is empty). Attach audio first: " + f"transistor episode-update --id {args.id} --audio-url (or episode-create --audio-url). " + "Pass --force to publish anyway.") + if client.dry_run: + plan = client.publish_episode(args.id, args.status, args.published_at or "") + return emit(f"[dry-run] PATCH /episodes/{args.id}/publish (episode[status]={args.status})", plan) + d = client.publish_episode(args.id, args.status, args.published_at or "") or {} + data = _single(d) + a = _attrs(data) + verb = {"published": "published", "scheduled": "scheduled", "draft": "reverted to draft"}[args.status] + emit(f"Episode {data.get('id', args.id)} {verb}.", + {"id": data.get("id", args.id), "type": data.get("type", "episode"), + "status": a.get("status"), "published_at": a.get("published_at"), + "media_url": a.get("media_url", "")}) + +def cmd_authorize_upload(client, raw_args): + p = argparse.ArgumentParser(prog="transistor authorize-upload", + description="Authorize a local audio/video upload (max 5GB)") + p.add_argument("--filename", required=True, help="Filename of the audio file") + p.add_argument("--file", default=None, help="Local path to PUT now (otherwise you upload it yourself)") + args = p.parse_args(raw_args) + if client.dry_run: + plan = client.authorize_upload(args.filename) + plan["then_put"] = {"file": args.file or "(none — upload yourself)", + "how": "HTTP PUT the file bytes to attributes.upload_url with " + "Content-Type: attributes.content_type; then attach attributes.audio_url"} + return emit("[dry-run] GET /episodes/authorize_upload", plan) + d = client.authorize_upload(args.filename) or {} + a = _attrs(_single(d)) + if args.file: + client.upload_audio(a.get("upload_url", ""), a.get("content_type", ""), args.file) + log(f"Uploaded {args.file} to the authorized URL.") + emit(f"Authorized upload for {args.filename} (expires in {a.get('expires_in', '?')}s).\n" + f" audio_url to attach: {a.get('audio_url', '?')}", + {"audio_url": a.get("audio_url"), "upload_url": a.get("upload_url"), + "content_type": a.get("content_type"), "expires_in": a.get("expires_in"), + "uploaded": bool(args.file)}) + +def cmd_analytics(client, raw_args): + p = argparse.ArgumentParser(prog="transistor analytics", + description="Show downloads per day (default: last 14 days)") + p.add_argument("--show", required=True, help="Show ID or slug") + _add_analytics_flags(p) + args = p.parse_args(raw_args) + if client.dry_run: return emit(f"[dry-run] GET /analytics/{args.show}", client.show_analytics( + args.show, args.start_date or "", args.end_date or "")) + d = client.show_analytics(args.show, args.start_date or "", args.end_date or "") or {} + a = _attrs(_single(d)) + total, days = _sum_downloads(a) + emit(f"Show {args.show} — {total} downloads over {days} day(s)" + f" ({a.get('start_date', '?')} .. {a.get('end_date', '?')})", + {"show_id": args.show, "downloads_total": total, "days": days, + "start_date": a.get("start_date"), "end_date": a.get("end_date"), + "downloads": a.get("downloads", [])}) + +def cmd_episode_analytics(client, raw_args): + p = argparse.ArgumentParser(prog="transistor episode-analytics", + description="One episode's downloads per day (default: last 14 days)") + p.add_argument("--id", required=True, help="Episode ID or slug") + _add_analytics_flags(p) + args = p.parse_args(raw_args) + if client.dry_run: return emit(f"[dry-run] GET /analytics/episodes/{args.id}", client.episode_analytics( + args.id, args.start_date or "", args.end_date or "")) + d = client.episode_analytics(args.id, args.start_date or "", args.end_date or "") or {} + a = _attrs(_single(d)) + total, days = _sum_downloads(a) + emit(f"Episode {args.id} — {total} downloads over {days} day(s)" + f" ({a.get('start_date', '?')} .. {a.get('end_date', '?')})", + {"episode_id": args.id, "downloads_total": total, "days": days, + "start_date": a.get("start_date"), "end_date": a.get("end_date"), + "downloads": a.get("downloads", [])}) + +def cmd_subscribers(client, raw_args): + p = argparse.ArgumentParser(prog="transistor subscribers", + description="List a private podcast's subscribers") + p.add_argument("--show", required=True, help="Show ID or slug") + _add_subscriber_list_flags(p) + args = p.parse_args(raw_args) + if client.dry_run: return emit(f"[dry-run] GET /subscribers?show_id={args.show}", client.list_subscribers( + args.show, query=args.query or "", activated=args.activated, page=args.page, per=args.per)) + d = client.list_subscribers(args.show, query=args.query or "", activated=args.activated, + page=args.page, per=args.per) or {} + subs = _items(d) + if not subs: return emit("No subscribers.", {"subscribers": [], "meta": _meta(d)}) + lines, out = [], [] + for s in subs: + a = _attrs(s) + email = a.get("email", "?") + lines.append(f" {email} id={s.get('id', '?')}") + out.append({"id": s.get("id"), "type": s.get("type", "subscriber"), "email": email, + "status": a.get("status", ""), "feed_url": a.get("feed_url", ""), + "subscribe_url": a.get("subscribe_url", ""), "has_downloads": a.get("has_downloads")}) + emit(f"{len(subs)} subscriber(s):\n" + "\n".join(lines), {"subscribers": out, "meta": _meta(d)}) + +def cmd_subscriber_create(client, raw_args): + p = argparse.ArgumentParser(prog="transistor subscriber-create", + description="Add one private-podcast subscriber") + p.add_argument("--show", required=True, help="Show ID or slug") + p.add_argument("--email", required=True) + p.add_argument("--skip-welcome-email", dest="skip_welcome_email", action="store_true") + args = p.parse_args(raw_args) + if client.dry_run: + return emit(f"[dry-run] POST /subscribers (add {args.email} to show {args.show})", + client.create_subscriber(args.show, args.email, skip_welcome=args.skip_welcome_email)) + d = client.create_subscriber(args.show, args.email, skip_welcome=args.skip_welcome_email) or {} + data = _single(d) + a = _attrs(data) + emit(f"Subscribed {a.get('email', args.email)} to show {args.show}.", + {"id": data.get("id"), "email": a.get("email"), "status": a.get("status", ""), + "subscribe_url": a.get("subscribe_url", "")}) + +def cmd_subscriber_batch(client, raw_args): + p = argparse.ArgumentParser(prog="transistor subscriber-batch", + description="Add several subscribers (repeat --email)") + p.add_argument("--show", required=True, help="Show ID or slug") + p.add_argument("--email", action="append", required=True, help="Repeat for each address") + p.add_argument("--skip-welcome-email", dest="skip_welcome_email", action="store_true") + args = p.parse_args(raw_args) + if client.dry_run: + return emit(f"[dry-run] POST /subscribers/batch (add {len(args.email)} subscriber(s) to show {args.show})", + client.create_subscribers_batch(args.show, args.email, skip_welcome=args.skip_welcome_email)) + d = client.create_subscribers_batch(args.show, args.email, skip_welcome=args.skip_welcome_email) or {} + out = [{"id": s.get("id"), "email": _attrs(s).get("email")} for s in _items(d)] + emit(f"Added {len(out)} subscriber(s) to show {args.show}.", {"subscribers": out}) + +def cmd_subscriber_delete(client, raw_args): + p = argparse.ArgumentParser(prog="transistor subscriber-delete", + description="Revoke a subscriber's private-feed access") + p.add_argument("--id", default=None, help="Subscriber ID (or use --show + --email)") + p.add_argument("--show", default=None, help="Show ID or slug (with --email)") + p.add_argument("--email", default=None, help="Email address (with --show)") + args = p.parse_args(raw_args) + if not args.id and not (args.show and args.email): + die("Specify either --id, or both --show and --email (delete by email address).") + if client.dry_run: + plan = (client.delete_subscriber(subscriber_id=args.id) if args.id + else client.delete_subscriber(show_id=args.show, email=args.email)) + return emit("[dry-run] DELETE subscriber (revokes private-feed access)", plan) + d = (client.delete_subscriber(subscriber_id=args.id) if args.id + else client.delete_subscriber(show_id=args.show, email=args.email)) or {} + a = _attrs(_single(d)) + emit(f"Revoked private-feed access for {a.get('email', args.email or args.id)}.", + {"id": _single(d).get("id"), "email": a.get("email")}) + +def cmd_webhooks(client, raw_args): + p = argparse.ArgumentParser(prog="transistor webhooks", description="List a show's webhooks") + p.add_argument("--show", required=True, help="Show ID or slug") + args = p.parse_args(raw_args) + if client.dry_run: return emit(f"[dry-run] GET /webhooks?show_id={args.show}", client.list_webhooks(args.show)) + d = client.list_webhooks(args.show) or {} + hooks = _items(d) + if not hooks: return emit("No webhooks.", {"webhooks": []}) + lines, out = [], [] + for w in hooks: + a = _attrs(w) + lines.append(f" {a.get('event_name', '?')} -> {a.get('url', '?')} (id={w.get('id', '?')})") + out.append({"id": w.get("id"), "event_name": a.get("event_name"), "url": a.get("url")}) + emit(f"{len(hooks)} webhook(s):\n" + "\n".join(lines), {"webhooks": out}) + +def cmd_webhook_create(client, raw_args): + p = argparse.ArgumentParser(prog="transistor webhook-create", + description="Subscribe a webhook (max 50 per account)") + p.add_argument("--show", required=True, help="Show ID or slug") + p.add_argument("--event", required=True, choices=list(WEBHOOK_EVENTS)) + p.add_argument("--url", required=True, help="Delivery target URL") + args = p.parse_args(raw_args) + if client.dry_run: + return emit(f"[dry-run] POST /webhooks ({args.event} on show {args.show})", + client.create_webhook(args.show, args.event, args.url)) + d = client.create_webhook(args.show, args.event, args.url) or {} + data = _single(d) + a = _attrs(data) + emit(f"Webhook {data.get('id', '?')} created ({a.get('event_name', args.event)}).", + {"id": data.get("id"), "event_name": a.get("event_name"), "url": a.get("url")}) + +def cmd_webhook_delete(client, raw_args): + p = argparse.ArgumentParser(prog="transistor webhook-delete", description="Unsubscribe a webhook") + p.add_argument("--id", required=True, help="Webhook ID") + args = p.parse_args(raw_args) + if client.dry_run: return emit(f"[dry-run] DELETE /webhooks/{args.id}", client.delete_webhook(args.id)) + client.delete_webhook(args.id) + emit(f"Webhook {args.id} deleted.", {"id": args.id, "deleted": True}) + +def _check_dates(args): + for v in (args.start_date, args.end_date): + if v and not DATE_RE.match(v): + die(f"Invalid date '{v}': analytics dates use dd-mm-yyyy (e.g. 01-09-2026).") + if bool(args.start_date) != bool(args.end_date): + die("start_date and end_date must be used together (or omit both for the default window).") + +def main(): + global GLOBAL_FLAGS, QUIET + GLOBAL_FLAGS, filtered_argv = _preparse(sys.argv) + if GLOBAL_FLAGS.get("quiet"): QUIET = True + parser = argparse.ArgumentParser( + prog="transistor", + description="Transistor.fm podcast hosting from the terminal.", + epilog=f"Set {ENV_KEY} — find your key under API Access on the Account page " + "(https://dashboard.transistor.fm/account). --help and --dry-run never need it. " + "API docs: https://developers.transistor.fm/") + sub = parser.add_subparsers(dest="command") + + sub.add_parser("user", help="Who am I? (GET /v1 authorization check)") + + p_shows = sub.add_parser("shows", help="List your shows (newest-updated first)") + _add_shows_flags(p_shows) + p_show = sub.add_parser("show", help="One show's full attributes") + p_show.add_argument("--id", required=True, help="Show ID or slug") + p_showu = sub.add_parser("show-update", help="Update show metadata (PATCH /shows/:id)") + p_showu.add_argument("--id", required=True, help="Show ID or slug") + for flag, dest, choices in _SHOW_UPDATE_FIELDS: + p_showu.add_argument(flag, dest=dest, choices=choices, default=None) + + p_eps = sub.add_parser("episodes", help="List episodes (ordered by publish date)") + _add_episodes_flags(p_eps) + p_ep = sub.add_parser("episode", help="One episode's full attributes") + _add_episode_id_flags(p_ep, with_include=True) + p_ec = sub.add_parser("episode-create", help="Create an episode (always created as DRAFT)") + p_ec.add_argument("--show", required=True, help="Show ID or slug to attach the episode to") + _add_episode_metadata_flags(p_ec, create=True) + p_eu = sub.add_parser("episode-update", help="Update episode metadata or attach audio (never publishes)") + _add_episode_id_flags(p_eu) + _add_episode_metadata_flags(p_eu, create=False) + p_epb = sub.add_parser("episode-publish", help="Publish, schedule, or unpublish (PATCH /episodes/:id/publish)") + _add_episode_id_flags(p_epb) + p_epb.add_argument("--status", choices=list(EPISODE_STATUSES), default="published", + help="Target state (default: published; scheduled needs --published-at; draft unpublishes)") + p_epb.add_argument("--published-at", dest="published_at", default=None, + help="Publish datetime in the show's time zone (with --status published or scheduled)") + p_epb.add_argument("--force", action="store_true", help="Publish even if no audio is attached") + + p_au = sub.add_parser("authorize-upload", help="Authorize a local audio/video upload (max 5GB)") + p_au.add_argument("--filename", required=True, help="Filename of the audio file") + p_au.add_argument("--file", default=None, help="Local path to PUT now (otherwise you upload it yourself)") + + p_ana = sub.add_parser("analytics", help="Show downloads per day (default: last 14 days)") + p_ana.add_argument("--show", required=True, help="Show ID or slug") + _add_analytics_flags(p_ana) + p_eana = sub.add_parser("episode-analytics", help="One episode's downloads per day (default: last 14 days)") + p_eana.add_argument("--id", required=True, help="Episode ID or slug") + _add_analytics_flags(p_eana) + + p_subs = sub.add_parser("subscribers", help="List a private podcast's subscribers") + p_subs.add_argument("--show", required=True, help="Show ID or slug") + _add_subscriber_list_flags(p_subs) + p_subc = sub.add_parser("subscriber-create", help="Add one private-podcast subscriber") + p_subc.add_argument("--show", required=True, help="Show ID or slug") + p_subc.add_argument("--email", required=True) + p_subc.add_argument("--skip-welcome-email", dest="skip_welcome_email", action="store_true") + p_subb = sub.add_parser("subscriber-batch", help="Add several subscribers (repeat --email)") + p_subb.add_argument("--show", required=True, help="Show ID or slug") + p_subb.add_argument("--email", action="append", required=True, help="Repeat for each address") + p_subb.add_argument("--skip-welcome-email", dest="skip_welcome_email", action="store_true") + p_subd = sub.add_parser("subscriber-delete", help="Revoke a subscriber's private-feed access") + p_subd.add_argument("--id", default=None, help="Subscriber ID (or use --show + --email)") + p_subd.add_argument("--show", default=None, help="Show ID or slug (with --email)") + p_subd.add_argument("--email", default=None, help="Email address (with --show)") + + p_wh = sub.add_parser("webhooks", help="List a show's webhooks") + p_wh.add_argument("--show", required=True, help="Show ID or slug") + p_whc = sub.add_parser("webhook-create", help="Subscribe a webhook (max 50 per account)") + p_whc.add_argument("--show", required=True, help="Show ID or slug") + p_whc.add_argument("--event", required=True, choices=list(WEBHOOK_EVENTS)) + p_whc.add_argument("--url", required=True, help="Delivery target URL") + p_whd = sub.add_parser("webhook-delete", help="Unsubscribe a webhook") + p_whd.add_argument("--id", required=True, help="Webhook ID") + + args = parser.parse_args(filtered_argv[1:]) + if not args.command: parser.print_help(); sys.exit(1) + if getattr(args, "start_date", None) or getattr(args, "end_date", None): + _check_dates(args) + client = TransistorClient(dry_run=GLOBAL_FLAGS.get("dry_run", False)) + handlers = {"user": cmd_user, "shows": cmd_shows, "show": cmd_show, "show-update": cmd_show_update, + "episodes": cmd_episodes, "episode": cmd_episode, "episode-create": cmd_episode_create, + "episode-update": cmd_episode_update, "episode-publish": cmd_episode_publish, + "authorize-upload": cmd_authorize_upload, "analytics": cmd_analytics, + "episode-analytics": cmd_episode_analytics, "subscribers": cmd_subscribers, + "subscriber-create": cmd_subscriber_create, "subscriber-batch": cmd_subscriber_batch, + "subscriber-delete": cmd_subscriber_delete, "webhooks": cmd_webhooks, + "webhook-create": cmd_webhook_create, "webhook-delete": cmd_webhook_delete} + # Hoisted flags were removed from filtered_argv, so the first occurrence of + # the command token is the dispatch point; hand the rest to the handler's + # own parser (it re-declares the same single-sourced flags). + start = filtered_argv.index(args.command) + 1 + handlers[args.command](client, filtered_argv[start:]) + +if __name__ == "__main__": main() diff --git a/transistor/scripts/transistor-cli b/transistor/scripts/transistor-cli deleted file mode 100755 index 5d3d3f3..0000000 --- a/transistor/scripts/transistor-cli +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python3 -"""transistor-cli — Transistor.fm podcast hosting from the terminal. - -Manage shows, episodes, subscribers, and analytics on Transistor.fm. -API key from transistor.fm → Settings → API Keys. -""" - -import argparse, json, os, sys, warnings -from typing import Any, Dict, List, Optional, Tuple -warnings.simplefilter("ignore") -import requests - -ENV_KEY = os.getenv("TRANSISTOR_API_KEY", "") -API_BASE = "https://api.transistor.fm/v1" - -QUIET = False -GLOBAL_FLAGS: Dict[str, Any] = {"json": False, "dry_run": False, "quiet": False, "verbose": False} -def log(m): global QUIET; (not QUIET and not GLOBAL_FLAGS.get("json")) and print(m) -def warn(m): print(f"Warning: {m}", file=sys.stderr) -def die(m, c=1): print(f"Error: {m}", file=sys.stderr); sys.exit(c) -def emit(h, d): - if GLOBAL_FLAGS.get("json"): print(json.dumps(d, default=str)) - else: print(h) - -def _preparse(argv): - BOOLS = {"--json","--dry-run","--force","--quiet","--verbose"} - f, fl = {}, [argv[0]] - i = 1 - while i < len(argv): - a = argv[i] - if a in BOOLS: f[a.lstrip("-").replace("-","_")] = True; i += 1 - elif a in ("--help","-h"): return f, argv - elif a == "--": fl.extend(argv[i:]); break - else: fl.append(a); i += 1 - return f, fl - -class TransistorClient: - def __init__(self, key="", dry_run=False): - self.key = key or ENV_KEY; self.dry_run = dry_run - def _headers(self): - return {"x-api-key": self.key, "Accept": "application/json"} - def _get(self, path, params=None): - url = f"{API_BASE}{path}" - if self.dry_run: return {"data":[], "dry_run":True, "url":url, "params":params} - if not self.key: die("TRANSISTOR_API_KEY not set. Get one from transistor.fm → Settings → API Keys.") - try: - r = requests.get(url, params=params, headers=self._headers(), timeout=30) - except ConnectionError as e: die(f"Cannot connect: {e}") - if r.status_code in (401,403): die(f"Auth failed ({r.status_code}). Check TRANSISTOR_API_KEY.") - if r.status_code >= 400: - try: d = r.json(); die(f"API error: {d.get('errors', d)}") - except: die(f"API error ({r.status_code}): {r.text[:200]}") - return r.json() - def _post(self, path, data=None): - url = f"{API_BASE}{path}" - if self.dry_run: return {"dry_run":True, "url":url, "data":data} - if not self.key: die("TRANSISTOR_API_KEY not set.") - r = requests.post(url, json=data, headers={**self._headers(), "Content-Type":"application/json"}, timeout=30) - if r.status_code >= 400: - try: d = r.json(); die(f"API error: {d.get('errors', d)}") - except: die(f"API error ({r.status_code}): {r.text[:200]}") - return r.json() - def get_user(self): return self._get("/user") - def list_shows(self): return self._get("/shows") - def get_show(self, sid): return self._get(f"/shows/{sid}") - def update_show(self, sid, **kw): return self._get(f"/shows/{sid}", kw) - def list_episodes(self, show_id="", limit=50): - p = {"pagination[limit]": limit} - if show_id: p["filter[show_id]"] = show_id - return self._get("/episodes", p) - def get_episode(self, eid): return self._get(f"/episodes/{eid}") - def create_episode(self, show_id, title, **kw): - data = {"episode": {"show_id": show_id, "title": title, **kw}} - return self._post("/episodes", data) - def show_analytics(self, show_id=""): - p = {} - if show_id: p["filter[show_id]"] = show_id - return self._get("/analytics/show", p) - def list_subscribers(self, show_id="", page=1): - p = {"page": page} - if show_id: p["filter[show_id]"] = show_id - return self._get("/subscribers", p) - -def cmd_user(client, args): - if client.dry_run: return emit("[dry-run] Get user", {"dry_run":True}) - d = client.get_user() or {} - u = d.get("data",{}).get("attributes",{}) - emit(f"👤 {u.get('email','?')} ({u.get('time_zone','?')})", - {"email":u.get("email"),"timezone":u.get("time_zone")}) - -def cmd_shows(client, args): - if client.dry_run: return emit("[dry-run] List shows", {"dry_run":True}) - data = client.list_shows() or {} - shows = data.get("data",[]) - if not shows: return emit("No shows.", {"shows":[]}) - lines, out = [], [] - for s in shows: - a = s.get("attributes",{}); sid = s.get("id","?") - t = a.get("title","?"); es = a.get("episodes_count",0); subs = a.get("subscribers_count",0) - lines.append(f" {t:40} {es} episodes {subs} subscribers id={sid}") - out.append({"id":sid,"title":t,"episodes":es,"subscribers":subs}) - emit(f"{len(shows)} show(s):\n"+"\n".join(lines), {"shows":out}) - -def cmd_episodes(client, args): - p = argparse.ArgumentParser(prog="transistor episodes") - p.add_argument("--show", help="Show ID filter") - p.add_argument("--limit", type=int, default=20) - parsed, _ = p.parse_known_args(args) - if client.dry_run: return emit("[dry-run] List episodes", {"dry_run":True}) - data = client.list_episodes(show_id=parsed.show or "", limit=parsed.limit) or {} - eps = data.get("data",[]) - if not eps: return emit("No episodes.", {"episodes":[]}) - lines, out = [], [] - for e in eps: - a = e.get("attributes",{}); eid = e.get("id","?") - t = a.get("title","?"); status = a.get("status","?"); dur = a.get("duration","?") - pub = (a.get("published_at") or "")[:10] - seas = a.get("season",0); epnum = a.get("number",0) - lines.append(f" S{seas:02d}E{epnum:02d} {t:45} [{status}] {dur}min pub {pub} id={eid}") - out.append({"id":eid,"title":t,"status":status,"duration":dur,"published":pub,"season":seas,"number":epnum}) - emit(f"{len(eps)} episode(s):\n"+"\n".join(lines), {"episodes":out}) - -def cmd_analytics(client, args): - p = argparse.ArgumentParser(prog="transistor analytics") - p.add_argument("--show", help="Show ID filter") - parsed, _ = p.parse_known_args(args) - if client.dry_run: return emit("[dry-run] Get analytics", {"dry_run":True}) - data = client.show_analytics(show_id=parsed.show or "") or {} - d = data.get("data",{}) - attrs = d.get("attributes",{}) if isinstance(d, dict) else {} - totals = attrs.get("totals",{}) - downloads = totals.get("downloads",{}).get("total","?") - plays = totals.get("plays",{}).get("total","?") if "plays" in totals else "N/A" - emit(f"📊 Downloads: {downloads} Plays: {plays}", - {"downloads":downloads,"plays":plays,"totals":totals}) - -def main(): - global GLOBAL_FLAGS, QUIET - GLOBAL_FLAGS, filtered_argv = _preparse(sys.argv) - if GLOBAL_FLAGS.get("quiet"): QUIET = True - if GLOBAL_FLAGS.get("json"): warnings.simplefilter("ignore") - parser = argparse.ArgumentParser(prog="transistor", description="Transistor.fm podcast hosting.", - epilog="Set TRANSISTOR_API_KEY. Get one from transistor.fm → Settings → API Keys.") - sub = parser.add_subparsers(dest="command") - sub.add_parser("user") - sub.add_parser("shows") - p_eps = sub.add_parser("episodes", help="List episodes") - p_eps.add_argument("--show") - p_eps.add_argument("--limit", type=int, default=20) - sub.add_parser("analytics", help="Show analytics").add_argument("--show") - args = parser.parse_args(filtered_argv[1:]) - if not args.command: parser.print_help(); sys.exit(1) - client = TransistorClient(dry_run=GLOBAL_FLAGS.get("dry_run",False)) - handlers = {"user":cmd_user,"shows":cmd_shows,"episodes":cmd_episodes,"analytics":cmd_analytics} - h = handlers.get(args.command) - if not h: parser.print_help(); sys.exit(1) - r = filtered_argv[filtered_argv.index(args.command)+1:] - h(client, r) - -if __name__ == "__main__": main() From a31381bd37ea449aaccad23ce1f5844d5adfec5d Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sat, 29 Aug 2026 23:03:15 -0400 Subject: [PATCH 37/40] docs(tempest): thicken weather station skill against current API research Full skill-builder rebuild of tempest per issue #407: - references/: four dense files replacing the single layouts crib sheet - rest-api-and-auth.md (personal-use token via tempestwx.com Settings -> Data Authorizations, token-as-query-parameter auth, StationSet wrapper, device_type HB/AR/SK/ST enum, observation parameters day_offset vs time_start/time_end, better_forecast unit-selection, error signatures), udp-broadcast-protocol.md (port 50222 listen-only broadcast, dispatch- by-type rule, obs_st 18-position UDP record, rapid_wind ob, evt_precip/ evt_strike, hub_status/device_status named fields), observation-layouts- and-units.md (REST 22-position obs_st vs UDP 18, obs_air 8, obs_sky 17 vs 14, daily obs_*_ext summaries, metric-native unit tables), and cli- worked-recipes.md (six executable pipelines). Every file ends with a Sources footer citing live-verified official docs (apidocs.tempestwx.com, weatherflow.github.io/Tempest). - scripts/tempest: fixed researched bugs - rapid_wind handler iterated the single ob array element-wise (TypeError on real datagrams), hub_status printed undocumented freq field, forecast human display double-converted Fahrenheit stations (units_temp=f is documented and honored), SK/AR device types now matched alongside SKY/AIR, StationSet unwrap handles stations/locations/bare-list shapes, missing ~/.tempest.env fallback implemented as documented, dry-run stations plan, handler-owns-flags dispatch. Added decode_message()/handle_datagram() type-dispatch layer covering all seven UDP message families. - scripts/test_tempest.py: 42 offline tests (pytest + unittest green, proxy-trap clean) - canned UDP datagram bytes fed to the decoder with no sockets, mocked REST transport, help/arg-error/dry-run classes, and the documented pipelines (stations->current, obs day totals, forecast units). - SKILL.md: lastfm-model rewrite (275 lines) - Setup, intent-grouped commands, UDP family dispatch table, pipeline recipes, jq guidance, ten grounded gotchas, when-to-use/when-not-to-use boundaries, reference routing table. - README.md: human-format refresh with hub-on-LAN prerequisite. - evals/evals.json: 8 schema-v1 cases incl. two negative probes (Shakespeare The Tempest, generic city forecast). - Root README blurb and references/skill-triggers.md row synced to the new description; .claude-plugin/marketplace.json and llms.txt regenerated (both embed descriptions; check modes exit 0; codex artifact unaffected). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/marketplace.json | 2 +- README.md | 2 +- llms.txt | 2 +- references/skill-triggers.md | 2 +- tempest/README.md | 48 +- tempest/SKILL.md | 327 ++++++--- tempest/evals/evals.json | 92 +++ tempest/references/cli-worked-recipes.md | 160 +++++ .../observation-layouts-and-units.md | 175 +++++ tempest/references/rest-api-and-auth.md | 251 +++++++ .../references/tempest-api-field-layouts.md | 128 ---- tempest/references/udp-broadcast-protocol.md | 294 ++++++++ tempest/scripts/tempest | 591 ++++++++++------ tempest/scripts/test_tempest.py | 646 ++++++++++++++++++ 14 files changed, 2244 insertions(+), 476 deletions(-) create mode 100644 tempest/evals/evals.json create mode 100644 tempest/references/cli-worked-recipes.md create mode 100644 tempest/references/observation-layouts-and-units.md create mode 100644 tempest/references/rest-api-and-auth.md delete mode 100644 tempest/references/tempest-api-field-layouts.md create mode 100644 tempest/references/udp-broadcast-protocol.md create mode 100644 tempest/scripts/test_tempest.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 24e8eab..88f323b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1245,7 +1245,7 @@ "./tempest" ], "strict": false, - "description": "Query hyper-local weather from a WeatherFlow Tempest station: current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. Use when the user asks about the weather, temperature, rain, wind, humidity, forecast, or wants conditions from their own station rather than a generic weather service." + "description": "Query hyper-local weather from a WeatherFlow Tempest station over its REST API and the hub's local UDP broadcast: current conditions, forecast, historical observations, and real-time decoded datagrams (obs_st, rapid_wind, evt_precip, evt_strike, hub_status). Use when the user asks about weather, temperature, rain, wind, humidity, or forecast data from their own Tempest/WeatherFlow station, or wants to parse the hub's UDP port 50222 broadcast. Do not use this skill for generic or city forecasts without a Tempest station (public weather services serve those), for Shakespeare's play The Tempest or other literature questions, or for weather hardware from other vendors - the REST endpoints require a personal-use token and the UDP broadcast only exists on a Tempest hub's LAN." }, { "name": "terraform", diff --git a/README.md b/README.md index 622c382..d5ba857 100644 --- a/README.md +++ b/README.md @@ -565,7 +565,7 @@ Operate the Prometheus + OpenTelemetry + Loki observability stack as one unit: s ### [tempest](tempest/SKILL.md) -Hyper-local weather from a WeatherFlow Tempest station. Query current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. A complete reference implementation of the cli-builder patterns in a working, testable project — including the CLI binary and full API field layout reference. +Hyper-local weather from a WeatherFlow Tempest station over the REST API and the hub's local UDP broadcast: current conditions, forecast, historical observations, and real-time decoded datagrams (obs_st, rapid_wind, evt_precip, evt_strike, hub_status) with metric-native values and conversion guidance. Not for generic city forecasts without a station, or weather hardware from other vendors. ### [terraform](terraform/SKILL.md) diff --git a/llms.txt b/llms.txt index 342cc38..58e1615 100644 --- a/llms.txt +++ b/llms.txt @@ -139,7 +139,7 @@ - [technical-documentation](technical-documentation/SKILL.md): Create and review technical documentation, including READMEs, agent-facing instructions, API references, and CLI help. Use when documentation must help someone complete real work. Do not use for marketing copy, brand messaging, or long-form editorial content. - [technology-radar](technology-radar/SKILL.md): Build and maintain technology radars for adoption, trial, assessment, and hold decisions, and choose proportionate architecture-governance paths for technology portfolios. Use when governing technology choices, build-versus-buy decisions, architecture standards, exceptions, or engineering portfolio risk. Do not use for enterprise capability or target-state architecture, writing ADRs, implementing systems, security engineering, or operational incident/runbook work. - [telemetry](telemetry/SKILL.md): Operate the observability stack that deploys as one unit: Prometheus scrape configuration, recording and alerting rules, relabeling, retention, and high availability; OpenTelemetry Collector pipelines (receivers, processors, exporters, sampling, trace/span correlation); and Loki ingest, LogQL, retention, and label design — with a bundled read-only telemetry-check script for Prometheus rule sanity and scrape-target reachability. Use when running, tuning, or troubleshooting a Prometheus, OpenTelemetry Collector, or Loki deployment, or reviewing the collection/ingest/retention layer. Do not use for observability strategy, SLI/SLO design, or paging policy (that is platform-engineering) or Grafana dashboards, panels, and Grafana-side alerting (that is grafana). -- [tempest](tempest/SKILL.md): Query hyper-local weather from a WeatherFlow Tempest station: current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. Use when the user asks about the weather, temperature, rain, wind, humidity, forecast, or wants conditions from their own station rather than a generic weather service. +- [tempest](tempest/SKILL.md): Query hyper-local weather from a WeatherFlow Tempest station over its REST API and the hub's local UDP broadcast: current conditions, forecast, historical observations, and real-time decoded datagrams (obs_st, rapid_wind, evt_precip, evt_strike, hub_status). Use when the user asks about weather, temperature, rain, wind, humidity, or forecast data from their own Tempest/WeatherFlow station, or wants to parse the hub's UDP port 50222 broadcast. Do not use this skill for generic or city forecasts without a Tempest station (public weather services serve those), for Shakespeare's play The Tempest or other literature questions, or for weather hardware from other vendors - the REST endpoints require a personal-use token and the UDP broadcast only exists on a Tempest hub's LAN. - [terraform](terraform/SKILL.md): Operate Terraform and OpenTofu across the whole infrastructure lifecycle: module structure, state backends and locking, plan/apply workflow, drift detection, remote state, upgrade and refactor flows, and evidence-based diagnostics. Use when running or inspecting terraform plans, applies, state files, imports, or state surgery, or when the bundled tfops script should handle the task. Do not use for IaC methodology or cloud design decisions - those route up to platform-engineering. - [three](three/SKILL.md): Build browser-based Three.js and WebGL scenes, animations, and interactive 3D visualizations with a small vanilla JavaScript starting point. - [tmdb](tmdb/SKILL.md): Query TMDb metadata for films and television, then enrich results with details, credits, providers, and external IDs. Do not use this skill for torrent search, streaming playback, or personal watch-history tracking. diff --git a/references/skill-triggers.md b/references/skill-triggers.md index 80fe1f7..0d9426c 100644 --- a/references/skill-triggers.md +++ b/references/skill-triggers.md @@ -23,7 +23,7 @@ Each skill's `description` field is the canonical routing contract. This conveni | "Jira", "Atlassian Jira", "JQL", "ticket PROJ-123", "sprint work", "JIRA_API_TOKEN" | [jira](../jira/SKILL.md) | | "Open Library", "openlibrary", "book search", "ISBN lookup", "author records", "work details", "book editions", "book ratings", "cover image" | [openlibrary](../openlibrary/SKILL.md) | | "PeerTube", "peertube", "federated video", "SepiaSearch", "decentralized video platform", "PEERTUBE_SERVER", "my PeerTube instance" | [peertube](../peertube/SKILL.md) | -| "weather", "forecast", "temperature", "is it raining", "Tempest" | [tempest](../tempest/SKILL.md) | +| "weather", "forecast", "temperature", "is it raining", "Tempest", "WeatherFlow", "my station" | [tempest](../tempest/SKILL.md) | | "TMDb", "The Movie Database", "movie search", "trending movies", "upcoming TV releases", "TMDB_ACCESS_TOKEN" | [tmdb](../tmdb/SKILL.md) | | "traefik", "reverse proxy", "load balancer", "API gateway", "Let's Encrypt", "ACME", "Docker routing", "traefik.yml", "entry point", "middleware", "TLS termination", "forward auth", "rate limit" | [traefik](../traefik/SKILL.md) | | "reverse-engineer", "understand this codebase", "PRD from code", "architecture document", "architecture health", "coupling analysis", "modularity", "decomposition readiness", "data ownership map", "distributed workflow analysis", "reconciliation path" | [software-architecture-analysis](../software-architecture-analysis/SKILL.md) | diff --git a/tempest/README.md b/tempest/README.md index bb72a4e..3aedf9d 100644 --- a/tempest/README.md +++ b/tempest/README.md @@ -1,37 +1,51 @@ -# Tempest — Hyper-Local Weather from Your Station +# Tempest — Hyper-Local Weather from Your Own Station -Query live weather data from a WeatherFlow Tempest station. Current conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. +Query live weather from a WeatherFlow Tempest station: current conditions, 7-day forecast, historical observations, and real-time broadcasts from your hub's local network — with every positional sensor array and UDP message family decoded for you. ## Why Install This Skill -When your agent loads this skill, it can **check hyper-local weather from your own station** — more accurate than generic services. That means: +Generic weather services tell you what the model thinks the sky is doing kilometers away. This skill reads **your actual station**: the Tempest sitting in your yard, via WeatherFlow's documented REST API and the hub's local UDP broadcast. Once installed, your agent can: -- **Current conditions** — temperature, humidity, wind, rain, UV, solar radiation, barometric pressure -- **7-day forecast** — daily and hourly outlook with precipitation probability -- **Historical data** — past observations for analysis -- **Real-time UDP** — local broadcast reception without cloud dependency -- **Auto-discovery** — finds your station and sensors automatically +- **Current conditions** — temperature, humidity, wind (lull/avg/gust + direction), rain, UV, solar radiation, barometric pressure +- **7-day forecast** — daily and hourly outlook with precipitation probabilities, unit-aware +- **Historical observations** — past UTC days of minute-level data for analysis +- **Real-time UDP stream** — decoded `obs_st`, `rapid_wind`, `evt_precip`, `evt_strike`, and `hub_status` messages straight from the hub on port 50222, no cloud round-trip +- **Station discovery** — finds your stations and sensors automatically, never mistaking the hub for a sensor + +The tricky parts of the Tempest API are handled for you: observations arrive as *positional arrays* whose meaning depends on the index, UDP message families have three different payload shapes, and forecast responses are unit-selectable (a naive script double-converts Fahrenheit data into 172-degree nonsense). The CLI decodes all of it, keeps JSON output in metric-native wire units, and converts only for human display. ## What You Get -| Directory | Purpose | -|-----------|---------| -| `SKILL.md` | Complete command reference with examples | -| `scripts/tempest` | CLI tool for WeatherFlow Tempest API | -| `references/` | API field layout reference | +| Path | What it provides | +|------|------------------| +| `SKILL.md` | Command reference, pipeline recipes, and the gotchas that actually bite | +| `scripts/tempest` | CLI: `stations`, `current`, `obs`, `forecast`, `udp listen` with `--json`/`--dry-run` | +| `scripts/test_tempest.py` | Offline test suite (canned datagram bytes + mocked REST, no network) | +| `references/rest-api-and-auth.md` | Token auth, endpoint catalog, response shapes, error signatures | +| `references/udp-broadcast-protocol.md` | Port 50222 transport, every message family's exact layout | +| `references/observation-layouts-and-units.md` | Index-by-index field maps for obs_st/obs_air/obs_sky + conversion tables | +| `references/cli-worked-recipes.md` | Copy-paste multi-step recipes with jq stages | ## Quick Start ```bash +# Create a token in the Tempest web app: Settings -> Data Authorizations -> Create Token export TEMPEST_TOKEN="your-token-here" -tempest current -tempest forecast + +tempest stations # discover your station and device IDs +tempest current # conditions right now +tempest forecast # current + daily + hourly outlook +tempest udp listen --timeout 30 # real-time broadcast from the hub (no token needed) ``` +Every command accepts `--json` for machine-readable output and `--dry-run` to preview the plan offline. + ## Triggers -Load this for weather, temperature, rain, wind, humidity, forecast, or conditions from a specific Tempest station. +Load this skill when the user mentions Tempest, WeatherFlow, their weather station, hyper-local conditions, station observations, or parsing the hub's UDP port 50222 broadcast — temperature, rain, wind, humidity, lightning, or forecast questions tied to a personal station. ## Requirements -Python 3.8+ with `requests` library. Free token from weatherflow.com. +- Python 3.8+ with the `requests` library (the only dependency) +- A `TEMPEST_TOKEN` (free, personal use) for REST commands — created in the Tempest web app under Settings → Data Authorizations +- **For UDP listening: a Tempest hub on the same LAN** — the hub broadcasts on UDP port 50222 to the local network only; broadcasts do not cross routers, and no token or cloud account is involved. REST commands work from anywhere with internet access. diff --git a/tempest/SKILL.md b/tempest/SKILL.md index a361efd..82ac057 100644 --- a/tempest/SKILL.md +++ b/tempest/SKILL.md @@ -1,161 +1,276 @@ --- name: tempest -description: 'Query hyper-local weather from a WeatherFlow Tempest station: current - conditions, 7-day forecast, historical observations, and real-time UDP broadcasts. - Use when the user asks about the weather, temperature, rain, wind, humidity, forecast, - or wants conditions from their own station rather than a generic weather service.' +description: >- + Query hyper-local weather from a WeatherFlow Tempest station over its REST + API and the hub's local UDP broadcast: current conditions, forecast, + historical observations, and real-time decoded datagrams (obs_st, + rapid_wind, evt_precip, evt_strike, hub_status). Use when the user asks + about weather, temperature, rain, wind, humidity, or forecast data from + their own Tempest/WeatherFlow station, or wants to parse the hub's UDP port + 50222 broadcast. Do not use this skill for generic or city forecasts + without a Tempest station (public weather services serve those), for + Shakespeare's play The Tempest or other literature questions, or for + weather hardware from other vendors - the REST endpoints require a + personal-use token and the UDP broadcast only exists on a Tempest hub's + LAN. license: MIT -compatibility: Requires TEMPEST_TOKEN env var (free from weatherflow.com), Python - 3.8+, and the `requests` library. +compatibility: >- + Requires TEMPEST_TOKEN env var for REST (create it in the Tempest web app + under Settings -> Data Authorizations), Python 3.8+, and `requests`. UDP + listening needs a Tempest hub on the LAN and no token. `--help` and + `--dry-run` work without credentials. metadata: - tags: weather, tempest, forecast, weatherflow, station, hyper-local - sources: https://weatherflow.com, https://swd.weatherflow.com/swd/rest + tags: weather, tempest, weatherflow, forecast, station, udp, hyper-local + sources: https://apidocs.tempestwx.com/reference/quick-start, https://weatherflow.github.io/Tempest/api/udp/v171/ --- -# tempest — Hyper-Local Weather from Your Tempest Station +# tempest — Hyper-local weather from your Tempest station -Query live weather data from a WeatherFlow Tempest station. Supports REST API access to current conditions, forecasts, and history via the cloud, plus local UDP broadcast reception from your hub on the same LAN. +Drive a WeatherFlow Tempest station from the terminal. Two transports, both +first-class: the documented REST API (`swd.weatherflow.com/swd/rest`, +personal-use token) for conditions, forecast, and history — officially the +primary data source — and the hub's unauthenticated UDP broadcast on port +50222 for real-time, lowest-latency readings on your LAN. The bundled CLI +decodes the positional observation arrays and every UDP message family, keeps +`--json` output metric-native, and converts units only for human display. ## Setup -1. Get a personal access token at [weatherflow.com](https://weatherflow.com) (Account → API Tokens) -2. Set it in your environment: +1. Create a personal access token: sign in to the Tempest web app + (tempestwx.com), then **Settings → Data Authorizations → Create Token**. + (This is the documented non-graphical auth method; OAuth exists for web + apps but is not what a CLI uses.) +2. Export it: ```bash -export TEMPEST_TOKEN="your-token-here" +export TEMPEST_TOKEN="" ``` -The CLI reads `TEMPEST_TOKEN` from the environment. It also falls back to reading `~/.tempest.env` if the env var is not set (for agent subprocesses that don't inherit env vars). `--help` and `--dry-run` work without a token. +The token travels to the API as a **query parameter** (`?token=...`) per the +official docs — the CLI handles this. If the env var is not set, the CLI +falls back to reading `TEMPEST_TOKEN=` from `~/.tempest.env` (handy for agent +subprocesses that skip shell profiles). `--help` and `--dry-run` never need a +token. UDP listening never needs one either — the hub broadcast is +unauthenticated and LAN-only. ## Essential Commands -### current — Current conditions +### stations — discover your stations and devices ```bash -tempest current # human-readable -tempest current --station-id 12345 --device-id 67890 # specific hardware -tempest current --json # machine-readable +tempest stations # names, station ids, device types, serials +tempest stations --json | jq '.stations[] | {station_id, name, + devices: [.devices[] | {device_id, device_type, serial_number}]}' ``` -If you have one station, it auto-selects it and picks the best sensor (ST > SKY > AIR, skips the HB hub). Pass `--station-id` or `--device-id` to override. +Every station response nests a `devices` array: `device_type` is `ST` (the +Tempest all-in-one), `AR`/`AIR`, `SK`/`SKY`, or `HB` (the hub — it has **no** +observations; always filter it out before querying observations). Run this +first when you don't know your ids. -### forecast — Multi-day forecast (+ current conditions + hourly) +### current — latest conditions ```bash -tempest forecast # current + 5-day daily + 12-hour hourly -tempest forecast --days 3 # fewer days -tempest forecast --station-id 12345 # specific station -tempest forecast --json # machine-readable +tempest current # human-readable, converted +tempest current --json # metric-native, jq-ready +tempest current --station-id 12799 --device-id 60526 # pin exact hardware ``` -### stations — List your stations and devices +With one station it auto-selects and picks the best sensor (`ST`, then +`SKY`/`SK`, then `AIR`/`AR`, skipping `HB`). Output `.observation` carries the +decoded positional array as named fields with `_unit` companions. + +### forecast — current conditions + daily + hourly ```bash -tempest stations # shows station names, IDs, device types, serials -tempest stations --json # full device inventory +tempest forecast # current + 5-day daily + next 12 hours +tempest forecast --days 7 --json +tempest forecast --station-id 12799 --days 3 ``` -Use this first if you don't know your station ID or want to see what sensors are online. +The `better_forecast` response nests daily/hourly under a `forecast` wrapper +key, and it is unit-selectable (`units_temp=c|f` and friends, default metric) +— the CLI reads the response's `units` before converting anything. -### obs — Historical observations +### obs — historical observations ```bash -tempest obs --device-id 67890 --days 1 # last 24 hours -tempest obs --device-id 67890 --days 7 # last week -tempest obs --device-id 67890 --json # machine-readable +tempest obs --device-id 60526 --days 1 # last UTC day (day_offset) +tempest obs --device-id 60526 --days 7 +tempest obs --device-id 60526 --json ``` -### udp listen — Real-time broadcasts from the hub +`--days N` maps to the API's `day_offset` (whole UTC days). The underlying +endpoint also accepts `time_start`/`time_end` epoch ranges (one-minute +resolution guaranteed up to 5 days) — use raw calls for those; see +references/rest-api-and-auth.md. + +## UDP broadcasts from your hub (port 50222, listen-only) ```bash -tempest udp listen # listen indefinitely (Ctrl-C to stop) +tempest udp listen # live stream until Ctrl-C tempest udp listen --timeout 30 # auto-stop after 30s -tempest udp listen --show-all # include hub_status messages +tempest udp listen --timeout 60 --json # one JSON object per datagram +tempest udp listen --show-all # include hub_status/device_status ``` -Requires being on the same LAN as the hub (port 50222 UDP broadcast). Receives observations, rapid wind updates, lightning strike events, and precipitation start events in real time. +Requires being on the same LAN as the hub (routed connectivity is not enough +— broadcasts don't cross routers). No token involved. The listener decodes +every message family, dispatching on `type` before touching array positions: -## Data Reference +| Family | Payload shape | Decoded fields | +|---|---|---| +| `obs_st` / `obs_air` / `obs_sky` | list of report rows under `obs` | named observation fields | +| `rapid_wind` | ONE 3-element array under `ob` | wind_speed_mps, wind_direction | +| `evt_precip` | ONE array under `evt` | timestamp (rain started) | +| `evt_strike` | ONE array under `evt` | distance_km, energy | +| `hub_status`, `device_status` | named fields, no array | uptime, rssi, seq, voltage, sensor_status | -### obs_st field layout (Tempest all-in-one) +## Multi-step pipeline recipes -Observations from the Tempest sensor arrive as positional arrays. The CLI decodes them, but if you're reading raw JSON output, this map tells you what each index means: +### Discover, then observe -| Index | Field | Units | Notes | -|-------|-------|-------|-------| -| 0 | epoch | seconds UTC | | -| 1 | wind_lull | m/s | Minimum 3-second sample | -| 2 | wind_avg | m/s | Average over report interval | -| 3 | wind_gust | m/s | Maximum 3-second sample | -| 4 | wind_direction | degrees | 0=N | -| 5 | wind_sample_interval | seconds | | -| 6 | station_pressure | MB | | -| 7 | air_temperature | C | CLI converts to °F | -| 8 | relative_humidity | % | | -| 9 | illuminance | lux | | -| 10 | uv | index | | -| 11 | solar_radiation | W/m² | | -| 12 | rain_accumulation | mm | Over last interval | -| 13 | precipitation_type | enum | 0=none 1=rain 2=hail | -| 14 | avg_strike_distance | km | Lightning | -| 15 | strike_count | count | Lightning | -| 16 | battery | volts | ~2.6V normal, ~2.5V low | -| 17 | report_interval | minutes | | -| 18 | local_day_rain_accumulation | mm | | +```bash +# Stage 1 -> stage 2: stations --json emits integer ids that current consumes +tempest stations --json | jq -r '.stations[].devices[] + | select(.device_type == "ST") | .device_id' | head -1 +tempest current --device-id --json +``` -See [references/tempest-api-field-layouts.md](references/tempest-api-field-layouts.md) for the full obs_air and obs_sky field layouts. +### Rain watch: yesterday's total, then live rain events -### Unit conversions the CLI applies +```bash +tempest obs --device-id 60526 --days 1 --json \ + | jq '{samples: (.observations | length), + day_rain_mm: .observations[-1].local_day_rain_accumulation}' +tempest udp listen --timeout 600 --json | jq 'select(.type == "evt_precip")' +``` -| Input | Output | Conversion | -|-------|--------|------------| -| °C | °F | `c * 9/5 + 32` | -| m/s | mph | `mps * 2.237` | -| MB | inHg | `mb * 0.02953` | -| mm | in | `mm / 25.4` | -| degrees | cardinal | N, NNE, NE, ..., NNW | +`obs --json` ends with decoded observations carrying +`local_day_rain_accumulation` (mm, number); `evt_precip` datagrams decode to +`{type, serial_number, timestamp}` — both stages emit typed fields the next +stage can consume. + +### Unit-aware forecast slice + +```bash +tempest forecast --days 7 --json \ + | jq '{units_temp: .forecast.units.units_temp, + highs_f: [.forecast.forecast.daily[] | .air_temp_high * 9 / 5 + 32], + rain_hours: [.forecast.forecast.hourly[] + | select(.precip_probability > 30) | .local_hour]}' +``` + +The jq math here is safe **only because** it checks `units_temp` first — see +gotcha 2. + +## JSON output and jq processing + +`--json` output is **metric-native** — the raw wire units (m/s wind, mm rain, +°C temperature, MB pressure) with `_unit` companion fields naming each. +Convert at the consumption edge: + +```bash +tempest current --json | jq '{temp_c: .observation.air_temperature, + temp_f: (.observation.air_temperature * 9 / 5 + 32), + wind_mph: (.observation.wind_avg * 2.237), + rain_in: (.observation.rain_accumulation / 25.4)}' +``` + +Global flags work in any position: `tempest --json current --device-id 60526` +and `tempest current --device-id 60526 --json` are identical. `--quiet` +silences the progress logs (data on stdout, logs on stderr). +`--dry-run` prints a plan object and exits 0 without touching the network. ## Known Gotchas -### The API is metric-native +1. **Observations are positional arrays, not objects.** Raw `obs` rows have + no field names; meaning comes from the index (obs_st: 0 epoch, 2 wind avg + m/s, 4 wind direction, 6 pressure MB, 7 temperature °C, 12 rain mm, 16 + battery V, 17 report interval). Reading index 6 as temperature gives you a + plausible-looking wrong number — decode with the CLI or the layout tables + in references/observation-layouts-and-units.md. +2. **`/better_forecast` is unit-selectable, not Celsius-locked.** It defaults + to metric but honors `units_temp=f`, `units_wind=mph`, `units_pressure=inhg`, + `units_precip=in`. It reports what it used in `response.units`. Converting + an already-Fahrenheit response doubles it (25.4 °C → 77.7 °F → 172 "°F"). + Always read `units` before converting; the CLI does this for you. +3. **UDP message families differ structurally — dispatch on `type` first.** + obs families nest rows under `obs`; `rapid_wind` carries one array under + `ob`; `evt_precip`/`evt_strike` carry one array under `evt`; + `hub_status`/`device_status` carry named fields with no payload array. + Iterating `rapid_wind`'s `ob` element-wise is the classic TypeError; the + bundled `decode_message()` shows the correct dispatch. +4. **UDP obs_st rows stop at index 17; REST rows run to 21.** The four + Nearcast/analysis fields (18–21) exist only in REST responses. Decoders + must tolerate both lengths — the CLI emits `None` for missing tails. +5. **Pressure is MB (millibars), numerically hPa — not kPa.** It is also + *station* pressure (raw sensor). The Tempest app's "relative pressure" + adds an elevation adjustment; don't compare raw station pressure against + the app and conclude the sensor drifted. +6. **Forecast timestamps are epoch integers, never ISO strings.** + `day_start_local`, `sunrise`, `sunset`, hourly `time` are epoch seconds; + hourly objects carry `local_hour` (0–23) and `local_day` (day of month). + There is **no** `local_time` or `time_string` field — code expecting one + silently falls back to its default branch. +7. **The forecast nests under a `forecast` wrapper key.** `data["daily"]` is + always empty; read `data["forecast"]["daily"]` and + `data["forecast"]["hourly"]` (the CLI's `--json` preserves the full + response, wrapper and all). +8. **Hubs (`HB`) have no observations.** They only relay. Auto-selection + skips them; if you call the API directly, filter `device_type == "HB"` + out before hitting `/observations/device/{id}` (documented 404 otherwise). +9. **UDP is LAN-only and unauthenticated.** Broadcasts don't cross routers + and can't be token-gated — anyone on the network can read your station. + WeatherFlow officially positions REST/WebSocket as primary and UDP as the + off-grid/backup interface. +10. **`obs_sky` UDP day-rain is always null.** Local-day rain accumulation + (index 11) is `null` in UDP SKY broadcasts; REST supplies the real value. + Don't build day-rain totals from UDP SKY rows. -All raw observation data comes in metric (Celsius, m/s, MB, mm). The CLI converts for human display. If you're parsing raw `--json` output, expect metric values. The `better_forecast` endpoint returns unit-converted values based on station preferences — check the `units` key in the response — but **temperatures are always in Celsius** regardless. +## When to use -### Timestamps are epoch integers, not strings - -The API returns Unix epoch timestamps, not ISO 8601 strings. The `daily[].day_start_local` field is an `int`, not `"2026-05-10T00:00:00"`. Hourly objects have `local_hour` (int 0-23) and `local_day` (int) — there is **no** `local_time` or `time_string` field. The CLI handles this, but raw JSON consumers need to convert with `datetime.fromtimestamp(ts)`. - -### Nested forecast response - -The `better_forecast` endpoint nests daily and hourly arrays under a `forecast` wrapper key, not at the top level: - -```python -# Correct path: -fc = data.get("forecast", {}) -days = fc.get("daily", []) -hours = fc.get("hourly", []) -``` - -The top-level keys are: `current_conditions` (dict), `forecast` (dict with `daily` + `hourly`), `station` (metadata), `units`, `status`, `timezone`. - -### Device type filtering - -A station returns all devices including the hub (device_type `HB`). The hub cannot serve observations. The CLI auto-filters HB devices and prefers ST > SKY > AIR. If you're bypassing the CLI and calling the API directly, always filter out device_type `HB` before querying observation endpoints. - -### Global flags in any position - -`--json`, `--dry-run`, `--quiet`, and `--verbose` work anywhere in the command: - -```bash -tempest --json current --device-id 67890 # flag before subcommand -tempest current --device-id 67890 --json # flag after subcommand -``` - -## References - -- [references/tempest-api-field-layouts.md](references/tempest-api-field-layouts.md) — Full field index maps for obs_st, obs_air, and obs_sky observation arrays. Read when decoding raw JSON output or building on top of the Tempest API. -- [scripts/tempest](scripts/tempest) — The CLI binary itself. Designed following the cli-builder patterns: non-interactive, `--json`, `--dry-run`, `--quiet`, `--verbose`, idempotent, dual-output via `emit()`, and structured logging. +- The user owns or manages a WeatherFlow Tempest / Air / Sky station and asks + about its readings, forecast, or history. +- Parsing or integrating with the hub's local UDP broadcast (port 50222). +- Rain/wind/lightning monitoring scripts, dashboards, or home-automation + hooks fed from the station. ## When not to use -Do not use this skill for weather questions that do not involve a personal WeatherFlow station (a public forecast service serves those better), for aviation METAR/TAF data, or for hardware from other vendors — every endpoint here requires a Tempest account token and talks to WeatherFlow's consumer API. +- **Generic city forecasts or users without a station** — every endpoint + requires the user's own Tempest station and a personal-use token; use a + public weather service instead. +- **Shakespeare's play *The Tempest*, or any literary/meteorological-theory + question** — this is a station-data CLI, not an encyclopedia. +- **Other vendors' hardware** (Netatmo, Ecowitt, Davis, Ambient) — different + APIs entirely; no endpoint here will accept their devices. +- **Commercial/network-wide data products** — those need WeatherFlow's + TempestONE agreements, not a personal token (see the remote developer + policy). + +## Reference Files + +| File | Read when | +|---|---| +| [references/rest-api-and-auth.md](references/rest-api-and-auth.md) | Working with REST endpoints directly: token auth, StationSet shapes, observation parameters, forecast units, error signatures | +| [references/udp-broadcast-protocol.md](references/udp-broadcast-protocol.md) | Parsing raw UDP datagrams: port 50222 transport, every message family's layout, the type-dispatch rule | +| [references/observation-layouts-and-units.md](references/observation-layouts-and-units.md) | Decoding positional observation arrays by index (obs_st/obs_air/obs_sky, UDP vs REST lengths) and unit conversion tables | +| [references/cli-worked-recipes.md](references/cli-worked-recipes.md) | Copy-paste multi-step CLI recipes with jq stages, dry-run plans, and expected error paths | + +## Available Scripts + +- [scripts/tempest](scripts/tempest) — the CLI: `stations`, `current`, `obs`, + `forecast`, `udp listen`; global `--json`, `--dry-run`, `--quiet`, + `--verbose` accepted in any position; offline dry-run plans for every + command. +- [scripts/test_tempest.py](scripts/test_tempest.py) — offline suite: canned + UDP datagram bytes fed to the decoder (no sockets), mocked REST transport, + both pytest and unittest runners. + +## Prerequisites + +- Python 3.8+ with `requests` (the only dependency). +- `TEMPEST_TOKEN` for REST commands (free, personal use; created in the + Tempest web app). UDP listening needs no token, only line-of-sight to the + hub's LAN. diff --git a/tempest/evals/evals.json b/tempest/evals/evals.json new file mode 100644 index 0000000..6f2e950 --- /dev/null +++ b/tempest/evals/evals.json @@ -0,0 +1,92 @@ +{ + "schema_version": 1, + "skill_name": "tempest", + "evals": [ + { + "id": "current-conditions-from-station", + "prompt": "What's the temperature, wind, and rain at my Tempest station right now? Give it to me as JSON I can pipe to jq.", + "expected_output": "Export TEMPEST_TOKEN (create it in the Tempest web app: Settings -> Data Authorizations -> Create Token), then run tempest current --json. Auto-selection picks your first station and its ST device (skipping HB hubs). The .observation object carries metric-native named fields: air_temperature (C), wind_avg (m/s), rain_accumulation (mm), station_pressure (MB), relative_humidity (%). Use --station-id/--device-id only when you own several stations.", + "assertions": [ + "exports TEMPEST_TOKEN and runs tempest current --json", + "reads metric-native fields air_temperature, wind_avg, and rain_accumulation from .observation", + "does not invent a local_time or hourly.local_time field anywhere", + "does not present imperial units as the wire values without converting" + ] + }, + { + "id": "stations-to-current-pipeline", + "prompt": "I have two Tempest stations. Figure out their IDs and then pull the latest reading from the backyard one, chained so I can re-run it.", + "expected_output": "Discover first: tempest stations --json emits {\"stations\": [...]} with integer station_id and a devices[] array where each device has device_id, device_type (HB hub, ST Tempest, AR Air, SK Sky), and serial_number. Filter device_type == \"ST\" (never HB - hubs carry no observations) and feed those integers to tempest current --station-id --device-id --json. The two stages compose because stations --json device_id/station_id are the same integer types current's flags accept.", + "assertions": [ + "runs tempest stations --json first and extracts station_id and device_id as integers", + "filters out device_type HB hubs before choosing the observation device", + "passes the extracted ids to tempest current --station-id/--device-id", + "does not call an undocumented /user/devices endpoint" + ] + }, + { + "id": "forecast-units-double-conversion-gotcha", + "prompt": "Why does my script show 172 degrees for my Tempest forecast after I switched the station display to Fahrenheit? The high today is definitely not 172.", + "expected_output": "Double conversion. The /better_forecast endpoint is unit-selectable, not Celsius-locked: it honors units_temp=f (default c) and reports what it used in response.units. Your script converted an already-Fahrenheit response with C->F math (77.7 * 9/5 + 32 = 172). Fix: read .forecast.units.units_temp before converting anything, or request explicit units. With the CLI: tempest forecast --json already handles this - its human output converts only Celsius stations, and raw --json values stay in the units the response declared.", + "assertions": [ + "explains the 172 value as a double conversion of an already-Fahrenheit response", + "states the endpoint honors units_temp=f and reports units in the response", + "instructs reading .forecast.units.units_temp before converting", + "does not claim the forecast endpoint is always Celsius regardless of parameters" + ] + }, + { + "id": "udp-message-family-dispatch", + "prompt": "I'm parsing my Tempest hub's UDP broadcast on port 50222 in Python. I keep getting TypeError when a rapid wind message shows up, and my parser never sees rain-start events. What's wrong?", + "expected_output": "Message families are structurally different - dispatch on the top-level \"type\" before indexing. obs_st/obs_air/obs_sky nest report rows under \"obs\" (msg[\"obs\"][0][7] is temperature); rapid_wind carries ONE 3-element array under \"ob\" ([epoch, m/s, degrees]) - iterating it element-wise like an obs row list is exactly the TypeError you hit; evt_precip and evt_strike carry ONE array under \"evt\" ([epoch] and [epoch, km, energy]); hub_status and device_status have named fields (uptime, rssi, seq, reset_flags, sensor_status) and no payload array at all. Each UDP datagram is one complete JSON object; bind 0.0.0.0:50222 and listen only - the hub never expects a reply.", + "assertions": [ + "dispatches on the type field before any positional indexing", + "reads rapid_wind speed from the single ob array as ob[1], not by iterating it", + "distinguishes obs families (list under obs) from evt families (single array under evt) and status families (named fields)", + "binds the listener to UDP port 50222 and treats it as listen-only broadcast" + ] + }, + { + "id": "obs-st-positional-array-decode", + "prompt": "Decode this raw obs_st payload from my Tempest: [1588948614, 0.18, 0.22, 0.27, 144, 6, 1017.57, 22.37, 50.26, 328, 0.03, 3, 0.0, 0, 0, 0, 2.410, 1]. What's the temperature and wind?", + "expected_output": "obs_st is a positional array - meaning comes from the index. Index 0 epoch 1588948614 (2020-05-07 UTC); index 1-3 wind lull/avg/gust 0.18/0.22/0.27 m/s; index 4 wind direction 144 degrees (SE); index 6 station pressure 1017.57 MB (millibars, same as hPa); index 7 air temperature 22.37 C (72.3 F); index 8 humidity 50.26%; index 12 rain 0.0 mm this minute; index 16 battery 2.410 V (healthy, about 2.4 nominal); index 17 report interval 1 minute. The UDP broadcast record ends at index 17; REST adds Nearcast rain fields 18-21 for 22 positions - tolerate both lengths.", + "assertions": [ + "maps index 7 to air temperature 22.37 C and index 6 to pressure in MB/hPa", + "maps indices 1-3 to wind lull/average/gust in m/s and index 4 to direction", + "notes the UDP record stops at index 17 while the REST record has 22 positions", + "does not misread index 6 pressure as temperature or vice versa" + ] + }, + { + "id": "metric-native-units-and-conversions", + "prompt": "Are the values from my Tempest station in Fahrenheit and mph? I want mph wind and inches of rain in my dashboard.", + "expected_output": "No - the wire is metric-native everywhere: wind m/s, rain mm, temperature C, pressure MB (millibars, numerically hPa - not kPa), lightning distance km. Conversion is the caller's job: mph = m/s * 2.237, inches = mm / 25.4, F = C * 9/5 + 32, inHg = MB * 0.02953. The CLI converts only for human display; --json stays metric-native so jq can convert: tempest current --json | jq '{wind_mph: (.observation.wind_avg * 2.237), rain_in: (.observation.rain_accumulation / 25.4)}'.", + "assertions": [ + "states observations are metric-native (m/s, mm, C, MB) with conversion as the caller's job", + "provides the m/s-to-mph and mm-to-inches conversion formulas or a jq snippet", + "does not claim UDP or raw JSON values arrive in imperial units", + "uses MB or hPa for pressure, not kPa" + ] + }, + { + "id": "not-shakespeare-the-tempest", + "prompt": "Analyze the opening storm scene of Shakespeare's play The Tempest and explain how Prospero raises the tempest.", + "expected_output": "This must not trigger the tempest skill: it is a literature question about Shakespeare's play, not a request for WeatherFlow weather-station data. The tempest skill operates a personal weather station (REST token auth, UDP port 50222 broadcasts) and has nothing to say about the play. Route this to literary analysis instead.", + "assertions": [ + "must not trigger the tempest skill for the Shakespeare play", + "recognizes the question as literary analysis of The Tempest", + "does not invoke station APIs, tokens, or UDP ports for this prompt" + ] + }, + { + "id": "not-generic-weather-forecast", + "prompt": "What's the weather forecast for Paris tomorrow? I don't own any weather station.", + "expected_output": "This must not trigger the tempest skill: every endpoint it drives requires the user's own WeatherFlow Tempest station and a personal-use token, and UDP listening requires a hub on the LAN. A user with no station asking for a generic city forecast needs a public weather service or forecast skill, not this station tool. Only load tempest when the user owns or manages a Tempest/WeatherFlow station.", + "assertions": [ + "must not trigger the tempest skill for a generic city forecast", + "notes the skill requires the user's own Tempest station and token", + "routes the request to a public forecast service instead" + ] + } + ] +} diff --git a/tempest/references/cli-worked-recipes.md b/tempest/references/cli-worked-recipes.md new file mode 100644 index 0000000..347f2b0 --- /dev/null +++ b/tempest/references/cli-worked-recipes.md @@ -0,0 +1,160 @@ +# CLI Worked Recipes (tempest) + +Multi-step, executable recipes for the bundled `tempest` CLI. Global flags +`--json`, `--dry-run`, `--quiet`, `--verbose` work in any position on the +command line. `--json` output is metric-native (raw wire units); human output +is converted. `--dry-run` never touches the network and always exits 0 with a +plan object. + +## Recipe 1: Discover the station, then read current conditions + +```bash +# Step 1: find station and device ids (works even before you memorize ids) +tempest stations --json | jq '.stations[] | {station_id, name, + devices: [.devices[] | {device_id, device_type, serial_number}]}' + +# Step 2: current conditions, machine-readable +tempest current --json | jq '{station, device_id, type, + temp_c: .observation.air_temperature, + wind_mps: .observation.wind_avg, + rain_mm: .observation.rain_accumulation}' + +# Step 3 (pin a specific station/device when several exist) +tempest current --station-id 12799 --device-id 60526 --json +``` + +Stage compatibility: `stations --json` emits `{"stations": [...]}` with +integer `station_id`/`device_id` fields — feed those ints to +`--station-id`/`--device-id` on `current`. `current --json` emits +`{station, device_id, type, observation}` where `observation` carries the +decoded positional array as named fields (metric-native types: numbers for +measurements, `timestamp` as ISO-8601 string). + +Auto-selection rules when you don't pass ids: the first station is used; the +device is the first `ST` (Tempest), then `SKY`/`SK`, then `AIR`/`AR`, always +skipping `HB` hubs (hubs carry no observations). If only a hub exists the CLI +dies with a clear error instead of guessing. + +## Recipe 2: 7-day forecast slice for scripts + +```bash +tempest forecast --days 7 --json \ + | jq '{units_temp: .forecast.units.units_temp, + today: (.forecast.forecast.daily[0] + | {day_start_local, air_temp_high, air_temp_low, precip_probability}), + next12: [.forecast.forecast.hourly[:12][] + | {local_hour, air_temperature, precip_probability}]}' +``` + +Converting highs to °F with jq (read `units` from the same document before +converting anything): + +```bash +tempest forecast --json \ + | jq '{units_temp: .forecast.units.units_temp, + highs_f: [.forecast.forecast.daily[] | .air_temp_high * 9 / 5 + 32], + rain_hours: [.forecast.forecast.hourly[] | select(.precip_probability > 30) | .local_hour]}' +``` + +**Convert only after reading `units`:** the endpoint honors unit overrides +(`units_temp=f` etc.), so hard-coded Celsius math double-converts Fahrenheit +responses. When the CLI displays forecast values it converts °C→°F only for +stations whose `units_temp` is `c`. Human output prints current conditions, +then the daily table, then the next 12 hours. + +## Recipe 3: Rain-watch (yesterday's total + live rain events) + +```bash +# What fell yesterday (UTC day): obs from history, day_offset=1 +DEVICE_ID=$(tempest stations --json | jq -r ' + .stations[].devices[] | select(.device_type == "ST") | .device_id' | head -1) +tempest obs --device-id "$DEVICE_ID" --days 1 --json \ + | jq '{type, samples: (.observations | length), + day_rain_mm: .observations[-1].local_day_rain_accumulation}' + +# Live: rain-start events and rapid wind from the hub broadcast +tempest udp listen --timeout 600 --json | jq 'select(.type == "evt_precip")' +``` + +Stage compatibility: `obs --json` emits `{device_id, type, count, +observations}` with each decoded observation carrying +`local_day_rain_accumulation` (mm, number) — the `-1` index grabs the newest +sample of the day. `udp listen --json` emits one JSON object per datagram; +`evt_precip` objects carry `{type, serial_number, timestamp}`. + +## Recipe 4: Decode any raw UDP datagram positionally + +Feed canned datagram bytes to the same decoder the listener uses — no +sockets, no hub required (this is exactly how `scripts/test_tempest.py` +exercises the parser): + +```python +# /tmp/decode_one.py +import importlib.machinery, importlib.util, json +loader = importlib.machinery.SourceFileLoader("t", "tempest/scripts/tempest") +spec = importlib.util.spec_from_loader(loader.name, loader) +mod = importlib.util.module_from_spec(spec) +loader.exec_module(mod) + +datagram = (b'{"serial_number":"ST-00000512","type":"obs_st","hub_sn":"HB-00013030",' + b'"obs":[[1588948614,0.18,0.22,0.27,144,6,1017.57,22.37,50.26,328,0.03,3,' + b'0.0,0,0,0,2.410,1]],"firmware_revision":129}') +msg = json.loads(datagram.decode()) +for row in msg["obs"]: # obs families: list of rows + decoded = mod.decode_obs(row, msg["type"]) + print(decoded["air_temperature"], "°C", decoded["air_temperature_unit"]) + +rapid = json.loads(b'{"type":"rapid_wind","ob":[1493322445,2.3,128],"serial_number":"SK-1"}'.decode()) +speed, direction = rapid["ob"][1], rapid["ob"][2] # rapid_wind: ONE array under "ob" +``` + +The three structural keys to remember (see udp-broadcast-protocol.md): +observation families nest rows under `obs`; `rapid_wind` carries one array +under `ob`; events (`evt_precip`, `evt_strike`) carry one array under `evt`; +`hub_status`/`device_status` have named fields and no array at all. Dispatch +on `type` before indexing. + +## Recipe 5: Dry-run previews and flag behavior + +```bash +# Plan, don't execute: valid JSON, exit 0, zero network +tempest forecast --station-id 12799 --days 3 --dry-run --json +# -> {"dry_run": true, "command": "forecast", "station_id": 12799, "days": 3} + +# Every documented command has a dry-run plan: current, obs, forecast, stations +tempest obs --device-id 60526 --days 2 --dry-run --json + +# Quiet/verbose piping: logs on stderr, data on stdout +tempest current --json --quiet | jq .observation.air_temperature +``` + +Behavior contract: `--dry-run` works without `TEMPEST_TOKEN` set (no credential +needed to see a plan); `--help` and `--dry-run` are always offline. Without +`--dry-run`, a missing token exits 1 with +`Error: TEMPEST_TOKEN not set...` before any request is attempted. + +## Recipe 6: JSON error paths you'll actually see + +```bash +tempest current --station-id 99999999 +# Error: Station 99999999 not found. (exit 1) + +tempest obs --device-id 123 # hub or wrong device +# Error: API error (404): ... (exit 1) + +unset TEMPEST_TOKEN; tempest stations +# Error: TEMPEST_TOKEN not set. Get one at https://weatherflow.com (exit 1) +``` + +The client maps 401 → token message, 403 → access-denied message, 404 → +not-found-with-path, and any other ≥400 dumps the response body. In `--json` +mode errors still go to stderr as text; only success payloads print to stdout, +so `jq` pipelines fail loudly instead of parsing prose. + +## Sources + +- https://apidocs.tempestwx.com/reference/quick-start (token setup, REST examples, primary-source guidance) +- https://apidocs.tempestwx.com/reference/get_stations (StationSet shape feeding the stations command) +- https://apidocs.tempestwx.com/reference/getobservationsbydeviceid (device observation parameters used by current/obs) +- https://apidocs.tempestwx.com/reference/get_better-forecast-1 (forecast unit selection used by recipe 2) +- https://weatherflow.github.io/Tempest/api/udp/v171/ (UDP message families used by recipes 3–4) diff --git a/tempest/references/observation-layouts-and-units.md b/tempest/references/observation-layouts-and-units.md new file mode 100644 index 0000000..be40fb9 --- /dev/null +++ b/tempest/references/observation-layouts-and-units.md @@ -0,0 +1,175 @@ +# Observation Layouts and Units (obs_st, obs_air, obs_sky) + +Observations arrive as **positional arrays**: a list of values whose meaning +depends on the array index. The `type` field on the containing object selects +the layout (`obs_st` = Tempest all-in-one, `obs_air` = Air, `obs_sky` = Sky). +There are no field names on the wire — any decoder is a table like the ones +below, and reading the wrong index silently yields a wrong value (e.g. +treating index 6 pressure as index 7 temperature). + +Two different record lengths exist for obs_st: REST returns **22 positions** +and the UDP broadcast returns **18** (the four Nearcast/analysis fields are +REST-only). obs_air is 8 positions in both transports; obs_sky is 17 over +REST and 14 over UDP. + +## obs_st — Tempest all-in-one (REST record, 22 positions) + +| Index | Field | Units | Notes | +|---:|---|---|---| +| 0 | timestamp | epoch seconds, UTC | | +| 1 | wind lull | m/s | minimum 3-second sample | +| 2 | wind average | m/s | average over report interval | +| 3 | wind gust | m/s | maximum 3-second sample | +| 4 | wind direction | degrees | 0 = N | +| 5 | wind sample interval | seconds | | +| 6 | station pressure | MB (millibars) | ≡ hPa; raw sensor pressure, not sea-level | +| 7 | air temperature | °C | | +| 8 | relative humidity | % | | +| 9 | illuminance | lux | | +| 10 | UV | index | | +| 11 | solar radiation | W/m² | | +| 12 | rain accumulation | mm | during the reporting interval | +| 13 | precipitation type | enum | 0 none, 1 rain, 2 hail, 3 rain + hail (experimental) | +| 14 | lightning strike average distance | km | | +| 15 | lightning strike count | count | during the reporting interval | +| 16 | battery | volts | ≈2.4 nominal; below ≈2.3 plan service | +| 17 | report interval | minutes | | +| 18 | local day rain accumulation | mm | midnight-to-midnight, station timezone | +| 19 | Nearcast rain accumulation | mm | REST only | +| 20 | local day Nearcast rain accumulation | mm | REST only | +| 21 | precipitation analysis type | enum | 0 none, 1 Nearcast display on, 2 off — REST only | + +UDP `obs_st` datagrams end at index 17 (see udp-broadcast-protocol.md). + +## obs_air — Air sensor (8 positions, both transports) + +| Index | Field | Units | Notes | +|---:|---|---|---| +| 0 | timestamp | epoch seconds, UTC | | +| 1 | station pressure | MB (millibars) | ≡ hPa | +| 2 | air temperature | °C | | +| 3 | relative humidity | % | | +| 4 | lightning strike count | count | during the reporting interval | +| 5 | lightning strike average distance | km | | +| 6 | battery | volts | | +| 7 | report interval | minutes | | + +## obs_sky — Sky sensor (REST record, 17 positions) + +| Index | Field | Units | Notes | +|---:|---|---|---| +| 0 | timestamp | epoch seconds, UTC | | +| 1 | illuminance | lux | | +| 2 | UV | index | | +| 3 | rain accumulation | mm | during the reporting interval | +| 4 | wind lull | m/s | | +| 5 | wind average | m/s | | +| 6 | wind gust | m/s | | +| 7 | wind direction | degrees | | +| 8 | battery | volts | | +| 9 | report interval | minutes | | +| 10 | solar radiation | W/m² | | +| 11 | local day rain accumulation | mm | **always null over UDP** — REST supplies it | +| 12 | precipitation type | enum | 0 none, 1 rain, 2 hail, 3 rain + hail | +| 13 | wind sample interval | seconds | | +| 14 | Nearcast rain accumulation | mm | REST only | +| 15 | local day Nearcast rain accumulation | mm | REST only | +| 16 | precipitation analysis type | enum | 0 none, 1 Nearcast display on, 2 off — REST only | + +UDP `obs_sky` datagrams end at index 13 and always carry `null` at index 11. + +## Daily summary records (obs_*_ext) + +The API also emits midnight-to-midnight daily summaries with their own +discriminators: `obs_st_ext` (34 positions — avg/high/low pressure, +temperature, humidity, illuminance, UV, solar, wind stats, strikes, battery, +day rain, precipitation minutes), `obs_air_ext` (14), and `obs_sky_ext` (22). +They appear in stats/history contexts, not in the minute firehose. Decode +them only from their own `type` — never with the minute-record tables. + +## The units story: metric-native, caller converts + +Every raw value is metric: wind **m/s**, rain **mm**, temperature **°C**, +pressure **MB** (millibars — numerically identical to hPa, *not* kPa), +distance **km**, illuminance **lux**, solar radiation **W/m²**, battery +**volts**. Nothing on the wire is imperial; conversions are the consumer's +job: + +| Wire unit | Imperial | Formula | +|---|---|---| +| °C | °F | `c * 9/5 + 32` | +| m/s | mph | `mps * 2.23694` (≈ ×2.237) | +| m/s | km/h | `mps * 3.6` | +| m/s | knots | `mps * 1.94384` | +| MB (hPa) | inHg | `mb * 0.02953` | +| mm | inches | `mm / 25.4` | +| km | miles | `km / 1.60934` | + +Two traps: + +1. **`/better_forecast` is unit-selectable, not Celsius-locked.** It defaults + to metric (`units_temp=c`), honors overrides (`units_temp=f`, + `units_wind=mph`, `units_pressure=inhg`, `units_precip=in`, + `units_distance=mi`), and reports what it used in `response.units`. Read + `units` before converting anything, or a Fahrenheit response gets + double-converted into absurd values. +2. **Station vs sea-level pressure.** Index 6 / index 1 pressure is the raw + station pressure. The Tempest app's "relative pressure" adds an elevation + adjustment — don't compare your raw value against the app and conclude the + sensor is broken. + +The bundled CLI keeps `--json` output in metric-native wire units (raw, +lossless — convert with your own jq) and converts only in human display. +Decode positionally with jq like: + +```bash +tempest current --json \ + | jq '{temp_c: .observation.air_temperature, + temp_f: (.observation.air_temperature * 9 / 5 + 32), + wind_mps: .observation.wind_avg, + wind_mph: (.observation.wind_avg * 2.237), + pressure_mb: .observation.station_pressure}' +``` + +## Field type traps in /better_forecast + +The forecast endpoint uses epoch integers where you'd expect date strings, +and field names that differ from what common sense suggests: + +| Field | Actual type | Common mistake | Fix | +|---|---|---|---| +| `daily[].day_start_local` | epoch int (e.g. 1778385600) | assumed ISO string | `datetime.fromtimestamp(ts).strftime(...)` | +| `hourly[].local_hour` | int (0–23) | assumed timestamp string | format directly `{h:02d}:00` | +| `hourly[].local_day` | int (day of month) | N/A | use alongside `local_hour` | +| `hourly[].local_time` | **does not exist** | commonly assumed field | use `local_hour` instead | + +Code looking for `local_time` silently falls back to its default/"?" branch — +no error is raised. + +## Decoding recipe (jq, no script needed) + +Latest REST observation, positionally decoded to named fields: + +```bash +curl -s "https://swd.weatherflow.com/swd/rest/observations/device/$DEVICE_ID?token=$TEMPEST_TOKEN" \ + | jq --argjson layout '["timestamp","wind_lull","wind_avg","wind_gust","wind_direction", + "wind_sample_interval","station_pressure","air_temperature","relative_humidity", + "illuminance","uv","solar_radiation","rain_accumulation","precipitation_type", + "avg_strike_distance","strike_count","battery","report_interval", + "local_day_rain","nc_rain","local_day_nc_rain","precip_analysis_type"]' ' + {type: .type, + obs: (.obs[-1] | [$layout, .] | transpose | map({(.[0]): .[1]}) | add)}' +``` + +The bundled CLI does the same in Python (`decode_obs` in `scripts/tempest`, +driven by the `OBS_ST_FIELDS`/`OBS_AIR_FIELDS`/`OBS_SKY_FIELDS` tables) and +tolerates both UDP-length and REST-length rows. + +## Sources + +- https://apidocs.tempestwx.com/reference/observation-record-format (canonical index tables: obs_st 22, obs_air 8, obs_sky 17, daily _ext records, evt_strike, rapid_wind) +- https://weatherflow.github.io/Tempest/api/swagger/ (legacy response models; better_forecast field types; obs_sky UDP day-rain null note) +- https://weatherflow.github.io/Tempest/api/udp/v171/ (UDP obs_st 18-position record; metric-native units) +- https://apidocs.tempestwx.com/reference/get_better-forecast-1 (unit selection parameters and response `units` object) +- https://apidocs.tempestwx.com/reference/getobservationsbydeviceid (observation set envelope: `obs` array + `type` discriminator) +- https://help.weatherflow.com/hc/en-us/articles/360052101413-Tempest-FAQs (station vs sea-level pressure; battery guidance) diff --git a/tempest/references/rest-api-and-auth.md b/tempest/references/rest-api-and-auth.md new file mode 100644 index 0000000..3319a91 --- /dev/null +++ b/tempest/references/rest-api-and-auth.md @@ -0,0 +1,251 @@ +# Tempest REST API and Authentication + +The Tempest REST API is the cloud service at `https://swd.weatherflow.com/swd/rest`. +It is the primary, recommended data source even for programs running on the same +LAN as the hub; the local UDP broadcast (see udp-broadcast-protocol.md) is +officially positioned as an off-grid backup. Base URL used throughout: + +``` +https://swd.weatherflow.com/swd/rest +``` + +## Authentication: the personal access token + +There are exactly two documented authentication methods, and the bundled CLI +uses the first: + +1. **Personal Access Token** — the right choice for scripts and integrations + without a graphical interface. Sign in to the Tempest Web App + (tempestwx.com), then go to **Settings → Data Authorizations → Create + Token**, and copy the generated token. This is what `TEMPEST_TOKEN` + carries. +2. **OAuth 2.0** (Authorization Code, optionally with PKCE) — the documented + choice for production apps with a web UI. Apps are registered from the + account's Developers page; authorization and token endpoints are documented + separately in the OAuth reference. The CLI does not implement OAuth. + +On the wire, the token travels as a **query parameter**: + +``` +GET https://swd.weatherflow.com/swd/rest/stations?token= +``` + +The official quick-start examples use `token=[your_access_token]` and show no +`Authorization` header alternative for this API. Do not send the token as a +header or assume bearer syntax is supported. The OpenAPI document describes the +scheme as `apiKey` with `in: query`, which matches. + +Policy notes (remote-developer-policy): personal-use access covers station +metadata, observations, and forecasts with "rate/volume limits (enough for +personal use)". No numeric quota is published, and no 429 response behavior is +documented. Higher-volume or network-wide access requires a commercial +agreement (TempestONE). Keep personal integrations to your own stations. + +## Endpoint catalog (personal-use surface) + +### GET /stations — your stations with devices + +Parameters: `limit` (int64, default 10000), `next_cursor` (string; present +when more than 10,000 stations are provisioned), optional geographic filters +(`lat_min`/`lon_min`/`lat_max`/`lon_max` bounding box, or +`center_lat`/`center_lon`/`radius` in meters). + +Response is a **StationSet wrapper**, not a bare list: + +```json +{ + "status": { "status_code": 0, "status_message": "SUCCESS" }, + "stations": [ + { + "station_id": 12799, + "location_id": 12799, + "name": "Home", + "public_name": "Home", + "latitude": 42.37, + "longitude": -71.06, + "timezone": "America/New_York", + "timezone_offset_minutes": -300, + "station_meta": { "elevation": 1567.65, "share_with_wf": true, "share_with_wu": true }, + "is_local_mode": false, + "devices": [ + { + "device_id": 60526, + "serial_number": "ST-00012345", + "device_type": "ST", + "hardware_revision": "3", + "firmware_revision": "165", + "device_meta": { "agl": 2.2, "name": "Backyard", "environment": "outdoor" }, + "device_settings": { "show_precip_final": false }, + "notes": "" + } + ], + "station_items": [ { "item": "air_temperature_humidity", "device_id": 60526, "sort": 0 } ] + } + ] +} +``` + +`device_type` values: `HB` (hub — has **no** observation data), +`ST` (Tempest all-in-one), `AR` (Air sensor), `SK` (Sky sensor). The OpenAPI +enum lists exactly these four. Note that `AR`/`SK` are metadata codes for the +Air/Sky hardware; the observation `type` discriminator for the same hardware is +`obs_air`/`obs_sky`. Always filter `HB` out before auto-selecting a device for +observation calls — the hub has no `/observations/device/{id}` data. A null or +missing `serial_number` on a device means inactive hardware per the legacy docs. + +### GET /stations/{station_id} — one station + +Same Station model; documented responses are 200 and 404 ("Station not found"). +Per the legacy Swagger the body still arrives in the `{stations: [...]}`-style +wrapper shape with the selected station inside, so unwrap defensively rather +than assuming a bare station object. + +### GET /observations/device/{device_id} — device observations + +Query parameters (mutually exclusive modes): + +| Parameter | Meaning | +|---|---| +| `day_offset` | Whole UTC day: `0` = current UTC day, `1` = yesterday UTC | +| `time_start` + `time_end` | UTC epoch-seconds range; one-minute resolution guaranteed for ranges ≤ 5 days | +| `latest=true` | Latest single observation (the CLI's `current` default) | +| `format=csv` | CSV instead of JSON | + +Response is an observation set: `obs` (array of positional arrays, oldest to +newest), `type` (`obs_st` | `obs_air` | `obs_sky` — the layout discriminator), +plus device identity/status fields. Field layouts are in +observation-layouts.md. Documented errors: 404 "Device not found". Passing a +hub `HB` device id yields no observation data. + +### GET /observations/stn/{station_id} — station observations + +Note the segment is **`stn`**, not `stations`. Optional parameters: +`time_start`/`time_end`, `bucket` (`1` | `5` | `30` | `180` minutes; mapped to +1 day / 5 days / 30 days / 180 days of history, and the docs mention `1440` ≈ 4 +years), `ob_fields` selection, and the standard unit parameters. Station +observations are **federated from the station's designated primary sensors**; +device observations are one physical device's raw data. Use station +observations when you want "the station's" reading, device observations when +you care about a specific unit. + +### GET /better_forecast — conditions + daily + hourly + +Parameters: `station_id` (or `lat`/`lon` with optional +`snap_to_nearest_owned_station=true` for within-5 km snapping), plus unit +overrides: `units_temp` (`c`|`f`), `units_wind` (`mph`|`kph`|`kts`|`mps`|`bft`| +`lfm`), `units_pressure` (`mb`|`inhg`|`mmhg`|`hpa`), `units_precip` +(`mm`|`cm`|`in`), `units_distance` (`km`|`mi`). + +Response top level: + +```json +{ + "status": { "status_code": 0, "status_message": "SUCCESS" }, + "current_conditions": { "air_temperature": 18.2, "conditions": "Mostly Clear", "icon": "partly-cloudy-day", "relative_humidity": 61, "station_pressure": 1015.4, "wind_avg": 2.1, "wind_direction": 225, "feels_like": 18.2 }, + "forecast": { + "daily": [ { "day_start_local": 1778385600, "air_temp_high": 25.4, "air_temp_low": 15.1, "conditions": "Partly cloudy", "precip_probability": 10, "precip_type": "rain", "sunrise": 1778378400, "sunset": 1778425200 } ], + "hourly": [ { "time": 1778388000, "local_hour": 10, "local_day": 10, "air_temperature": 19.8, "precip_probability": 5, "conditions": "Sunny" } ] + }, + "units": { "units_temp": "c", "units_wind": "mps", "units_precip": "mm", "units_pressure": "mb", "units_distance": "km" }, + "latitude": 42.37, "longitude": -71.06, + "timezone": "America/New_York", "timezone_offset_minutes": -300 +} +``` + +The critical structural fact: **daily and hourly live under the `forecast` +wrapper key**, not at top level. Reading `data["daily"]` returns nothing. + +Unit behavior: the response honors the requested units and reports what it used +in `units`. Default is Celsius/m/s/mm/mb, but the endpoint is **unit-selectable +— not Celsius-locked**. `units_temp=f` is documented and honored. Any consumer +that hard-codes Celsius conversion must first read `units.units_temp`, or it +will double-convert Fahrenheit responses (see units-and-conversions.md). + +Timestamps: `day_start_local`, `sunrise`, `sunset`, and hourly `time` are +integer epoch seconds. Hourly objects carry `local_hour` (int 0–23) and +`local_day` (int day-of-month); there is **no** `local_time` or `time_string` +field — code expecting one silently falls back to its default branch. + +### Other documented endpoints + +- `GET /diagnostics/{station_id}` — latest station status; 200/401/404. +- `GET /stats/station/{station_id}` — daily/weekly/monthly/annual/all-time + high-low-average statistics; 200/401. +- `GET /metadata/network/stations` and `GET /observations/network/stations` — + network-wide access governed by the remote data policy (not part of the + personal single-station flow). +- Lightning endpoints exist but documented access is for paid subscribers. +- The current docs index does not document `/user/devices` for the consumer + surface — use `/stations` and its nested `devices` array. There is no + `/better_forecast/hourly` route; hourly data is `forecast.hourly` inside the + standard `/better_forecast` response. + +## Error signatures + +| Status | Documented meaning | Practical symptom | +|---|---|---| +| 401 | Unauthorized (documented on forecast/diagnostics/stats) | Missing, revoked, or mistyped token — regenerate at tempestwx.com Settings → Data Authorizations | +| 403 | Not documented for this API | Treat as access-denied to that station/device; verify the token belongs to the station owner | +| 404 | "Station not found" / "Device not found" (documented) | Wrong station/device id, or an `HB` hub id passed to an observation endpoint | + +No JSON error-body schema is published, so parse defensively. No numeric rate +limit or 429 behavior is documented; the policy only promises personal-use +volume is acceptable. The CLI maps 401/403/404 to targeted messages and dumps +the response body for anything else. + +## Worked recipes + +### Recipe A: stations → pick sensor → latest observation + +```bash +# 1. List stations (StationSet wrapper) +curl -s "https://swd.weatherflow.com/swd/rest/stations?token=$TEMPEST_TOKEN" +# 2. Choose a device: devices[].device_type must not be "HB"; prefer ST +# 3. Latest observation for that device +curl -s "https://swd.weatherflow.com/swd/rest/observations/device/$DEVICE_ID?token=$TEMPEST_TOKEN" +``` + +The observation response's `type` field selects the positional layout +(`obs_st`: temperature is index 7, epoch is index 0). One command does all +three steps: `tempest current --json`. + +### Recipe B: station forecast with explicit units + +```bash +curl -s "https://swd.weatherflow.com/swd/rest/better_forecast?station_id=$STATION_ID&units_temp=c&units_wind=mps&units_pressure=mb&units_precip=mm&token=$TEMPEST_TOKEN" \ + | jq '{current: .current_conditions.air_temperature, + days: [.forecast.daily[] | {day_start_local, air_temp_high, air_temp_low}], + units: .units.units_temp}' +``` + +Read `units` instead of assuming units. Extract daily/hourly from +`.forecast.daily` / `.forecast.hourly`. + +### Recipe C: a UTC day of device history + +```bash +# day_offset=1 is yesterday UTC; day_offset=0 is today +curl -s "https://swd.weatherflow.com/swd/rest/observations/device/$DEVICE_ID?day_offset=1&token=$TEMPEST_TOKEN" \ + | jq '{type, count: (.obs | length), first: .obs[0], last: .obs[-1]}' +``` + +For a custom range, send both `time_start` and `time_end` as epoch seconds and +keep the span ≤ 5 days to guarantee one-minute resolution. Do not mix +`day_offset` with `time_start`/`time_end` in one call. + +## Sources + +- https://apidocs.tempestwx.com/reference/quick-start (auth flows, REST examples, primary-source guidance) +- https://apidocs.tempestwx.com/reference/oauth (OAuth 2.0 grant types, app registration) +- https://apidocs.tempestwx.com/reference/get_stations (StationSet/Station/Device OpenAPI schemas) +- https://apidocs.tempestwx.com/reference/getstationbyid-1 (single station, 404 semantics) +- https://apidocs.tempestwx.com/reference/getobservationsbydeviceid (device observation parameters, 404) +- https://apidocs.tempestwx.com/reference/get_observations-stn-station-id (station observations, bucket) +- https://apidocs.tempestwx.com/reference/station-vs-device (device vs station observation semantics) +- https://apidocs.tempestwx.com/reference/get_better-forecast-1 (forecast parameters, unit selection) +- https://apidocs.tempestwx.com/reference/get_diagnostics-station-id-1 (diagnostics endpoint) +- https://apidocs.tempestwx.com/reference/get_stats-station-station-id-1 (stats endpoint) +- https://apidocs.tempestwx.com/reference/observation-record-format (type discriminators, record lengths) +- https://apidocs.tempestwx.com/reference/tempest-udp-broadcast (UDP as backup to REST) +- https://weatherflow.github.io/Tempest/api/swagger/ (legacy response models: forecast nesting, obs_sky null day-rain) +- https://weatherflow.github.io/Tempest/api/remote-developer-policy.html (personal-use policy, rate/volume limits) diff --git a/tempest/references/tempest-api-field-layouts.md b/tempest/references/tempest-api-field-layouts.md deleted file mode 100644 index 16c291e..0000000 --- a/tempest/references/tempest-api-field-layouts.md +++ /dev/null @@ -1,128 +0,0 @@ -# Tempest API Field Layouts - -Quick reference for the obs_st (Tempest all-in-one) observation array layout. -The API returns observations as positional arrays — these index maps are -required for any CLI or script that reads raw observation data. - -## obs_st (Tempest Device) - -| Index | Field | Units | Notes | -|-------|-------|-------|-------| -| 0 | epoch | seconds UTC | | -| 1 | wind_lull | m/s | Minimum 3-second sample | -| 2 | wind_avg | m/s | Average over report interval | -| 3 | wind_gust | m/s | Maximum 3-second sample | -| 4 | wind_direction | degrees | 0-360 | -| 5 | wind_sample_interval | seconds | | -| 6 | station_pressure | MB | | -| 7 | air_temperature | C | | -| 8 | relative_humidity | % | | -| 9 | illuminance | lux | | -| 10 | uv | index | | -| 11 | solar_radiation | W/m² | | -| 12 | rain_accumulation | mm | Over last report interval | -| 13 | precipitation_type | 0=none 1=rain 2=hail | | -| 14 | avg_strike_distance | km | | -| 15 | strike_count | count | | -| 16 | battery | volts | | -| 17 | report_interval | minutes | | -| 18 | local_day_rain_accumulation | mm | | -| 19 | nc_rain_accumulation | mm | | -| 20 | local_day_nc_rain_accumulation | mm | | -| 21 | precip_analysis_type | enum | 0=none, 1=RainCheck display on, 2=off | - -## obs_air (Air Sensor) - -| Index | Field | Units | -|-------|-------|-------| -| 0 | epoch | seconds UTC | -| 1 | station_pressure | MB | -| 2 | air_temperature | C | -| 3 | relative_humidity | % | -| 4 | lightning_strike_count | count | -| 5 | lightning_avg_distance | km | -| 6 | battery | volts | -| 7 | report_interval | minutes | - -## obs_sky (Sky Sensor) - -| Index | Field | Units | Notes | -|-------|-------|-------|-------| -| 0 | epoch | seconds UTC | | -| 1 | illuminance | lux | | -| 2 | uv | index | | -| 3 | rain_accumulation | mm | | -| 4 | wind_lull | m/s | | -| 5 | wind_avg | m/s | | -| 6 | wind_gust | m/s | | -| 7 | wind_direction | degrees | | -| 8 | battery | volts | | -| 9 | report_interval | minutes | | -| 10 | solar_radiation | W/m² | | -| 11 | local_day_rain_accumulation | mm | | -| 12 | precipitation_type | 0=none 1=rain 2=hail | | -| 13 | wind_sample_interval | seconds | | -| 14 | nc_rain | mm | | -| 15 | local_day_nc_rain | mm | | -| 16 | precip_analysis_type | 0=none 1=RainCheck on 2=off | | - -## API Response Quirks - -### better_forecast nesting -The `daily` and `hourly` arrays live under a `forecast` wrapper key, NOT at the -top level of the response. If reading from the raw API: - -```python -# Wrong (assumes top-level): -days = data.get("daily", []) # returns [] - -# Right (respects nesting): -fc = data.get("forecast", {}) -days = fc.get("daily", []) -hours = fc.get("hourly", []) -``` - -The top-level keys of `/better_forecast` are: -- `current_conditions` — dict with air_temperature, conditions, icon, etc. -- `forecast` — dict containing `daily` (list) and `hourly` (list) -- `station` — metadata (elevation, agl, station_id) -- `units` — unit system for the response -- `status` — status_code, status_message -- `timezone`, `timezone_offset_minutes`, `latitude`, `longitude`, `location_name` - -### Device types in /stations -Devices within a station have a `device_type` field. Known values: -- `HB` — Hub (cannot query observations — no `/observations/device/{id}` endpoint) -- `ST` — Tempest all-in-one (preferred sensor) -- `SKY` — Sky sensor -- `AIR` — Air sensor - -Always filter out HB devices before auto-selecting a device for observation queries. - -### Auth -Token is passed as query parameter: `?token=XXX` -No header-based auth for the swd.weatherflow.com REST API. -Personal access tokens generated at https://weatherflow.com (account → API Tokens). - -### Units -Raw observations use metric (C, m/s, MB, mm). -The `better_forecast` endpoint returns unit-converted values based on station -preferences. The `units` key in the response documents which units are in use. -All temperature values are in **Celsius** regardless of station preference — CLI -must convert to °F if displaying imperial. Unit labels from the API are authority. - -### Field type traps in better_forecast - -The forecast endpoint uses epoch integers where you'd expect date strings, and -field names that differ from what common sense suggests: - -| Field | Actual type | Common mistake | Fix | -|-------|-----------|---------------|-----| -| `daily[].day_start_local` | epoch int (e.g. 1778385600) | Assumed ISO string "2026-05-10T..." | `datetime.fromtimestamp(ts).strftime(...)` | -| `hourly[].local_hour` | int (e.g. 10) | Assumed ISO timestamp string | Use directly as `{h:02d}:00` | -| `hourly[].local_day` | int (e.g. 10 for the 10th) | N/A | Use alongside `local_hour` for time-of-day | -| `hourly[].local_time` | **does not exist** | Commonly assumed field | Use `local_hour` instead | - -The hourly objects do NOT have a `local_time` or `time_string` field — just -`time` (epoch int), `local_day` (int), and `local_hour` (int, 0-23). Any code -looking for `local_time` will silently fall back to its default/"?" branch. diff --git a/tempest/references/udp-broadcast-protocol.md b/tempest/references/udp-broadcast-protocol.md new file mode 100644 index 0000000..3206965 --- /dev/null +++ b/tempest/references/udp-broadcast-protocol.md @@ -0,0 +1,294 @@ +# Tempest UDP Broadcast Protocol (Port 50222) + +The Tempest hub broadcasts JSON messages to the local network on **UDP port +50222**. A listener on the same LAN receives every message the hub publishes: +observations, rapid wind updates, precipitation and lightning events, and +hub/device status. No subscription, pairing, or token is involved — the hub +broadcasts regardless; point a listener at port 50222 and read. + +Positioning per WeatherFlow: REST/WebSocket are the primary data interfaces, +and the UDP broadcast is officially recommended for completely off-grid +applications or as a backup. It is nevertheless the lowest-latency feed on +your LAN (rapid wind arrives every ~3 seconds; hub status roughly once a +minute). + +## Transport facts + +- **Port:** 50222, UDP, local broadcast. Routed/internet reachability is not + enough — the listener must share the hub's L2 network (same subnet/VLAN, or + a DHCP/helper forwarding broadcasts). +- **Direction:** the hub sends, listeners receive. The protocol defines no + acknowledgement or response message; treat it as listen-only. Bind to + `0.0.0.0:50222` with `SO_REUSEADDR` and read datagrams. +- **Framing:** each UDP datagram carries one complete JSON message (UTF-8). + Never concatenate datagrams or expect TCP-style stream framing. (UTF-8 and + one-JSON-per-datagram are the interoperable reading of the protocol's JSON + examples; the official pages do not spell the encoding out.) +- **No auth:** the broadcast carries no token and cannot be restricted from + the hub; anyone on the LAN can read your station's data. This is why the + broadcast is LAN-only. + +## THE dispatch rule: message families are structurally different + +Every message carries a top-level `"type"`. The payload key and array shape +**change with the type** — a parser that blindly indexes a position will +crash or misread. Dispatch on `type` BEFORE indexing: + +| `type` | Payload key | Payload shape | +|---|---|---| +| `obs_st`, `obs_air`, `obs_sky` | `obs` | list containing observation arrays (one per report): `msg["obs"][0][7]` | +| `rapid_wind` | `ob` | ONE 3-element array: `msg["ob"][1]` is wind speed | +| `evt_precip` | `evt` | ONE 1-element array: `msg["evt"][0]` is epoch | +| `evt_strike` | `evt` | ONE 3-element array: epoch, distance km, energy | +| `hub_status` | (named fields) | no payload array: `uptime`, `rssi`, `seq`, `fs`, `radio_stats`, `mqtt_stats` | +| `device_status` | (named fields) | no payload array: `uptime`, `voltage`, `rssi`, `hub_rssi`, `sensor_status` | + +The observation families nest arrays inside a list; `rapid_wind` and the +events carry a single array under a *different key* (`ob` / `evt`); the +status families carry named scalar fields and small status arrays. Iterating +`rapid_wind`'s `ob` array element-wise the way you would `obs` rows is a +classic crash (TypeError on the epoch number) — this is exactly the trap the +dispatch rule exists for. + +## obs_st — Tempest all-in-one observation (UDP form) + +Broadcast roughly once per report interval (default 1 minute). The UDP +datagram carries **18 positions (indices 0–17)**: + +```json +{ + "serial_number": "ST-00000512", + "type": "obs_st", + "hub_sn": "HB-00013030", + "obs": [[1588948614, 0.18, 0.22, 0.27, 144, 6, 1017.57, 22.37, 50.26, 328, 0.03, 3, 0.000000, 0, 0, 0, 2.410, 1]], + "firmware_revision": 129 +} +``` + +| Index | Field | Units | +|---:|---|---| +| 0 | timestamp | epoch seconds, UTC | +| 1 | wind lull (min 3-second sample) | m/s | +| 2 | wind average | m/s | +| 3 | wind gust (max 3-second sample) | m/s | +| 4 | wind direction | degrees (0 = N) | +| 5 | wind sample interval | seconds | +| 6 | station pressure | MB (millibars; numerically identical to hPa) | +| 7 | air temperature | °C | +| 8 | relative humidity | % | +| 9 | illuminance | lux | +| 10 | UV | index | +| 11 | solar radiation | W/m² | +| 12 | rain accumulation over previous minute | mm | +| 13 | precipitation type | 0 none, 1 rain, 2 hail, 3 rain + hail (experimental) | +| 14 | lightning strike average distance | km | +| 15 | lightning strike count | count | +| 16 | battery | volts (≈2.4 nominal; low below ≈2.3) | +| 17 | report interval | minutes | + +**UDP vs REST length:** the REST observation record extends the same array +with four Nearcast/analysis fields — index 18 local-day rain accumulation +(mm), 19 Nearcast rain accumulation (mm), 20 local-day Nearcast rain +accumulation (mm), 21 precipitation analysis type (0 none, 1 Nearcast display +on, 2 off) — for 22 positions total. The UDP broadcast stops at 17. A decoder +must tolerate both lengths (the bundled `decode_obs` does) and never assume +the extra fields exist over UDP. + +## rapid_wind — 3-second wind snapshot + +Broadcast every ~3 seconds between observation reports. Layout differs from +obs_st: payload key is `ob`, a single 3-element array (speed is already +m/s — no conversion on the wire, only when displaying mph): + +```json +{ + "serial_number": "SK-00008453", + "type": "rapid_wind", + "hub_sn": "HB-00000001", + "ob": [1493322445, 2.3, 128] +} +``` + +| Index | Field | Units | +|---:|---|---| +| 0 | timestamp | epoch seconds, UTC | +| 1 | wind speed | m/s | +| 2 | wind direction | degrees | + +## evt_precip — rain-start event + +Fires when the haptic rain sensor detects the start of rainfall (more than +five seconds of continuous rain). Payload key `evt`, one element: + +```json +{ + "serial_number": "SK-00008453", + "type": "evt_precip", + "hub_sn": "HB-00000001", + "evt": [1493322445] +} +``` + +| Index | Field | Units | +|---:|---|---| +| 0 | timestamp | epoch seconds, UTC | + +## evt_strike — lightning strike event + +Payload key `evt`, three elements. The energy unit is not specified in the +official reference: + +```json +{ + "serial_number": "AR-00004049", + "type": "evt_strike", + "hub_sn": "HB-00000001", + "evt": [1493322445, 27, 3848] +} +``` + +| Index | Field | Units | +|---:|---|---| +| 0 | timestamp | epoch seconds, UTC | +| 1 | distance | km | +| 2 | energy | undocumented unit | + +## hub_status — hub heartbeat (roughly once a minute) + +**No payload array at all** — named scalar fields plus small status arrays. +Note `firmware_revision` arrives as a string here (number in observation +messages): + +```json +{ + "serial_number": "HB-00000001", + "type": "hub_status", + "firmware_revision": "35", + "uptime": 1670133, + "rssi": -62, + "timestamp": 1495724691, + "reset_flags": "BOR,PIN,POR", + "seq": 48, + "fs": [1, 0, 15675411, 524288], + "radio_stats": [2, 1, 0, 3, 2839], + "mqtt_stats": [1, 0] +} +``` + +- `uptime` (s), `rssi` (dBm; closer to 0 is stronger), `timestamp` (epoch + seconds), `seq` (monotonic message counter — gaps mean lost datagrams). +- `reset_flags`: comma-separated reset causes — BOR, PIN, POR, SFT, WDG, + WWD, LPW, HRDFLT. Repeated watchdog flags suggest power trouble. +- `radio_stats`: [version, reboot count, I2C bus error count, radio status, + radio network ID]; radio status 0 = off, 1 = on, 3 = active, 7 = BLE + connected. +- `fs` and `mqtt_stats` are documented as internal use. +- There is no `freq` or `fs_version` field in the current protocol (both + appear in old integration notes; do not read them — they are always + `None`). + +## device_status — sensor device health (roughly once a minute) + +Also named fields, no payload array: + +```json +{ + "serial_number": "AR-00004049", + "type": "device_status", + "hub_sn": "HB-00000001", + "timestamp": 1510855923, + "uptime": 2189, + "voltage": 3.50, + "firmware_revision": 17, + "rssi": -17, + "hub_rssi": -87, + "sensor_status": 0, + "debug": 0 +} +``` + +`sensor_status` is a decimal **bit flag** field: bits indicate lightning +failed / noise / disturber, pressure failed, temperature failed, humidity +failed, wind failed, precipitation failed, light/UV failed, plus power-booster +flags. `0` means all sensors healthy. Unknown high bits are reserved — ignore +them rather than erroring. + +## Legacy sensors: obs_air and obs_sky + +Older Air/Sky hardware still broadcasts with the same envelope: + +**obs_air** (`obs` list, 8 positions): 0 epoch · 1 pressure MB · 2 air temp °C +· 3 relative humidity % · 4 lightning strike count · 5 lightning average +distance km · 6 battery volts · 7 report interval minutes. + +```json +{"serial_number": "AR-00004049", "type": "obs_air", "hub_sn": "HB-00000001", + "obs": [[1493164835, 835.0, 10.0, 45, 0, 0, 3.46, 1]], "firmware_revision": 17} +``` + +**obs_sky** (`obs` list, 14 positions): 0 epoch · 1 illuminance lux · 2 UV · +3 rain mm · 4 wind lull m/s · 5 wind avg m/s · 6 wind gust m/s · 7 wind +direction deg · 8 battery volts · 9 report interval min · 10 solar radiation +W/m² · 11 local-day rain mm (**always null over UDP** — REST provides it) · +12 precipitation type · 13 wind sample interval s. + +```json +{"serial_number": "SK-00008453", "type": "obs_sky", "hub_sn": "HB-00000001", + "obs": [[1493321340, 9000, 10, 0.0, 2.6, 4.6, 7.4, 187, 3.12, 1, 130, null, 0, 3]], + "firmware_revision": 29} +``` + +## Units are metric-native — conversion is the caller's job + +Every value on the wire is metric: wind **m/s**, rain **mm**, temperature +**°C**, pressure **MB** (≡ hPa — NOT kPa), distance **km**, illuminance +**lux**, solar radiation **W/m²**, battery **volts**. The UDP protocol ships +no unit-selection and no conversion tables; imperial output is entirely your +code's job. The bundled CLI converts for human display and leaves `--json` +values in the metric-native wire units. Station pressure (raw sensor) is not +sea-level pressure — the Tempest app's "relative pressure" applies an +elevation adjustment you must compute separately if you want it. + +## Minimal listener + +```bash +# See the raw firehose before writing any code: +tempest udp listen --timeout 30 # decodes families, hides hub_status +tempest udp listen --timeout 30 --show-all # include hub_status and unknown types +``` + +```python +# Zero-dependency decoder skeleton — dispatch on type, then index. +import json, socket + +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +sock.bind(("0.0.0.0", 50222)) + +while True: + msg = json.loads(sock.recvfrom(65535)[0].decode("utf-8", errors="replace")) + t = msg.get("type") + if t in ("obs_st", "obs_air", "obs_sky"): + row = msg["obs"][-1] # list of report rows + elif t == "rapid_wind": + row = msg["ob"] # ONE array: [epoch, m/s, degrees] + elif t in ("evt_precip", "evt_strike"): + row = msg["evt"] # ONE array: [epoch] / [epoch, km, energy] + elif t in ("hub_status", "device_status"): + continue # named fields, nothing to index + else: + continue # unknown type: skip, don't crash + print(t, msg.get("serial_number"), row[0]) +``` + +The bundled CLI implements this dispatch in `udp_listen` (see +`scripts/tempest`) with per-family decoders and `--json` output. + +## Sources + +- https://weatherflow.github.io/Tempest/api/udp/v171/ (current UDP protocol reference: all message families, layouts, examples) +- https://weatherflow.github.io/Tempest/api/udp/v143/ (prior protocol revision; family set unchanged) +- https://apidocs.tempestwx.com/reference/tempest-udp-broadcast (UDP documented as backup to REST/WebSocket) +- https://apidocs.tempestwx.com/reference/observation-record-format (REST obs_st Nearcast fields 18–21; evt_strike and rapid_wind record tables) +- https://apidocs.tempestwx.com/reference/quick-start (UDP positioned as backup; REST primary guidance) +- https://help.weatherflow.com/hc/en-us/articles/360052101413-Tempest-FAQs (haptic rain-start behavior, RSSI interpretation, station vs sea-level pressure) diff --git a/tempest/scripts/tempest b/tempest/scripts/tempest index bd5b050..0592f09 100755 --- a/tempest/scripts/tempest +++ b/tempest/scripts/tempest @@ -3,20 +3,27 @@ Two data sources: REST API — stations, observations, forecast via WeatherFlow cloud - UDP/local — real-time broadcast from your hub on port 50222 + (primary, documented source; personal-use token) + UDP/local — real-time JSON broadcast from your hub on port 50222 + (LAN-only backup; no auth, listen-only) -Requires TEMPEST_TOKEN env var (personal access token from weatherflow.com). +Message families over UDP are structurally different (obs_* nest rows under +"obs", rapid_wind carries one array under "ob", evt_* under "evt", +hub_status/device_status use named fields) — decode_message() dispatches on +"type" before any positional indexing. + +Requires TEMPEST_TOKEN env var (personal access token created in the Tempest +web app: Settings -> Data Authorizations). Falls back to ~/.tempest.env. """ import argparse import json import os import socket -import struct import sys import time import warnings -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple # === Suppress dependency warnings before imports === @@ -28,9 +35,31 @@ import requests DEFAULT_SERVER = "https://swd.weatherflow.com/swd/rest" DEFAULT_UDP_PORT = 50222 UDP_BROADCAST_ADDR = "0.0.0.0" +ENV_FILE = os.path.join("~", ".tempest.env") + + +def _load_env_file() -> str: + """Fallback token source: ~/.tempest.env with a TEMPEST_TOKEN= line.""" + path = os.path.expanduser(ENV_FILE) + try: + with open(path, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line.startswith("TEMPEST_TOKEN="): + value = line.split("=", 1)[1].strip() + return value.strip('"').strip("'") + except OSError: + pass + return "" + + +def resolve_token() -> str: + """TEMPEST_TOKEN env var wins; ~/.tempest.env is the fallback.""" + token = os.getenv("TEMPEST_TOKEN", "").strip() + if token: + return token + return _load_env_file() -ENV_TOKEN = os.getenv("TEMPEST_TOKEN", "") -ENV_SERVER = os.getenv("TEMPEST_SERVER", DEFAULT_SERVER) # === Logging === QUIET = False @@ -74,7 +103,7 @@ def _preparse_global_flags(argv: List[str]) -> Tuple[Dict[str, Any], List[str]]: if arg in GLOBAL_BOOLS: flags[arg.lstrip("-").replace("-", "_")] = True i += 1 - elif arg == "--help" or arg == "-h": + elif arg in ("--help", "-h"): return flags, argv # let argparse handle help elif arg == "--": filtered.extend(argv[i:]) @@ -87,15 +116,15 @@ def _preparse_global_flags(argv: List[str]) -> Tuple[Dict[str, Any], List[str]]: # === Tempest API Client === class TempestClient: - """REST API client for WeatherFlow Tempest.""" + """REST API client for WeatherFlow Tempest (swd.weatherflow.com).""" def __init__(self, token: str = "", server: str = "", dry_run: bool = False): - self.token = token or ENV_TOKEN - self.server = (server or ENV_SERVER).rstrip("/") + self.token = token + self.server = (server or os.getenv("TEMPEST_SERVER", DEFAULT_SERVER)).rstrip("/") self.dry_run = dry_run def _get(self, path: str, params: Optional[Dict] = None) -> Any: - """Generic GET with token auth.""" + """Generic GET with token auth (query parameter per official docs).""" url = f"{self.server}{path}" if params is None: params = {} @@ -110,9 +139,10 @@ class TempestClient: die(f"Cannot connect to {self.server}: {e}\n Is the server reachable?") if resp.status_code == 401: - die("Auth failed (401). Check your TEMPEST_TOKEN or generate a new one at weatherflow.com.") + die("Auth failed (401). Check your TEMPEST_TOKEN or create a new one " + "in the Tempest web app (Settings -> Data Authorizations).") if resp.status_code == 403: - die("Forbidden (403). Your token may not have access to this station/device.") + die("Forbidden (403). Your token does not have access to this station/device.") if resp.status_code == 404: die(f"Not found (404) at {path}. Check station/device IDs.") if resp.status_code >= 400: @@ -130,19 +160,37 @@ class TempestClient: # === Endpoints === def get_stations(self) -> List[Dict]: - """List all stations and their devices.""" - data = self._get("/stations") - return data if isinstance(data, list) else data.get("stations", []) + """List all stations and their devices. - def get_observations(self, device_id: int, days_back: int = 0, time_start: Optional[int] = None) -> Dict: - """Get observations for a device. Use days_back=1 for last day, or time_start epoch.""" + The documented response is a StationSet wrapper ({stations: [...], + status}); the legacy model used {locations: [...]} and some proxies + have returned a bare list — unwrap all three shapes defensively. + """ + data = self._get("/stations") + if isinstance(data, list): + return data + if isinstance(data, dict): + for key in ("stations", "locations"): + if isinstance(data.get(key), list): + return data[key] + return [] + + def get_observations(self, device_id: int, days_back: int = 0, + time_start: Optional[int] = None, + time_end: Optional[int] = None) -> Dict: + """Get observations for a device. + + day_offset=N fetches whole UTC day N (0 = today); a time_start/time_end + epoch range overrides it (both, <= 5 days for minute resolution); with + no range parameters the API returns only the latest observation. + """ params: Dict[str, Any] = {} if days_back > 0: params["day_offset"] = days_back elif time_start: params["time_start"] = time_start - else: - params["latest"] = "true" + if time_end: + params["time_end"] = time_end return self._get(f"/observations/device/{device_id}", params) def get_forecast(self, station_id: int) -> Dict: @@ -150,7 +198,10 @@ class TempestClient: return self._get("/better_forecast", {"station_id": station_id}) -# === Observation decoders === +# === Observation decoders (positional arrays; layout per `type`) === +# REST record lengths: obs_st 22, obs_air 8, obs_sky 17. UDP truncates +# obs_st to 18 and obs_sky to 14 (Nearcast fields are REST-only) — decoders +# tolerate both lengths and emit None for missing trailing positions. OBS_ST_FIELDS = [ ("epoch", "seconds_utc"), @@ -166,7 +217,7 @@ OBS_ST_FIELDS = [ ("uv", "index"), ("solar_radiation", "W/m^2"), ("rain_accumulation", "mm"), - ("precipitation_type", "0=none 1=rain 2=hail"), + ("precipitation_type", "0=none 1=rain 2=hail 3=rain+hail"), ("avg_strike_distance", "km"), ("strike_count", "count"), ("battery", "volts"), @@ -201,23 +252,32 @@ OBS_SKY_FIELDS = [ ("report_interval", "minutes"), ("solar_radiation", "W/m^2"), ("local_day_rain_accumulation", "mm"), - ("precipitation_type", "0=none 1=rain 2=hail"), + ("precipitation_type", "0=none 1=rain 2=hail 3=rain+hail"), ("wind_sample_interval", "seconds"), + ("nc_rain_accumulation", "mm"), + ("local_day_nc_rain_accumulation", "mm"), + ("precip_analysis_type", "type"), ] +_OBS_LAYOUTS = { + "obs_st": OBS_ST_FIELDS, + "obs_air": OBS_AIR_FIELDS, + "obs_sky": OBS_SKY_FIELDS, +} + def decode_obs(obs_array: List, type_str: str) -> Dict: - """Decode a raw observation array into a dict with field names.""" - if type_str == "obs_st": - fields = OBS_ST_FIELDS - elif type_str == "obs_air": - fields = OBS_AIR_FIELDS - elif type_str == "obs_sky": - fields = OBS_SKY_FIELDS - else: + """Decode one positional observation array into named fields. + + Values stay metric-native (m/s, mm, C, MB) — conversion is the caller's + job. Rows may be shorter than the full layout (UDP truncates) or longer + (REST extends obs_st); missing positions decode as None. + """ + fields = _OBS_LAYOUTS.get(type_str) + if fields is None: return {f"field_{i}": v for i, v in enumerate(obs_array)} - result = {} + result: Dict[str, Any] = {} for i, (name, unit) in enumerate(fields): val = obs_array[i] if i < len(obs_array) else None if name == "epoch" and val is not None: @@ -238,7 +298,7 @@ def wind_dir_to_cardinal(deg: Optional[float]) -> str: def format_current(obs: Dict) -> str: - """Format current conditions for human display.""" + """Format current conditions for human display (converted from metric).""" lines = [] if obs.get("air_temperature") is not None: temp_c = obs["air_temperature"] @@ -292,10 +352,170 @@ def format_current(obs: Dict) -> str: return "\n".join(lines) if lines else "(no observations)" +# === UDP message-family dispatch === +# Families differ structurally: obs_* nest report rows under "obs"; +# rapid_wind carries ONE array under "ob"; evt_precip/evt_strike carry ONE +# array under "evt"; hub_status/device_status carry named fields only. +# Dispatch on "type" BEFORE any positional indexing. + +def _iso(ts: Any) -> Any: + """Epoch seconds -> ISO string; pass through anything else.""" + if isinstance(ts, (int, float)): + return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat() + return None + + +def _decode_event(msg: Dict, type_str: str) -> Tuple[str, Dict]: + evt = msg.get("evt") or [] + ts = _iso(evt[0]) if len(evt) > 0 else None + payload: Dict[str, Any] = {"type": type_str, + "serial_number": msg.get("serial_number", msg.get("hub_sn", "?")), + "timestamp": ts} + if type_str == "evt_strike": + dist = evt[1] if len(evt) > 1 else None + energy = evt[2] if len(evt) > 2 else None + payload.update({"distance_km": dist, "energy": energy}) + human = f"⚡ Lightning Strike — distance: {dist} km, energy: {energy} [{ts}]" + else: # evt_precip — rain started + human = f"🌧️ Rain started [{ts}]" + return human, payload + + +def _decode_obs_message(msg: Dict, type_str: str) -> List[Tuple[str, Dict]]: + out: List[Tuple[str, Dict]] = [] + label = {"obs_st": "Tempest Observation", "obs_air": "Air Observation", + "obs_sky": "Sky Observation"}.get(type_str, type_str) + sn = msg.get("serial_number", msg.get("hub_sn", "?")) + for obs_arr in msg.get("obs", []) or []: + decoded = decode_obs(obs_arr, type_str) + out.append((f"\n── {label} from {sn} ──\n{format_current(decoded)}", + {"type": type_str, "serial_number": sn, "observation": decoded})) + return out + + +def _decode_rapid_wind(msg: Dict) -> Tuple[str, Dict]: + # rapid_wind payload is a SINGLE 3-element array under "ob": + # [epoch, wind speed m/s, wind direction degrees]. Do NOT iterate it + # element-wise like an obs row list. + ob = msg.get("ob") or [] + sn = msg.get("serial_number", msg.get("hub_sn", "?")) + ts = _iso(ob[0]) if len(ob) > 0 else None + speed = ob[1] if len(ob) > 1 else None + direction = ob[2] if len(ob) > 2 else None + if isinstance(speed, (int, float)) and isinstance(direction, (int, float)): + card = wind_dir_to_cardinal(direction) + mph = speed * 2.237 + human = f"💨 Rapid Wind: {speed} m/s ({mph:.1f} mph) from {card} ({direction}°) [{ts}]" + else: + human = f"💨 Rapid Wind: {speed} m/s from {direction}° [{ts}]" + payload = {"type": "rapid_wind", "serial_number": sn, + "wind_speed_mps": speed, "wind_direction": direction, "timestamp": ts} + return human, payload + + +def _decode_hub_status(msg: Dict) -> Tuple[str, Dict]: + # hub_status carries named fields (uptime, rssi, seq, reset_flags, + # firmware_revision as a string, plus fs/radio_stats/mqtt_stats arrays). + # There is no "freq" or "fs_version" field in the current protocol. + sn = msg.get("serial_number", "?") + payload = {"type": "hub_status", "serial_number": sn, + "firmware_revision": msg.get("firmware_revision"), + "uptime": msg.get("uptime"), "rssi": msg.get("rssi"), + "seq": msg.get("seq"), "reset_flags": msg.get("reset_flags"), + "radio_stats": msg.get("radio_stats")} + human = (f"[hub_status] {sn} — uptime {payload['uptime']}s, " + f"rssi {payload['rssi']}, seq {payload['seq']}, " + f"reset_flags {payload['reset_flags']}") + return human, payload + + +def _decode_device_status(msg: Dict) -> Tuple[str, Dict]: + # device_status: named fields; sensor_status is a decimal bit-flag field + # (0 = all sensors healthy). + sn = msg.get("serial_number", "?") + payload = {"type": "device_status", "serial_number": sn, + "uptime": msg.get("uptime"), "voltage": msg.get("voltage"), + "firmware_revision": msg.get("firmware_revision"), + "rssi": msg.get("rssi"), "hub_rssi": msg.get("hub_rssi"), + "sensor_status": msg.get("sensor_status")} + human = (f"[device_status] {sn} — voltage {payload['voltage']}V, " + f"rssi {payload['rssi']}, hub_rssi {payload['hub_rssi']}, " + f"sensor_status {payload['sensor_status']}") + return human, payload + + +def decode_message(msg: Dict, show_all: bool = False) -> List[Tuple[str, Dict]]: + """Dispatch one decoded UDP datagram on its `type` and decode it. + + Returns a list of (human_text, json_payload) tuples (observations can + carry multiple report rows). Unknown types and status families are + suppressed unless show_all is set. + """ + msg_type = msg.get("type", "unknown") + if msg_type in ("obs_st", "obs_air", "obs_sky"): + return _decode_obs_message(msg, msg_type) + if msg_type == "rapid_wind": + return [_decode_rapid_wind(msg)] + if msg_type in ("evt_precip", "evt_strike"): + return [_decode_event(msg, msg_type)] + if msg_type == "hub_status": + return [_decode_hub_status(msg)] if show_all else [] + if msg_type == "device_status": + return [_decode_device_status(msg)] if show_all else [] + if show_all: + sn = msg.get("serial_number", msg.get("hub_sn", "?")) + return [(f"[{msg_type}] from {sn}", {"type": msg_type, "serial_number": sn, "raw": msg})] + return [] + + +def handle_datagram(data: bytes, show_all: bool = False) -> List[Tuple[str, Dict]]: + """Decode one raw UDP datagram (bytes) — JSON parse, then family dispatch. + + Bad JSON yields [] (or a raw preview when show_all is set). No sockets + are involved, so canned bytes can be fed directly in tests. + """ + try: + msg = json.loads(data.decode("utf-8", errors="replace")) + except json.JSONDecodeError: + if show_all: + preview = data[:200].decode("utf-8", errors="replace") + return [(f"[raw] {preview}", {"type": "unparseable", "raw": preview})] + return [] + if not isinstance(msg, dict): + return [] + return decode_message(msg, show_all) + + # === CLI Commands === -def cmd_stations(client: TempestClient, args: List[str]) -> None: +def _client_for() -> TempestClient: + return TempestClient(token=resolve_token(), dry_run=GLOBAL_FLAGS.get("dry_run", False)) + + +def _pick_device(devices: List[Dict]) -> Dict: + """Auto-select a sensor device: skip hubs, prefer ST, then SKY/SK, then AIR/AR. + + device_type values per the OpenAPI schema are HB/AR/SK/ST; SKY and AIR + are accepted as legacy aliases for SK and AR. + """ + sensors = [d for d in devices if d.get("device_type") not in ("HB", "hub")] + if not sensors: + die("No sensor devices found on this station (only a hub). Hubs carry no observations.") + for preferred in ("ST", "SKY", "SK", "AIR", "AR"): + match = next((d for d in sensors if d.get("device_type") == preferred), None) + if match: + return match + return sensors[0] + + +def cmd_stations(args: argparse.Namespace) -> None: # noqa: ARG001 (uniform handler signature) """List stations and attached devices.""" + if GLOBAL_FLAGS.get("dry_run", False): + emit("[dry-run] Would list stations and devices for your token.", + {"dry_run": True, "command": "stations"}) + return + + client = _client_for() stations = client.get_stations() if not stations: emit("No stations found for this token.", {"stations": []}) @@ -317,28 +537,23 @@ def cmd_stations(client: TempestClient, args: List[str]) -> None: emit("\n".join(output_human), {"stations": stations}) -def cmd_current(client: TempestClient, args: List[str]) -> None: +def cmd_current(args: argparse.Namespace) -> None: """Get latest observations from your station.""" - parser = argparse.ArgumentParser(prog="tempest current") - parser.add_argument("--station-id", type=int, help="Station ID (optional if only one station)") - parser.add_argument("--device-id", type=int, help="Device ID (default: first Tempest device)") - parsed, _ = parser.parse_known_args(args) - - if client.dry_run: + if GLOBAL_FLAGS.get("dry_run", False): emit("[dry-run] Would query latest observations from your station.", {"dry_run": True, "command": "current", - "station_id": parsed.station_id, "device_id": parsed.device_id}) + "station_id": args.station_id, "device_id": args.device_id}) return + client = _client_for() stations = client.get_stations() if not stations: die("No stations found. Verify your TEMPEST_TOKEN.") - # Pick station - if parsed.station_id: - station = next((s for s in stations if s.get("station_id") == parsed.station_id), None) + if args.station_id: + station = next((s for s in stations if s.get("station_id") == args.station_id), None) if not station: - die(f"Station {parsed.station_id} not found.") + die(f"Station {args.station_id} not found.") else: station = stations[0] @@ -346,22 +561,12 @@ def cmd_current(client: TempestClient, args: List[str]) -> None: if not devices: die(f"Station '{station.get('name', '?')}' has no devices.") - # Pick device — skip the hub (HB), prefer ST (Tempest) or SKY/AIR - if parsed.device_id: - device = next((d for d in devices if d.get("device_id") == parsed.device_id), None) + if args.device_id: + device = next((d for d in devices if d.get("device_id") == args.device_id), None) + if not device: + die(f"Device {args.device_id} not found on this station.") else: - # Filter out the hub (device_type=HB), use first sensor - sensors = [d for d in devices if d.get("device_type") not in ("HB", "hub")] - if not sensors: - die("No sensor devices found on this station (only a hub).") - # Prefer Tempest (ST), then Sky, then Air - for preferred in ("ST", "SKY", "AIR"): - match = next((d for d in sensors if d.get("device_type") == preferred), None) - if match: - device = match - break - else: - device = sensors[0] + device = _pick_device(devices) dev_id = device.get("device_id") log(f"Station: {station.get('name', '?')} Device: {device.get('device_type', '?')} (id={dev_id})") @@ -378,61 +583,54 @@ def cmd_current(client: TempestClient, args: List[str]) -> None: decoded = decode_obs(latest, obs_type) if GLOBAL_FLAGS.get("json", False): - emit("", {"station": station.get("name", "?"), "device_id": dev_id, "type": obs_type, "observation": decoded}) + emit("", {"station": station.get("name", "?"), "device_id": dev_id, + "type": obs_type, "observation": decoded}) else: print(format_current(decoded)) -def cmd_obs(client: TempestClient, args: List[str]) -> None: +def cmd_obs(args: argparse.Namespace) -> None: """Get historical observations.""" - parser = argparse.ArgumentParser(prog="tempest obs") - parser.add_argument("--device-id", type=int, required=True, help="Device ID (required)") - parser.add_argument("--days", type=int, default=1, help="Days back to fetch (default: 1)") - parsed, _ = parser.parse_known_args(args) - - if client.dry_run: + if GLOBAL_FLAGS.get("dry_run", False): emit("[dry-run] Would fetch historical observations.", {"dry_run": True, "command": "obs", - "device_id": parsed.device_id, "days": parsed.days}) + "device_id": args.device_id, "days": args.days}) return - obs_data = client.get_observations(parsed.device_id, days_back=parsed.days) + client = _client_for() + obs_data = client.get_observations(args.device_id, days_back=args.days) obs_list = obs_data.get("obs", []) obs_type = obs_data.get("type", "obs_st") decoded = [decode_obs(o, obs_type) for o in obs_list] if GLOBAL_FLAGS.get("json", False): - emit("", {"device_id": parsed.device_id, "type": obs_type, "count": len(decoded), "observations": decoded}) + emit("", {"device_id": args.device_id, "type": obs_type, + "count": len(decoded), "observations": decoded}) else: - print(f"{len(decoded)} observations from the last {parsed.days} day(s) (type: {obs_type}):") + print(f"{len(decoded)} observations from the last {args.days} day(s) (type: {obs_type}):") print("") - # Show latest 3 for o in decoded[-3:]: print("---") print(format_current(o)) print("") -def cmd_forecast(client: TempestClient, args: List[str]) -> None: +def cmd_forecast(args: argparse.Namespace) -> None: """Get forecast — current conditions + daily + hourly.""" - parser = argparse.ArgumentParser(prog="tempest forecast") - parser.add_argument("--station-id", type=int, help="Station ID (optional if only one station)") - parser.add_argument("--days", type=int, default=5, help="Days of daily forecast (default: 5)") - parsed, _ = parser.parse_known_args(args) - - if client.dry_run: + if GLOBAL_FLAGS.get("dry_run", False): emit("[dry-run] Would fetch hyper-local forecast for your station.", {"dry_run": True, "command": "forecast", - "station_id": parsed.station_id, "days": parsed.days}) + "station_id": args.station_id, "days": args.days}) return + client = _client_for() stations = client.get_stations() if not stations: die("No stations found.") - if parsed.station_id: - station = next((s for s in stations if s.get("station_id") == parsed.station_id), None) + if args.station_id: + station = next((s for s in stations if s.get("station_id") == args.station_id), None) else: station = stations[0] if not station: @@ -444,11 +642,23 @@ def cmd_forecast(client: TempestClient, args: List[str]) -> None: data = client.get_forecast(sid) fc_data = data.get("forecast", data) # prefer nested "forecast" key, fall back to top-level + if GLOBAL_FLAGS.get("json", False): emit("", {"station_id": sid, "station_name": name, "forecast": data}) return - # Current conditions + # The response honors unit overrides and reports what it used in `units`. + # Never assume Celsius/m/s: converting an already-imperial response + # double-converts it into absurd values. + units = data.get("units", {}) or {} + temp_is_f = units.get("units_temp", "c") == "f" + wind_unit = units.get("units_wind", "mps") + + def show_temp(value: Optional[float]) -> Optional[float]: + if value is None: + return None + return value if temp_is_f else value * 9 / 5 + 32 + current = data.get("current_conditions", {}) if current: print("── Current Conditions ──") @@ -456,32 +666,26 @@ def cmd_forecast(client: TempestClient, args: List[str]) -> None: cond = current.get("conditions", "") icon_str = f" ({icon})" if icon else "" print(f"Conditions: {cond}{icon_str}") - for key in ("air_temperature", "temperature"): - if current.get(key) is not None: - val = current[key] - feels = current.get("feels_like") - if feels is not None: - feels_f = feels * 9 / 5 + 32 - else: - feels_f = None - print(f"Temperature: {val * 9 / 5 + 32:.0f}°F (feels like {feels_f:.0f}°F)" if feels_f is not None else f"Temperature: {val * 9 / 5 + 32:.0f}°F") - break + temp = show_temp(current.get("air_temperature")) + if temp is not None: + feels = show_temp(current.get("feels_like")) + if feels is not None: + print(f"Temperature: {temp:.0f}°F (feels like {feels:.0f}°F)") + else: + print(f"Temperature: {temp:.0f}°F") if current.get("relative_humidity") is not None: print(f"Humidity: {current['relative_humidity']}%") if current.get("station_pressure") is not None: print(f"Pressure: {current['station_pressure']} MB") if current.get("wind_avg") is not None: wd = current.get("wind_direction_cardinal", "") - print(f"Wind: {current['wind_avg']} mph {wd}") - if current.get("conditions"): - print(f" [{current.get('conditions', '')}]") + print(f"Wind: {current['wind_avg']} {wind_unit} {wd}".rstrip()) print() - # Daily forecast daily = fc_data.get("daily", []) if daily: - print(f"── {parsed.days}-Day Forecast ──") - for day in daily[: parsed.days]: + print(f"── {args.days}-Day Forecast ──") + for day in daily[: args.days]: day_start = day.get("day_start_local") if isinstance(day_start, (int, float)): day_str = datetime.fromtimestamp(day_start).strftime("%a %b %d") @@ -489,28 +693,25 @@ def cmd_forecast(client: TempestClient, args: List[str]) -> None: day_str = str(day_start).split("T")[0] else: day_str = "?" - hi = day.get("air_temp_high") - lo = day.get("air_temp_low") - hi_f = hi * 9 / 5 + 32 if hi is not None else None - lo_f = lo * 9 / 5 + 32 if lo is not None else None + hi = show_temp(day.get("air_temp_high")) + lo = show_temp(day.get("air_temp_low")) cond = day.get("conditions", "") precip = day.get("precip_probability") precip_str = f" {precip}%" if precip is not None else "" precip_type = day.get("precip_type", "") - hi_str = f"{hi_f:.0f}" if hi_f is not None else "?" - lo_str = f"{lo_f:.0f}" if lo_f is not None else "?" + hi_str = f"{hi:.0f}" if hi is not None else "?" + lo_str = f"{lo:.0f}" if lo is not None else "?" print(f" {day_str}: {lo_str}–{hi_str}°F {cond}{precip_str} {precip_type}".strip()) print() - # Hourly hourly = fc_data.get("hourly", []) if hourly: print("── Next 12 Hours ──") for h in hourly[:12]: local_hour = h.get("local_hour") dt_str = f"{local_hour:02d}:00" if local_hour is not None else "?" - temp_c = h.get("air_temperature") - temp_str = f"{temp_c * 9 / 5 + 32:.0f}" if temp_c is not None else "?" + temp = show_temp(h.get("air_temperature")) + temp_str = f"{temp:.0f}" if temp is not None else "?" cond = h.get("conditions", "") precip = h.get("precip_probability") precip_str = f" {precip}%" if precip is not None else "" @@ -521,22 +722,24 @@ def cmd_forecast(client: TempestClient, args: List[str]) -> None: # === UDP Commands === -def udp_listen(args: List[str]) -> None: - """Listen for local UDP broadcasts from the Tempest hub.""" - parser = argparse.ArgumentParser(prog="tempest udp listen") - parser.add_argument("--port", type=int, default=DEFAULT_UDP_PORT, help=f"UDP port (default: {DEFAULT_UDP_PORT})") - parser.add_argument("--timeout", type=int, default=0, help="Listen for N seconds (0 = indefinite)") - parser.add_argument("--show-all", action="store_true", help="Show raw message type even if unknown") - parsed, _ = parser.parse_known_args(args) +def udp_listen(args: argparse.Namespace) -> None: + """Listen for local UDP broadcasts from the Tempest hub (port 50222). + + Listen-only: the hub broadcasts, nothing is ever sent back. Requires + being on the same LAN as the hub — routed connectivity is not enough. + """ + port = args.port + timeout = args.timeout + show_all = args.show_all sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((UDP_BROADCAST_ADDR, parsed.port)) - sock.settimeout(parsed.timeout if parsed.timeout > 0 else None) + sock.bind((UDP_BROADCAST_ADDR, port)) + sock.settimeout(timeout if timeout > 0 else None) - log(f"Listening for Tempest UDP broadcasts on port {parsed.port}...") - if parsed.timeout > 0: - log(f"Will stop after {parsed.timeout}s\n") + log(f"Listening for Tempest UDP broadcasts on port {port}...") + if timeout > 0: + log(f"Will stop after {timeout}s\n") try: start = time.time() @@ -547,63 +750,10 @@ def udp_listen(args: List[str]) -> None: log("Listen timeout reached.") break - try: - msg = json.loads(data.decode("utf-8", errors="replace")) - except json.JSONDecodeError: - if parsed.show_all: - log(f"[raw] from {addr[0]}: {data[:200]}") - continue + for human, payload in handle_datagram(data, show_all=show_all): + emit(human, payload) - msg_type = msg.get("type", "unknown") - sn = msg.get("serial_number", msg.get("hub_sn", "?")) - - if msg_type == "obs_st": - for obs_arr in msg.get("obs", []): - decoded = decode_obs(obs_arr, "obs_st") - emit(f"\n── Tempest Observation from {sn} ──\n{format_current(decoded)}", - {"type": "obs_st", "serial_number": sn, "observation": decoded}) - elif msg_type == "obs_air": - for obs_arr in msg.get("obs", []): - decoded = decode_obs(obs_arr, "obs_air") - emit(f"\n── Air Observation from {sn} ──\n{format_current(decoded)}", - {"type": "obs_air", "serial_number": sn, "observation": decoded}) - elif msg_type == "obs_sky": - for obs_arr in msg.get("obs", []): - decoded = decode_obs(obs_arr, "obs_sky") - emit(f"\n── Sky Observation from {sn} ──\n{format_current(decoded)}", - {"type": "obs_sky", "serial_number": sn, "observation": decoded}) - elif msg_type == "rapid_wind": - for ob in msg.get("ob", []): - ts = datetime.fromtimestamp(ob[0], tz=timezone.utc).isoformat() if len(ob) > 0 else "?" - speed = ob[1] if len(ob) > 1 else "?" - direction = ob[2] if len(ob) > 2 else "?" - card = wind_dir_to_cardinal(direction) - m_s_mph = f"{speed * 2.237:.1f} mph" if isinstance(speed, (int, float)) else "?" - emit(f"💨 Rapid Wind: {speed} m/s ({m_s_mph}) from {card} ({direction}°) [{ts}]", - {"type": "rapid_wind", "serial_number": sn, - "wind_speed_mps": speed, "wind_direction": direction, - "timestamp": ts}) - elif msg_type == "evt_strike": - evt = msg.get("evt", []) - ts = datetime.fromtimestamp(evt[0], tz=timezone.utc).isoformat() if len(evt) > 0 else "?" - dist = evt[1] if len(evt) > 1 else "?" - energy = evt[2] if len(evt) > 2 else "?" - emit(f"⚡ Lightning Strike — distance: {dist} km, energy: {energy} [{ts}]", - {"type": "evt_strike", "serial_number": sn, - "distance_km": dist, "energy": energy, "timestamp": ts}) - elif msg_type == "evt_precip": - evt = msg.get("evt", []) - ts = datetime.fromtimestamp(evt[0], tz=timezone.utc).isoformat() if len(evt) > 0 else "?" - emit(f"🌧️ Rain started [{ts}]", - {"type": "evt_precip", "serial_number": sn, "timestamp": ts}) - elif msg_type == "hub_status": - if parsed.show_all: - emit(f"[hub_status] {sn} — freq: {msg.get('freq', '?')}", - {"type": "hub_status", "serial_number": sn, "status": msg}) - elif parsed.show_all: - emit(f"[{msg_type}] from {sn}", {"type": msg_type, "serial_number": sn, "raw": msg}) - - if parsed.timeout > 0 and time.time() - start > parsed.timeout: + if timeout > 0 and time.time() - start > timeout: break except KeyboardInterrupt: @@ -612,16 +762,9 @@ def udp_listen(args: List[str]) -> None: sock.close() -# === Main === - -def main() -> None: - global GLOBAL_FLAGS, QUIET - GLOBAL_FLAGS, filtered_argv = _preparse_global_flags(sys.argv) - if GLOBAL_FLAGS.get("quiet", False): - QUIET = True - if GLOBAL_FLAGS.get("json", False): - warnings.simplefilter("ignore") +# === Parser === +def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="tempest", description="Hyper-local weather from your Tempest station.", @@ -629,67 +772,73 @@ def main() -> None: ) sub = parser.add_subparsers(dest="command", help="Available commands") - # stations sub.add_parser("stations", help="List stations and devices linked to your token") - # current p_current = sub.add_parser("current", help="Latest observations from your station") p_current.add_argument("--station-id", type=int, help="Station ID (optional)") p_current.add_argument("--device-id", type=int, help="Device ID (optional)") - # obs p_obs = sub.add_parser("obs", help="Historical observations") p_obs.add_argument("--device-id", type=int, required=True, help="Device ID") p_obs.add_argument("--days", type=int, default=1, help="Days back (default: 1)") - # forecast p_fcst = sub.add_parser("forecast", help="Hyper-local forecast (current + daily + hourly)") p_fcst.add_argument("--station-id", type=int, help="Station ID (optional)") p_fcst.add_argument("--days", type=int, default=5, help="Days of daily forecast (default: 5)") - # udp - p_udp = sub.add_parser("udp", help="Local UDP broadcast commands") + p_udp = sub.add_parser("udp", help="Local UDP broadcast commands (port 50222, listen-only)") udp_sub = p_udp.add_subparsers(dest="udp_command") p_listen = udp_sub.add_parser("listen", help="Listen for local UDP broadcasts from hub") p_listen.add_argument("--port", type=int, default=DEFAULT_UDP_PORT) p_listen.add_argument("--timeout", type=int, default=0, help="Listen N seconds (0=indefinite)") - p_listen.add_argument("--show-all", action="store_true", help="Show all message types incl. hub_status") + p_listen.add_argument("--show-all", action="store_true", + help="Show all message types incl. hub_status/device_status") + return parser + + +_HANDLERS = { + "stations": cmd_stations, + "current": cmd_current, + "obs": cmd_obs, + "forecast": cmd_forecast, + "udp": udp_listen, +} + + +# === Main === + +def main(argv: Optional[List[str]] = None) -> None: + global QUIET + base_argv = argv if argv is not None else sys.argv + GLOBAL_FLAGS.clear() + GLOBAL_FLAGS.update({"json": False, "dry_run": False, "force": False, "quiet": False, "verbose": False}) + flags, filtered_argv = _preparse_global_flags(base_argv) + GLOBAL_FLAGS.update(flags) + QUIET = bool(GLOBAL_FLAGS.get("quiet", False)) + if GLOBAL_FLAGS.get("json", False): + warnings.simplefilter("ignore") + + parser = build_parser() args = parser.parse_args(filtered_argv[1:]) if not args.command: parser.print_help() sys.exit(1) - if args.command == "stations": - if not ENV_TOKEN and not GLOBAL_FLAGS.get("dry_run", False): - die("TEMPEST_TOKEN not set. Get one at https://weatherflow.com") - cmd_stations(TempestClient(dry_run=GLOBAL_FLAGS.get("dry_run", False)), []) + if args.command == "udp": + if args.udp_command != "listen": + parser.error("udp requires a subcommand: listen") + # UDP listening needs no token — the hub broadcast is unauthenticated. + udp_listen(args) + return - elif args.command == "current": - if not ENV_TOKEN and not GLOBAL_FLAGS.get("dry_run", False): - die("TEMPEST_TOKEN not set.") - cmd_current(TempestClient(dry_run=GLOBAL_FLAGS.get("dry_run", False)), - filtered_argv[filtered_argv.index("current") + 1:]) + # REST commands need a token unless this is a dry-run plan. + if not resolve_token() and not GLOBAL_FLAGS.get("dry_run", False): + die("TEMPEST_TOKEN not set. Get one at https://weatherflow.com " + "(Tempest web app -> Settings -> Data Authorizations) or export TEMPEST_TOKEN.") - elif args.command == "obs": - if not ENV_TOKEN and not GLOBAL_FLAGS.get("dry_run", False): - die("TEMPEST_TOKEN not set.") - cmd_obs(TempestClient(dry_run=GLOBAL_FLAGS.get("dry_run", False)), - filtered_argv[filtered_argv.index("obs") + 1:]) - - elif args.command == "forecast": - if not ENV_TOKEN and not GLOBAL_FLAGS.get("dry_run", False): - die("TEMPEST_TOKEN not set.") - cmd_forecast(TempestClient(dry_run=GLOBAL_FLAGS.get("dry_run", False)), - filtered_argv[filtered_argv.index("forecast") + 1:]) - - elif args.command == "udp": - if args.udp_command == "listen": - udp_listen(filtered_argv[filtered_argv.index("listen") + 1:]) - else: - p_udp.print_help() - sys.exit(1) + _HANDLERS[args.command](args) if __name__ == "__main__": diff --git a/tempest/scripts/test_tempest.py b/tempest/scripts/test_tempest.py new file mode 100644 index 0000000..edfe085 --- /dev/null +++ b/tempest/scripts/test_tempest.py @@ -0,0 +1,646 @@ +"""Offline test suite for the bundled tempest CLI. + +All HTTP is mocked at the client seam (TempestClient._get is replaced by a +FakeTransport that records paths/params and returns canned REST documents), +and UDP paths are tested by feeding CANNED DATAGRAM BYTES to the pure +handle_datagram()/decode_message() decoders — no socket is ever created or +bound (the suite never touches socket.socket). The suite is fully offline and +passes the proxy-trap rerun. Tempest is a keyed API, so there are deliberately +NO live-call test cases (the AGENTS.md network policy is mock-everything for +keyed APIs). + +Covers the four contract behavior classes: --help output, argument-error +paths, --dry-run plans, and mocked-client logic — plus the documented +multi-step pipelines (stations -> current, stations -> forecast with unit +conversion, obs day history) and every UDP message family (obs_st UDP 18 +positions vs REST 22, obs_air, obs_sky, rapid_wind's single "ob" array, +evt_precip/evt_strike's single "evt" arrays, hub_status/device_status named +fields) dispatched by type. +""" + +import contextlib +import importlib.machinery +import importlib.util +import io +import json +import pathlib +import sys +import unittest +from unittest.mock import patch + +SCRIPT = pathlib.Path(__file__).resolve().parent / "tempest" +LOADER = importlib.machinery.SourceFileLoader("tempest_cli", str(SCRIPT)) +SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER) +ts = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = ts # so unittest.mock.patch("tempest_cli....") resolves +LOADER.exec_module(ts) + +# --------------------------------------------------------------------------- +# Canned UDP datagrams (bytes, exactly as the hub broadcasts them). +# obs_st uses the documented 18-position UDP record; REST returns 22. +# --------------------------------------------------------------------------- + +OBS_ST_DATAGRAM = ( + b'{"serial_number":"ST-00000512","type":"obs_st","hub_sn":"HB-00013030",' + b'"obs":[[1588948614,0.18,0.22,0.27,144,6,1017.57,22.37,50.26,328,0.03,3,' + b'0.0,0,0,0,2.410,1]],"firmware_revision":129}' +) +RAPID_WIND_DATAGRAM = ( + b'{"serial_number":"SK-00008453","type":"rapid_wind","hub_sn":"HB-00000001",' + b'"ob":[1493322445,2.3,128]}' +) +EVT_PRECIP_DATAGRAM = ( + b'{"serial_number":"SK-00008453","type":"evt_precip","hub_sn":"HB-00000001",' + b'"evt":[1493322445]}' +) +EVT_STRIKE_DATAGRAM = ( + b'{"serial_number":"AR-00004049","type":"evt_strike","hub_sn":"HB-00000001",' + b'"evt":[1493322445,27,3848]}' +) +HUB_STATUS_DATAGRAM = ( + b'{"serial_number":"HB-00000001","type":"hub_status","firmware_revision":"35",' + b'"uptime":1670133,"rssi":-62,"timestamp":1495724691,"reset_flags":"BOR,PIN,POR",' + b'"seq":48,"fs":[1,0,15675411,524288],"radio_stats":[2,1,0,3,2839],"mqtt_stats":[1,0]}' +) +DEVICE_STATUS_DATAGRAM = ( + b'{"serial_number":"AR-00004049","type":"device_status","hub_sn":"HB-00000001",' + b'"timestamp":1510855923,"uptime":2189,"voltage":3.50,"firmware_revision":17,' + b'"rssi":-17,"hub_rssi":-87,"sensor_status":0,"debug":0}' +) +OBS_AIR_DATAGRAM = ( + b'{"serial_number":"AR-00004049","type":"obs_air","hub_sn":"HB-00000001",' + b'"obs":[[1493164835,835.0,10.0,45,0,0,3.46,1]],"firmware_revision":17}' +) +OBS_SKY_DATAGRAM = ( + b'{"serial_number":"SK-00008453","type":"obs_sky","hub_sn":"HB-00000001",' + b'"obs":[[1493321340,9000,10,0.0,2.6,4.6,7.4,187,3.12,1,130,null,0,3]],' + b'"firmware_revision":29}' +) +GARBAGE_DATAGRAM = b"\x00\x01not-json-at-all" + + +def run_main(argv): + """Run the CLI main() with patched stdout; returns (exit_code, stdout). + + SystemExit is caught and converted to a code so error paths can assert + on exit codes without exception plumbing. + """ + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): + try: + ts.main(["tempest"] + argv) + code = 0 + except SystemExit as exc: + code = exc.code if isinstance(exc.code, int) else (0 if exc.code is None else 1) + return code, out.getvalue() + + +def run_main_err(argv): + """Like run_main but also captures stderr: (exit_code, stdout, stderr).""" + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + ts.main(["tempest"] + argv) + code = 0 + except SystemExit as exc: + code = exc.code if isinstance(exc.code, int) else (0 if exc.code is None else 1) + return code, out.getvalue(), err.getvalue() + + +# --------------------------------------------------------------------------- +# Fake REST transport: records requests, replays canned documents +# --------------------------------------------------------------------------- + +STATION_DOC = { + "status": {"status_code": 0, "status_message": "SUCCESS"}, + "stations": [{ + "station_id": 12799, "name": "Home", "public_name": "Home", + "latitude": 42.37, "longitude": -71.06, + "timezone": "America/New_York", "timezone_offset_minutes": -300, + "station_meta": {"elevation": 1567.65, "share_with_wf": True, "share_with_wu": True}, + "is_local_mode": False, + "devices": [ + {"device_id": 60526, "serial_number": "ST-00012345", "device_type": "ST", + "hardware_revision": "3", "firmware_revision": "165", + "device_meta": {"agl": 2.2, "name": "Backyard", "environment": "outdoor"}}, + {"device_id": 60500, "serial_number": "HB-00000001", "device_type": "HB", + "hardware_revision": "3", "firmware_revision": "35", + "device_meta": {"name": "Hub"}}, + {"device_id": 60599, "serial_number": None, "device_type": "SK", + "hardware_revision": "2", "firmware_revision": "29", + "device_meta": {"name": "Old Sky"}}, + ], + "station_items": [], + }], +} + + +class FakeTransport: + """Replaces TempestClient._get; records every request, replays canned docs.""" + + def __init__(self, responses=None): + self.requests = [] + self.responses = responses or {} + + def __call__(self, path, params=None): + self.requests.append({"path": path, "params": dict(params or {})}) + if path in self.responses: + return self.responses[path] + if path.startswith("/observations/device/"): + return {"obs": [OBS_ROW_ST], "type": "obs_st"} + if path == "/better_forecast": + return FORECAST_DOC + if path == "/stations": + return STATION_DOC + raise AssertionError(f"unexpected path {path}") + + +# Canned REST documents +OBS_ROW_ST = [1650843455, 0.18, 0.22, 0.27, 144, 6, 1017.57, 22.37, 50.26, 328, + 0.03, 3, 0.0, 0, 0, 0, 2.410, 1, 5.2, 4.8, 5.2, 1] +FORECAST_DOC = { + "status": {"status_code": 0, "status_message": "SUCCESS"}, + "current_conditions": {"air_temperature": 18.2, "conditions": "Mostly Clear", + "icon": "partly-cloudy-day", "relative_humidity": 61, + "station_pressure": 1015.4, "wind_avg": 2.1, + "wind_direction": 225, "wind_direction_cardinal": "SW", + "feels_like": 18.2}, + "forecast": { + "daily": [ + {"day_start_local": 1778385600, "air_temp_high": 25.4, "air_temp_low": 15.1, + "conditions": "Partly cloudy", "precip_probability": 10, "precip_type": "rain", + "sunrise": 1778378400, "sunset": 1778425200}, + {"day_start_local": 1778472000, "air_temp_high": 22.0, "air_temp_low": 12.0, + "conditions": "Rainy", "precip_probability": 80, "precip_type": "rain"}, + ], + "hourly": [ + {"time": 1778388000, "local_hour": 10, "local_day": 10, "air_temperature": 19.8, + "precip_probability": 5, "conditions": "Sunny"}, + {"time": 1778391600, "local_hour": 11, "local_day": 10, "air_temperature": 20.4, + "precip_probability": 45, "conditions": "Cloudy"}, + ], + }, + "units": {"units_temp": "c", "units_wind": "mps", "units_precip": "mm", + "units_pressure": "mb", "units_distance": "km"}, + "latitude": 42.37, "longitude": -71.06, + "timezone": "America/New_York", "timezone_offset_minutes": -300, +} +FORECAST_DOC_F = json.loads(json.dumps(FORECAST_DOC)) +FORECAST_DOC_F["units"] = {"units_temp": "f", "units_wind": "mph", "units_precip": "in", + "units_pressure": "inhg", "units_distance": "mi"} +FORECAST_DOC_F["current_conditions"]["air_temperature"] = 64.8 +FORECAST_DOC_F["forecast"]["daily"][0]["air_temp_high"] = 77.7 + +STATIONS_ONLY = {"/stations": STATION_DOC} + + +def patch_token(token="tok-test"): + return patch.object(ts, "resolve_token", return_value=token) + + +class CliTestCase(unittest.TestCase): + """Base: fresh GLOBAL_FLAGS per test, stdout captured via run_main.""" + + def setUp(self): + ts.GLOBAL_FLAGS.clear() + ts.GLOBAL_FLAGS.update( + {"json": False, "dry_run": False, "force": False, "quiet": False, "verbose": False}) + ts.QUIET = False + + +# --------------------------------------------------------------------------- +# Class 1: --help output +# --------------------------------------------------------------------------- + +class HelpTests(CliTestCase): + def test_help_lists_all_subcommands(self): + code, out = run_main(["--help"]) + self.assertEqual(code, 0) + for noun in ("stations", "current", "obs", "forecast", "udp"): + self.assertIn(noun, out) + + def test_udp_help_documents_listen(self): + code, out = run_main(["udp", "--help"]) + self.assertEqual(code, 0) + self.assertIn("listen", out) + self.assertIn("50222", out + ts.build_parser().format_help()) + + def test_forecast_help_shows_flags(self): + code, out = run_main(["forecast", "--help"]) + self.assertEqual(code, 0) + self.assertIn("--station-id", out) + self.assertIn("--days", out) + + def test_main_help_epilog_documents_global_flag_positions(self): + code, out = run_main(["--help"]) + self.assertEqual(code, 0) + self.assertIn("anywhere", out) + + +# --------------------------------------------------------------------------- +# Class 2: argument-error paths +# --------------------------------------------------------------------------- + +class ArgumentErrorsTests(CliTestCase): + def test_no_command_prints_help_and_exits_1(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + with self.assertRaises(SystemExit) as ctx: + ts.main(["tempest"]) + self.assertEqual(ctx.exception.code, 1) + self.assertIn("usage", out.getvalue()) + + def test_udp_without_subcommand_is_an_error(self): + code, _, err = run_main_err(["udp"]) + self.assertEqual(code, 2) + self.assertIn("udp requires a subcommand", err) + + def test_obs_requires_device_id(self): + code, _, err = run_main_err(["obs"]) + self.assertEqual(code, 2) + self.assertIn("--device-id", err) + + def test_missing_token_dies_with_guidance(self): + with patch_token(""): + code, _, err = run_main_err(["stations"]) + self.assertEqual(code, 1) + self.assertIn("TEMPEST_TOKEN not set", err) + + def test_missing_token_is_fine_for_dry_run(self): + with patch_token(""): + code, out = run_main(["stations", "--dry-run", "--json"]) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out)["dry_run"], True) + + def test_unknown_station_id_exits_1(self): + fake = FakeTransport(STATIONS_ONLY) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + code, _, err = run_main_err(["current", "--station-id", "99999999"]) + self.assertEqual(code, 1) + self.assertIn("99999999 not found", err) + + +# --------------------------------------------------------------------------- +# Class 3: --dry-run behavior (plans are JSON, exit 0, zero network) +# --------------------------------------------------------------------------- + +class DryRunTests(CliTestCase): + def test_current_dry_run_plan_shape(self): + code, out = run_main(["current", "--station-id", "12799", "--device-id", "60526", + "--dry-run", "--json"]) + self.assertEqual(code, 0) + plan = json.loads(out) + self.assertEqual(plan["dry_run"], True) + self.assertEqual(plan["command"], "current") + self.assertEqual(plan["station_id"], 12799) + self.assertEqual(plan["device_id"], 60526) + + def test_forecast_dry_run_plan_shape(self): + code, out = run_main(["forecast", "--station-id", "12799", "--days", "3", + "--dry-run", "--json"]) + self.assertEqual(code, 0) + plan = json.loads(out) + self.assertEqual(plan["command"], "forecast") + self.assertEqual(plan["days"], 3) + + def test_obs_dry_run_plan_shape(self): + code, out = run_main(["obs", "--device-id", "60526", "--days", "2", "--dry-run", "--json"]) + self.assertEqual(code, 0) + plan = json.loads(out) + self.assertEqual(plan["command"], "obs") + self.assertEqual(plan["device_id"], 60526) + self.assertEqual(plan["days"], 2) + + def test_stations_dry_run_plan_shape(self): + code, out = run_main(["stations", "--dry-run", "--json"]) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out)["command"], "stations") + + +# --------------------------------------------------------------------------- +# Class 4: mocked REST client logic (no real network anywhere) +# --------------------------------------------------------------------------- + +class RestClientTests(CliTestCase): + def test_stations_json_unwraps_stationset_wrapper(self): + fake = FakeTransport(STATIONS_ONLY) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + code, out = run_main(["stations", "--json"]) + self.assertEqual(code, 0) + doc = json.loads(out) + self.assertEqual(doc["stations"][0]["station_id"], 12799) + self.assertEqual(len(doc["stations"][0]["devices"]), 3) + + def test_current_pipeline_stations_then_latest_observation(self): + fake = FakeTransport(STATIONS_ONLY) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + code, out = run_main(["current", "--json"]) + self.assertEqual(code, 0) + doc = json.loads(out) + # auto-selection picked the ST device and skipped the HB hub + self.assertEqual(doc["device_id"], 60526) + self.assertEqual(doc["type"], "obs_st") + obs = doc["observation"] + self.assertIsInstance(obs["air_temperature"], (int, float)) + self.assertEqual(obs["air_temperature"], 22.37) + self.assertEqual(obs["air_temperature_unit"], "C") + self.assertEqual(fake.requests[-1]["path"], "/observations/device/60526") + # latest-only mode sends no day_offset / time range + self.assertNotIn("day_offset", fake.requests[-1]["params"]) + + def test_current_positional_flag_consumption_from_handler_argv(self): + # handler-owns-flags dispatch: "--device-id 60526" after "current" + fake = FakeTransport(STATIONS_ONLY) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + code, out = run_main(["current", "--device-id", "60526", "--json"]) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out)["device_id"], 60526) + + def test_obs_pipeline_requests_day_offset_and_decodes_rows(self): + fake = FakeTransport(STATIONS_ONLY) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + code, out = run_main(["obs", "--device-id", "60526", "--days", "2", "--json"]) + self.assertEqual(code, 0) + doc = json.loads(out) + self.assertEqual(doc["count"], 1) + self.assertEqual(doc["observations"][0]["local_day_rain_accumulation"], 5.2) + self.assertEqual(fake.requests[-1]["params"]["day_offset"], 2) + + def test_forecast_pipeline_reads_nested_forecast_key(self): + fake = FakeTransport(STATIONS_ONLY) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + code, out = run_main(["forecast", "--json"]) + self.assertEqual(code, 0) + doc = json.loads(out) + self.assertEqual(doc["station_id"], 12799) + daily = doc["forecast"]["forecast"]["daily"] + self.assertEqual(daily[0]["air_temp_high"], 25.4) + self.assertEqual(doc["forecast"]["units"]["units_temp"], "c") + + def test_forecast_human_output_celsius_station_converts_to_f(self): + fake = FakeTransport(STATIONS_ONLY) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + code, out = run_main(["forecast"]) + self.assertEqual(code, 0) + # 25.4C * 9/5 + 32 = 77.72 -> displayed as 78 with :.0f + self.assertIn("78", out) + # hourly local_hour rendered HH:00 + self.assertIn("10:00", out) + + def test_forecast_human_output_fahrenheit_station_not_double_converted(self): + fake = FakeTransport({**STATIONS_ONLY, "/better_forecast": FORECAST_DOC_F}) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + code, out = run_main(["forecast"]) + self.assertEqual(code, 0) + # units_temp=f: 77.7 stays 77.7 -> displayed 78; a double conversion + # would render 172 (77.7*9/5+32), which must not appear. + self.assertIn("78", out) + self.assertNotIn("172", out) + + def test_client_sends_token_as_query_parameter(self): + # Documented auth: token travels as a query parameter (apiKey in:query), + # never as a header. Verified at the requests.get seam. + class FakeResp: + status_code = 200 + text = "" + def json(self): + return STATION_DOC + recorded = {} + + def fake_get(url, params=None, timeout=None): + recorded["url"] = url + recorded["params"] = params + return FakeResp() + + with patch.object(ts.requests, "get", side_effect=fake_get): + ts.TempestClient(token="tok-query").get_stations() + self.assertEqual(recorded["params"]["token"], "tok-query") + self.assertIn("/stations", recorded["url"]) + self.assertIn("swd.weatherflow.com", recorded["url"]) + + def test_client_401_message_names_token(self): + class Err401: + status_code = 401 + text = "" + with patch.object(ts.requests, "get", return_value=Err401()): + client = ts.TempestClient(token="bad") + with contextlib.redirect_stderr(io.StringIO()) as err: + with self.assertRaises(SystemExit) as ctx: + client._get("/stations") + self.assertEqual(ctx.exception.code, 1) + self.assertIn("401", err.getvalue()) + + def test_env_file_fallback_token(self): + with patch.dict("os.environ", {"TEMPEST_TOKEN": ""}), \ + patch.object(ts, "ENV_FILE", "/nonexistent/.tempest.env"): + self.assertEqual(ts.resolve_token(), "") + with patch.dict("os.environ", {"TEMPEST_TOKEN": " "}), \ + patch.object(ts, "ENV_FILE", "/nonexistent/.tempest.env"): + self.assertEqual(ts.resolve_token(), "") + + +# --------------------------------------------------------------------------- +# UDP decoding from canned datagram bytes — no sockets, no binds +# --------------------------------------------------------------------------- + +class UdpDecoderTests(CliTestCase): + def test_obs_st_datagram_decodes_all_18_udp_positions(self): + results = ts.handle_datagram(OBS_ST_DATAGRAM) + self.assertEqual(len(results), 1) + _, payload = results[0] + self.assertEqual(payload["type"], "obs_st") + self.assertEqual(payload["serial_number"], "ST-00000512") + obs = payload["observation"] + self.assertEqual(obs["epoch"], 1588948614) + self.assertEqual(obs["wind_avg"], 0.22) # index 2 + self.assertEqual(obs["wind_direction"], 144) # index 4 + self.assertEqual(obs["station_pressure"], 1017.57) # index 6 (MB) + self.assertEqual(obs["air_temperature"], 22.37) # index 7 (C) + self.assertEqual(obs["rain_accumulation"], 0.0) # index 12 (mm) + self.assertEqual(obs["battery"], 2.410) # index 16 + self.assertEqual(obs["report_interval"], 1) # index 17 (last UDP position) + # UDP record ends at index 17: REST-only Nearcast fields decode as None + self.assertIsNone(obs["nc_rain_accumulation"]) + self.assertIsNone(obs["precip_analysis_type"]) + # metric-native units preserved on the payload + self.assertEqual(obs["wind_avg_unit"], "m/s") + self.assertEqual(obs["air_temperature_unit"], "C") + self.assertEqual(obs["rain_accumulation_unit"], "mm") + + def test_decode_obs_handles_full_rest_22_position_row(self): + decoded = ts.decode_obs(OBS_ROW_ST, "obs_st") + self.assertEqual(decoded["local_day_rain_accumulation"], 5.2) + self.assertEqual(decoded["nc_rain_accumulation"], 4.8) + self.assertEqual(decoded["precip_analysis_type"], 1) + + def test_decode_obs_tolerates_short_rows_with_none(self): + decoded = ts.decode_obs([1588948614, 0.18, 0.22], "obs_st") + self.assertEqual(decoded["wind_avg"], 0.22) + self.assertIsNone(decoded["air_temperature"]) + self.assertIsNone(decoded["battery"]) + + def test_rapid_wind_single_ob_array_not_iterated_elementwise(self): + # Regression: the old handler iterated msg["ob"] like an obs row list + # (TypeError: unsupported operand type(s) for -: 'int' and 'str'-style + # crash on the epoch number). rapid_wind carries ONE array under "ob". + results = ts.handle_datagram(RAPID_WIND_DATAGRAM) + self.assertEqual(len(results), 1) + _, payload = results[0] + self.assertEqual(payload["type"], "rapid_wind") + self.assertEqual(payload["wind_speed_mps"], 2.3) + self.assertEqual(payload["wind_direction"], 128) + self.assertIsNotNone(payload["timestamp"]) + + def test_evt_precip_single_evt_array(self): + results = ts.handle_datagram(EVT_PRECIP_DATAGRAM) + self.assertEqual(len(results), 1) + _, payload = results[0] + self.assertEqual(payload["type"], "evt_precip") + self.assertIsNotNone(payload["timestamp"]) + + def test_evt_strike_distance_and_energy(self): + results = ts.handle_datagram(EVT_STRIKE_DATAGRAM) + _, payload = results[0] + self.assertEqual(payload["type"], "evt_strike") + self.assertEqual(payload["distance_km"], 27) + self.assertEqual(payload["energy"], 3848) + + def test_hub_status_named_fields_dispatch(self): + # hub_status carries named fields (no payload array). The old handler + # printed msg["freq"], which does not exist in the current protocol. + results = ts.handle_datagram(HUB_STATUS_DATAGRAM, show_all=True) + self.assertEqual(len(results), 1) + _, payload = results[0] + self.assertEqual(payload["type"], "hub_status") + self.assertEqual(payload["serial_number"], "HB-00000001") + self.assertEqual(payload["uptime"], 1670133) + self.assertEqual(payload["reset_flags"], "BOR,PIN,POR") + self.assertEqual(payload["radio_stats"], [2, 1, 0, 3, 2839]) + + def test_hub_status_hidden_by_default(self): + self.assertEqual(ts.handle_datagram(HUB_STATUS_DATAGRAM), []) + + def test_device_status_named_fields(self): + results = ts.handle_datagram(DEVICE_STATUS_DATAGRAM, show_all=True) + _, payload = results[0] + self.assertEqual(payload["type"], "device_status") + self.assertEqual(payload["voltage"], 3.50) + self.assertEqual(payload["sensor_status"], 0) + + def test_obs_air_and_obs_sky_dispatch(self): + air = ts.handle_datagram(OBS_AIR_DATAGRAM)[0][1] + self.assertEqual(air["observation"]["station_pressure"], 835.0) + self.assertEqual(air["observation"]["air_temperature"], 10.0) + sky = ts.handle_datagram(OBS_SKY_DATAGRAM)[0][1] + self.assertEqual(sky["observation"]["illuminance"], 9000) + self.assertIsNone(sky["observation"]["local_day_rain_accumulation"]) # null over UDP + + def test_garbage_datagram_returns_no_results(self): + self.assertEqual(ts.handle_datagram(GARBAGE_DATAGRAM), []) + # show_all surfaces a raw preview instead of crashing + results = ts.handle_datagram(GARBAGE_DATAGRAM, show_all=True) + self.assertEqual(len(results), 1) + self.assertEqual(results[0][1]["type"], "unparseable") + + def test_unknown_type_ignored_by_default_and_listed_with_show_all(self): + weird = b'{"type":"something_new","serial_number":"XX-1"}' + self.assertEqual(ts.handle_datagram(weird), []) + results = ts.handle_datagram(weird, show_all=True) + self.assertEqual(results[0][1]["type"], "something_new") + + def test_decode_message_dispatches_on_type_before_indexing(self): + # non-obs families must never be routed into the obs positional decoder + self.assertEqual(ts.decode_message({"type": "rapid_wind", "ob": [1, 2.3, 128]})[0][1]["wind_speed_mps"], 2.3) + self.assertEqual(ts.decode_message({"type": "evt_precip", "evt": [1493322445]})[0][1]["type"], "evt_precip") + self.assertEqual(ts.decode_message({"type": "hub_status", "uptime": 5, "seq": 1}, show_all=True)[0][1]["type"], "hub_status") + + def test_listen_handler_consumes_canned_datagrams_without_sockets(self): + # udp_listen's socket is fully mocked: canned datagram BYTES are fed + # to the decoder through a fake recvfrom, so the suite never creates + # or binds a real socket anywhere. + canned = [OBS_ST_DATAGRAM, RAPID_WIND_DATAGRAM, EVT_PRECIP_DATAGRAM] + fake_sock = unittest.mock.MagicMock() + fake_sock.recvfrom.side_effect = [ + (canned[0], ("127.0.0.1", 50222)), + (canned[1], ("127.0.0.1", 50222)), + (canned[2], ("127.0.0.1", 50222)), + ts.socket.timeout("stop"), + ] + out = io.StringIO() + args = type("A", (), {"port": 50222, "timeout": 1, "show_all": False})() + with contextlib.redirect_stdout(out): + with patch.object(ts.socket, "socket", return_value=fake_sock): + ts.udp_listen(args) + text = out.getvalue() + self.assertIn("ST-00000512", text) + self.assertIn("Rapid Wind", text) + self.assertIn("Rain started", text) + fake_sock.close.assert_called_once() + + def test_listen_json_stream_carries_family_payloads(self): + fake_sock = unittest.mock.MagicMock() + fake_sock.recvfrom.side_effect = [ + (RAPID_WIND_DATAGRAM, ("127.0.0.1", 50222)), + ts.socket.timeout("stop"), + ] + ts.GLOBAL_FLAGS["json"] = True + out = io.StringIO() + args = type("A", (), {"port": 50222, "timeout": 1, "show_all": False})() + with contextlib.redirect_stdout(out): + with patch.object(ts.socket, "socket", return_value=fake_sock): + ts.udp_listen(args) + doc = json.loads(out.getvalue().strip().splitlines()[-1]) + self.assertEqual(doc["type"], "rapid_wind") + self.assertEqual(doc["wind_speed_mps"], 2.3) + fake_sock.close.assert_called_once() + + +# --------------------------------------------------------------------------- +# Documented pipeline wiring: each stage's output feeds the next +# --------------------------------------------------------------------------- + +class PipelineTests(CliTestCase): + def test_station_ids_from_stations_feed_current(self): + fake = FakeTransport(STATIONS_ONLY) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + _, stations_out = run_main(["stations", "--json"]) + sid = json.loads(stations_out)["stations"][0]["station_id"] + did = next(d["device_id"] for d in + json.loads(stations_out)["stations"][0]["devices"] + if d["device_type"] == "ST") + self.assertIsInstance(sid, int) + self.assertIsInstance(did, int) + _, current_out = run_main(["current", "--station-id", str(sid), + "--device-id", str(did), "--json"]) + doc = json.loads(current_out) + self.assertEqual(doc["device_id"], did) + # observation dict carries metric-native numeric types for jq math + self.assertIsInstance(doc["observation"]["air_temperature"], float) + self.assertIsInstance(doc["observation"]["rain_accumulation"], (int, float)) + + def test_rain_watch_pipeline_obs_day_total_then_evt_precip_stream(self): + fake = FakeTransport(STATIONS_ONLY) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + _, obs_out = run_main(["obs", "--device-id", "60526", "--days", "1", "--json"]) + doc = json.loads(obs_out) + self.assertEqual(doc["type"], "obs_st") + total = doc["observations"][-1]["local_day_rain_accumulation"] + self.assertEqual(total, 5.2) + # live half: evt_precip datagram decodes with a timestamp for the stream + _, payload = ts.handle_datagram(EVT_PRECIP_DATAGRAM)[0] + self.assertEqual(payload["type"], "evt_precip") + self.assertIsNotNone(payload["timestamp"]) + + def test_forecast_json_fields_are_jq_addressable(self): + fake = FakeTransport(STATIONS_ONLY) + with patch_token(), patch.object(ts.TempestClient, "_get", fake): + _, out = run_main(["forecast", "--json"]) + doc = json.loads(out) + # documented nesting: .forecast.forecast.daily / .forecast.units.units_temp + self.assertEqual(doc["forecast"]["units"]["units_temp"], "c") + self.assertEqual(doc["forecast"]["forecast"]["hourly"][0]["local_hour"], 10) + self.assertIsInstance(doc["forecast"]["forecast"]["daily"][0]["air_temp_high"], (int, float)) + + +if __name__ == "__main__": + unittest.main() From 2a9a81e29b50504e40666ea8dbbd174765e041d8 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sun, 30 Aug 2026 00:45:16 -0400 Subject: [PATCH 38/40] fix(jellyfin): harden dispatch against subcommand-named flag values main() sliced argv at the first occurrence of a known subcommand name, so a value-flag pair whose value names a subcommand (e.g. login's --server given `search`) made argparse dispatch the wrong subparser: `jellyfin --server search browse ...` errored inside the `search` sub-parser instead of running browse. Dispatch now splits such misplaced pairs out of the top-level argv (find_subcommand_token + split_misplaced_value_pairs) and re-attaches them to the command tail, where each handler's parse_known_args already tolerates unknown flags. A properly placed occurrence of the same flag later in the tail still wins. Any other pre-command token (unknown flags, stray positionals, `--`, dangling value flags) is untouched, so argparse errors stay byte-identical to the pre-hardening CLI. Also dedupe the twice-repeated "Bundled CLI --dry-run and exit-code contract" section in references/worked-recipes.md (user-testing round 1 finding); content merged into one section. Adds DispatchHardeningTests: the mis-slice scenario for every subcommand shape, clean-argv dispatch for all 11 subcommands, flag priority, argparse-owned error paths, and unit pinning of both helper return-value tables. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- jellyfin/references/worked-recipes.md | 22 ---- jellyfin/scripts/jellyfin | 81 ++++++++++++-- jellyfin/scripts/test_jellyfin_cli.py | 151 ++++++++++++++++++++++++++ 3 files changed, 224 insertions(+), 30 deletions(-) diff --git a/jellyfin/references/worked-recipes.md b/jellyfin/references/worked-recipes.md index 90bc52d..3629745 100644 --- a/jellyfin/references/worked-recipes.md +++ b/jellyfin/references/worked-recipes.md @@ -153,28 +153,6 @@ Exit codes: `0` on success (including dry-run previews), `1` on CLI errors (miss credentials, unreachable server, API 4xx/5xx, missing required `--user-id`), `2` on argparse misuse such as `--movies --episodes` together or a missing required flag. -## Bundled CLI `--dry-run` and exit-code contract - -The CLI's dry-run plans are pinned by its offline test suite (`scripts/test_jellyfin_cli.py`), -so jq keys match tested reality exactly. Every plan carries: - -```json -{ "dry_run": true, "path": "/Items/Latest", "params": { "userId": null, "limit": 10 } } -``` - -- `dry_run` (bool, always true), `path` (string) and `params` (object) appear on every - command plan; `login` instead emits `path: "/Users/AuthenticateByName"`, - `server`, `username`, `authorization_header`, and `pre_token_header: true` (its - `authorization_header` is the complete pre-token MediaBrowser header, no `Token=` - segment); `info` composes a `requests` array of `{path, params}` steps instead of a - single `path`/`params` pair. -- `params` mirrors the exact query the live call would send (`userId` is JSON `null` - when not supplied). - -Exit codes: `0` on success (including dry-run previews), `1` on CLI errors (missing -credentials, unreachable server, API 4xx/5xx, missing required `--user-id`), `2` on -argparse misuse such as `--movies --episodes` together or a missing required flag. - ## Cross-version-safe baseline (derive your own recipes from these rules) 1. Speak modern auth (`Authorization: MediaBrowser ...`) and put the token in exactly ONE diff --git a/jellyfin/scripts/jellyfin b/jellyfin/scripts/jellyfin index 909e611..d56d329 100755 --- a/jellyfin/scripts/jellyfin +++ b/jellyfin/scripts/jellyfin @@ -34,6 +34,16 @@ ENV_PASSWORD = os.getenv("JELLYFIN_PASSWORD", "") CLIENT_NAME = "jellyfin-cli" CLIENT_VERSION = "1.0.0" +# Option strings of the subcommand parsers that consume a value (store_true +# flags excluded). A flag whose VALUE names a subcommand (e.g. login's --server +# given `search`) must not be mistaken for the command itself; keep this in +# sync when adding subcommand flags. Flags listed here behave exactly as +# before when they appear after the subcommand. +VALUE_FLAGS = ( + "--server --username -u --password --query -q --type --limit --user-id " + "--series-id --season-id --id --library-id --device-id --start-index" +).split() + GLOBAL_FLAGS: Dict[str, Any] = {"json": False, "dry_run": False} @@ -94,6 +104,55 @@ def _preparse_global_flags(argv): return flags, filtered +def find_subcommand_token(argv, subcommands, value_flags): + """Locate the first subcommand token in argv that no flag consumes as a value. + + argv[0] is the program name. Returns (command_index, pair_start): the index + of the first subcommand token that is not a flag value, and the index where + the consumed flag/value pairs begin (None when no value names a subcommand). + A value-flag paired with a value that NAMES a subcommand (e.g. `--server + search`) must not hijack dispatch, so the pair is consumed and scanning + continues for the first unconsumed subcommand token. Any other token the + top-level parser would reject (unknown flags, stray positionals, `--`, a + dangling value-flag, or a value that is not a subcommand name) stops the + scan with command_index=None so argparse produces its usual error. + """ + i = 1 + pair_start = None + while i < len(argv): + token = argv[i] + if token in value_flags: + if i + 1 >= len(argv): + break # flag with no value: argparse reports the misuse + if pair_start is None and argv[i + 1] in subcommands: + pair_start = i # a value naming a subcommand must not dispatch + i += 2 # consume the flag and its value as a pair + continue + if token in subcommands: + return i, pair_start + break # unknown flag, stray positional, or `--`: argparse owns this error + return None, pair_start + + +def split_misplaced_value_pairs(argv, subcommands, value_flags): + """Split argv into (parse_argv, misplaced_pairs) around a hijacking flag value. + + A value-flag pair whose value NAMES a subcommand while sitting before the + real command (e.g. `--server search browse`) would make argparse dispatch + the wrong subparser. Such pairs are lifted out of parse_argv and returned + as misplaced_pairs so main() can re-attach them to the command tail, where + parse_known_args handles them as it handles any unknown flag today. Any + other pre-command token (unknown flags, stray positionals, `--`, a + dangling value-flag) is left in place so argparse keeps reporting it + exactly as before, and argv without the mis-slice shape comes back + unchanged with an empty misplaced_pairs list. + """ + command_index, pair_start = find_subcommand_token(argv, subcommands, value_flags) + if command_index is None or pair_start is None: + return argv, [] + return argv[:pair_start] + argv[command_index:], argv[pair_start:command_index] + + class JellyfinClient: """Jellyfin API client (10.8+ compatible, modern Authorization header).""" @@ -663,25 +722,31 @@ def main(): sub.add_parser("libraries", help="List libraries", description="List configured media libraries.", epilog="Example: jellyfin libraries") sub.add_parser("stats", help="Library statistics", description="Show media library item counts.", epilog="Example: jellyfin stats") - args = parser.parse_args(filtered_argv[1:]) - if not args.command: - parser.print_help() - sys.exit(1) - - client = JellyfinClient(dry_run=GLOBAL_FLAGS.get("dry_run", False)) - cmd_map = { "login": cmd_login, "info": cmd_info, "recent": cmd_recent, "search": cmd_search, "next-up": cmd_next_up, "item": cmd_item, "seasons": cmd_seasons, "episodes": cmd_episodes, "browse": cmd_browse, "libraries": cmd_libraries, "stats": cmd_stats, } + + parse_argv, misplaced = split_misplaced_value_pairs(filtered_argv, cmd_map, VALUE_FLAGS) + args = parser.parse_args(parse_argv[1:]) + if not args.command: + parser.print_help() + sys.exit(1) + + client = JellyfinClient(dry_run=GLOBAL_FLAGS.get("dry_run", False)) + handler = cmd_map.get(args.command) if not handler: parser.print_help() sys.exit(1) - remaining = filtered_argv[filtered_argv.index(args.command) + 1:] + # Misplaced flag/value pairs are re-attached ahead of the command tail so + # parse_known_args sees them where it tolerates unknown flags today; a + # properly placed occurrence of the same flag later in the tail still wins. + tail_start = parse_argv.index(args.command) + 1 + remaining = misplaced + parse_argv[tail_start:] handler(client, remaining) diff --git a/jellyfin/scripts/test_jellyfin_cli.py b/jellyfin/scripts/test_jellyfin_cli.py index c14ae18..32a2aac 100644 --- a/jellyfin/scripts/test_jellyfin_cli.py +++ b/jellyfin/scripts/test_jellyfin_cli.py @@ -612,5 +612,156 @@ class PipelineChainTests(unittest.TestCase): "DateCreated") +class DispatchHardeningTests(unittest.TestCase): + """main() must dispatch on the first UNCONSUMED subcommand token. + + A value-flag pair whose value NAMES a subcommand while sitting before the + real command (e.g. `--server search browse ...`) used to make argparse + dispatch the wrong subparser (the token was swallowed as the flag's value + and the real command shifted into the value slot). The hardened dispatch + lifts such pairs out of the top-level argv and re-attaches them to the + command tail, where parse_known_args already tolerates unknown flags. + Every other pre-command token still reaches argparse so its errors are + byte-identical to the pre-hardening CLI. + """ + + def run_cli(self, *args): + return subprocess.run([str(SCRIPT), *args], text=True, capture_output=True, + env=clean_env(), cwd=tempfile.gettempdir()) + + def test_flag_value_equal_to_subcommand_dispatches_browse_not_search(self): + # The exact mis-slice scenario: `--server search` must not dispatch + # the `search` sub-parser; the real command is `browse`. + result = self.run_cli("--server", "search", "browse", + "--library-id", "lib-1", "--dry-run", "--json") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(payload["path"], "/Items") + self.assertEqual(payload["params"]["parentId"], "lib-1") + + def test_value_hijack_variants_dispatch_the_real_command(self): + cases = ( + (("recent", "--user-id", "u1", "--limit", "3"), "/Items/Latest"), + (("search", "--query", "dune"), "/Search/Hints"), + (("next-up", "--user-id", "u1"), "/Shows/NextUp"), + (("item", "--id", "i1", "--user-id", "u1"), "/Items/i1"), + (("seasons", "--series-id", "s1", "--user-id", "u1"), "/Shows/s1/Seasons"), + (("episodes", "--series-id", "s1", "--user-id", "u1"), "/Shows/s1/Episodes"), + (("libraries",), "/Library/MediaFolders"), + (("stats",), "/Items/Counts"), + ) + for command, expected_path in cases: + with self.subTest(command=command): + result = self.run_cli("--server", "search", *command, + "--dry-run", "--json") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(payload["path"], expected_path) + + def test_hijack_variants_cover_flag_command_and_no_flag_value_commands(self): + # info emits a `requests` array instead of a path/params pair. + result = self.run_cli("--server", "search", "info", "--dry-run", "--json") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(payload["requests"][0]["path"], "/System/Info") + + # login keeps its plan shape; the misplaced pair rides along as the + # server value rather than being dropped. + result = self.run_cli("--server", "search", "login", "--username", "alice", + "--dry-run", "--json") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(payload["path"], "/Users/AuthenticateByName") + self.assertEqual(payload["server"], "search") + self.assertTrue(payload["pre_token_header"]) + + def test_all_subcommands_dispatch_from_clean_argv(self): + cases = ( + (("login", "--username", "alice"), "/Users/AuthenticateByName"), + (("info",), "/System/Info"), + (("recent", "--user-id", "u1"), "/Items/Latest"), + (("search", "--query", "dune"), "/Search/Hints"), + (("next-up", "--user-id", "u1"), "/Shows/NextUp"), + (("item", "--id", "i1", "--user-id", "u1"), "/Items/i1"), + (("seasons", "--series-id", "s1", "--user-id", "u1"), "/Shows/s1/Seasons"), + (("episodes", "--series-id", "s1", "--user-id", "u1"), "/Shows/s1/Episodes"), + (("browse", "--library-id", "lib-1"), "/Items"), + (("libraries",), "/Library/MediaFolders"), + (("stats",), "/Items/Counts"), + ) + for command, expected_path in cases: + with self.subTest(command=command[0]): + result = self.run_cli(*command, "--dry-run", "--json") + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + if "requests" in payload: # info composes a request array + self.assertEqual(payload["requests"][0]["path"], expected_path) + else: + self.assertEqual(payload["path"], expected_path) + + def test_properly_placed_flag_value_still_wins_over_misplaced_pair(self): + result = self.run_cli("--server", "search", "login", + "--server", "http://real:8096", + "--username", "alice", "--dry-run", "--json") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)["server"], "http://real:8096") + + def test_pre_command_tokens_argparse_owns_are_unchanged(self): + # Unknown flags, stray positionals, a non-subcommand --server value, + # a dangling value flag, and `--` all keep their pre-hardening + # argparse errors (exit 2, no traceback, no tolerant dispatch). + cases = ( + ("--bogus", "info"), + ("junk", "browse", "--library-id", "lib-1"), + ("--server", "http://x:8096", "info"), + ("--server",), + ("--", "search", "--query", "dune"), + ) + for argv in cases: + with self.subTest(argv=argv): + result = self.run_cli(*argv, "--dry-run", "--json") + self.assertEqual(result.returncode, 2) + self.assertIn("error:", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + def test_find_subcommand_token_returns_command_and_pair_indices(self): + cli = jellyfin_cli + subs = {"login", "info", "recent", "search", "next-up", "item", "seasons", + "episodes", "browse", "libraries", "stats"} + cases = ( + (["jf", "--server", "search", "browse", "--library-id", "L"], (3, 1)), + (["jf", "browse", "--library-id", "L"], (1, None)), + (["jf", "--server", "http://x", "info"], (3, None)), + (["jf", "login", "--server", "search"], (1, None)), + (["jf", "--bogus", "info"], (None, None)), + (["jf", "--server"], (None, None)), + (["jf", "--"], (None, None)), + ) + for argv, expected in cases: + with self.subTest(argv=argv): + self.assertEqual( + cli.find_subcommand_token(argv, subs, cli.VALUE_FLAGS), expected) + + def test_split_misplaced_value_pairs_lifts_only_hijacking_pair(self): + cli = jellyfin_cli + subs = {"login", "info", "recent", "search", "next-up", "item", "seasons", + "episodes", "browse", "libraries", "stats"} + parse_argv, misplaced = cli.split_misplaced_value_pairs( + ["jf", "--server", "search", "browse", "--library-id", "L"], + subs, cli.VALUE_FLAGS) + self.assertEqual(parse_argv, ["jf", "browse", "--library-id", "L"]) + self.assertEqual(misplaced, ["--server", "search"]) + + # A value that is not a subcommand name never lifts anything, and + # clean argv passes through untouched. + parse_argv, misplaced = cli.split_misplaced_value_pairs( + ["jf", "--server", "http://x", "info"], subs, cli.VALUE_FLAGS) + self.assertEqual((parse_argv, misplaced), (["jf", "--server", "http://x", "info"], [])) + parse_argv, misplaced = cli.split_misplaced_value_pairs( + ["jf", "login", "--server", "search", "--username", "a"], + subs, cli.VALUE_FLAGS) + self.assertEqual(misplaced, []) + + if __name__ == "__main__": unittest.main() From ce5f34a33d3fc1300e5b52f15ead5e3e2edf5b07 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sun, 30 Aug 2026 01:52:10 -0400 Subject: [PATCH 39/40] fix(tempest): honor --dry-run on udp listen without binding a socket udp listen ignored the universal --dry-run flag and bound UDP 50222, hanging when no hub is on the LAN. Add a dry-run plan branch that describes the listen parameters (bind address, port, timeout, show-all) and exits 0 without creating any socket, so doc claims of universal --dry-run support stay universal and true. Four regression tests pin the plan shape, the defaults/--show-all propagation, and prove no socket is constructed (and no token demanded). Recipe 5 documents the udp listen plan object alongside the other plans. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- tempest/references/cli-worked-recipes.md | 14 +++++-- tempest/scripts/tempest | 11 ++++++ tempest/scripts/test_tempest.py | 50 ++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/tempest/references/cli-worked-recipes.md b/tempest/references/cli-worked-recipes.md index 347f2b0..77908d2 100644 --- a/tempest/references/cli-worked-recipes.md +++ b/tempest/references/cli-worked-recipes.md @@ -121,16 +121,24 @@ on `type` before indexing. tempest forecast --station-id 12799 --days 3 --dry-run --json # -> {"dry_run": true, "command": "forecast", "station_id": 12799, "days": 3} -# Every documented command has a dry-run plan: current, obs, forecast, stations +# Every documented command has a dry-run plan — current, obs, forecast, +# stations, and udp listen (plans the bind, creates no socket, safe off-LAN) tempest obs --device-id 60526 --days 2 --dry-run --json +tempest udp listen --port 50222 --timeout 30 --dry-run --json +# -> {"dry_run": true, "command": "udp", "subcommand": "listen", +# "bind_address": "0.0.0.0", "port": 50222, "timeout_seconds": 30, +# "show_all": false} # Quiet/verbose piping: logs on stderr, data on stdout tempest current --json --quiet | jq .observation.air_temperature ``` Behavior contract: `--dry-run` works without `TEMPEST_TOKEN` set (no credential -needed to see a plan); `--help` and `--dry-run` are always offline. Without -`--dry-run`, a missing token exits 1 with +needed to see a plan); `--help` and `--dry-run` are always offline. For +`udp listen`, dry-run describes the listen parameters (bind address, port, +timeout, show-all) and exits 0 without creating or binding any socket — the +real listener waits for hub traffic on UDP 50222 and needs the hub's LAN. +Without `--dry-run`, a missing token exits 1 with `Error: TEMPEST_TOKEN not set...` before any request is attempted. ## Recipe 6: JSON error paths you'll actually see diff --git a/tempest/scripts/tempest b/tempest/scripts/tempest index 0592f09..6061107 100755 --- a/tempest/scripts/tempest +++ b/tempest/scripts/tempest @@ -732,6 +732,17 @@ def udp_listen(args: argparse.Namespace) -> None: timeout = args.timeout show_all = args.show_all + if GLOBAL_FLAGS.get("dry_run", False): + # Dry-run plans the listen instead of opening it: exits 0 without + # creating or binding any socket, so it is safe anywhere (no hub + # required, no LAN needed, no hanging on a broadcast port). + emit("[dry-run] Would listen for Tempest UDP broadcasts on " + f"{UDP_BROADCAST_ADDR}:{port} — binds no socket now.", + {"dry_run": True, "command": "udp", "subcommand": "listen", + "bind_address": UDP_BROADCAST_ADDR, "port": port, + "timeout_seconds": timeout, "show_all": show_all}) + return + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((UDP_BROADCAST_ADDR, port)) diff --git a/tempest/scripts/test_tempest.py b/tempest/scripts/test_tempest.py index edfe085..ab778ac 100644 --- a/tempest/scripts/test_tempest.py +++ b/tempest/scripts/test_tempest.py @@ -316,6 +316,56 @@ class DryRunTests(CliTestCase): self.assertEqual(code, 0) self.assertEqual(json.loads(out)["command"], "stations") + def test_udp_listen_dry_run_plan_shape(self): + # VAL-TEMP-011: udp listen honors --dry-run — a plan JSON, exit 0, + # and (pinned by the socket patch below) NO socket is ever created + # or bound, so the dry run cannot hang waiting for hub traffic. + code, out = run_main(["udp", "listen", "--port", "50222", + "--timeout", "30", "--dry-run", "--json"]) + self.assertEqual(code, 0) + plan = json.loads(out) + self.assertEqual(plan["dry_run"], True) + self.assertEqual(plan["command"], "udp") + self.assertEqual(plan["subcommand"], "listen") + self.assertEqual(plan["bind_address"], ts.UDP_BROADCAST_ADDR) + self.assertEqual(plan["port"], 50222) + self.assertEqual(plan["timeout_seconds"], 30) + self.assertEqual(plan["show_all"], False) + + def test_udp_listen_dry_run_defaults_and_show_all(self): + # Defaults land in the plan; --show-all propagates. + code, out = run_main(["udp", "listen", "--show-all", "--dry-run", "--json"]) + self.assertEqual(code, 0) + plan = json.loads(out) + self.assertEqual(plan["port"], ts.DEFAULT_UDP_PORT) + self.assertEqual(plan["timeout_seconds"], 0) + self.assertEqual(plan["show_all"], True) + + def test_udp_listen_dry_run_creates_no_socket(self): + # Prove the "binds no socket" half of the contract: if udp_listen + # reached its listen path, socket.socket() would be constructed and + # this fake's bind() would blow up the test. + bound = [] + + class NoBindSock: + def bind(self, *a, **k): + bound.append(a) + raise AssertionError("dry-run udp listen must not bind a socket") + + with patch.object(ts.socket, "socket", side_effect=AssertionError( + "dry-run udp listen must not create a socket")): + code, out = run_main(["udp", "listen", "--dry-run", "--json"]) + self.assertEqual(code, 0) + self.assertEqual(bound, []) + self.assertEqual(json.loads(out)["dry_run"], True) + + def test_udp_listen_dry_run_without_token_is_fine(self): + # UDP needs no token, and the dry run must not demand one either. + with patch_token(""): + code, out = run_main(["udp", "listen", "--dry-run", "--json"]) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out)["command"], "udp") + # --------------------------------------------------------------------------- # Class 4: mocked REST client logic (no real network anywhere) From d1e282a95dd943e094e2fe6b73f77ed6c4d02962 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sun, 30 Aug 2026 01:52:43 -0400 Subject: [PATCH 40/40] docs(transistor): scope publish audio-guard and add missing trigger rows Document that the publish audio-guard fires only for --status published (scheduling intentionally precedes audio attach) and that its pre-publish episode GET consumes one rate-limit slot, relevant to bulk re-publish loops. Add references/skill-triggers.md rows for transistor and trakt, the two thickened skills missing from the trigger index; phrasing follows each skill frontmatter description. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- references/skill-triggers.md | 2 ++ transistor/references/episode-publish-lifecycle.md | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/references/skill-triggers.md b/references/skill-triggers.md index 0d9426c..606d5d8 100644 --- a/references/skill-triggers.md +++ b/references/skill-triggers.md @@ -24,6 +24,8 @@ Each skill's `description` field is the canonical routing contract. This conveni | "Open Library", "openlibrary", "book search", "ISBN lookup", "author records", "work details", "book editions", "book ratings", "cover image" | [openlibrary](../openlibrary/SKILL.md) | | "PeerTube", "peertube", "federated video", "SepiaSearch", "decentralized video platform", "PEERTUBE_SERVER", "my PeerTube instance" | [peertube](../peertube/SKILL.md) | | "weather", "forecast", "temperature", "is it raining", "Tempest", "WeatherFlow", "my station" | [tempest](../tempest/SKILL.md) | +| "Transistor", "Transistor.fm", "podcast hosting", "podcast episodes", "episode publishing", "private podcast subscribers", "podcast download analytics" | [transistor](../transistor/SKILL.md) | +| "Trakt", "Trakt.tv", "trending movies", "trending shows", "popular movies", "anticipated movies", "what's trending", "TRAKT_CLIENT_ID" | [trakt](../trakt/SKILL.md) | | "TMDb", "The Movie Database", "movie search", "trending movies", "upcoming TV releases", "TMDB_ACCESS_TOKEN" | [tmdb](../tmdb/SKILL.md) | | "traefik", "reverse proxy", "load balancer", "API gateway", "Let's Encrypt", "ACME", "Docker routing", "traefik.yml", "entry point", "middleware", "TLS termination", "forward auth", "rate limit" | [traefik](../traefik/SKILL.md) | | "reverse-engineer", "understand this codebase", "PRD from code", "architecture document", "architecture health", "coupling analysis", "modularity", "decomposition readiness", "data ownership map", "distributed workflow analysis", "reconciliation path" | [software-architecture-analysis](../software-architecture-analysis/SKILL.md) | diff --git a/transistor/references/episode-publish-lifecycle.md b/transistor/references/episode-publish-lifecycle.md index 79d10f9..cc9536b 100644 --- a/transistor/references/episode-publish-lifecycle.md +++ b/transistor/references/episode-publish-lifecycle.md @@ -105,7 +105,12 @@ transistor episode-publish --id The bundled CLI guards step 3: if `attributes.media_url` is still empty it refuses to publish (an audio-less item would go out to every feed reader) -and prints the attach-audio recipe; `--force` overrides. +and prints the attach-audio recipe; `--force` overrides. The guard is scoped +to `--status published` (the default) — scheduling intentionally precedes +audio attach, so a `scheduled` publish needs no audio yet — and it performs +one pre-publish `GET /episodes/:id`, which consumes one of the +10-requests-per-10-seconds rate-limit slots: worth counting in bulk +re-publish loops. ## Worked recipe: raw curl