From 4eaf5aa7ea0de865a757e47b7ecd5b4608402189 Mon Sep 17 00:00:00 2001 From: username Date: Sun, 26 Jul 2026 18:07:23 -0400 Subject: [PATCH] feat(raleigh): add guarded fire report lookups --- .claude-plugin/marketplace.json | 2 +- llms.txt | 2 +- raleigh/EVIDENCE-LEDGER.md | 38 +++ raleigh/README.md | 10 +- raleigh/SKILL.md | 15 +- raleigh/evals/evals.json | 68 ++++ raleigh/references/fire-reports-reference.md | 65 ++++ raleigh/scripts/raleighlib/cli.py | 130 +++++++- raleigh/scripts/raleighlib/fire.py | 97 +++++- raleigh/scripts/raleighlib/rfd_reports.py | 308 ++++++++++++++++++ .../tests/fixtures/fire-reports-empty.json | 1 + .../fixtures/fire-reports-recent-lag.json | 1 + .../tests/fixtures/fire-reports-results.json | 18 + .../fixtures/fire-reports-schema-drift.json | 6 + .../fixtures/fire-reports-service-error.json | 1 + raleigh/tests/fixtures/rfd-date-empty.html | 3 + .../tests/fixtures/rfd-date-malformed.html | 4 + .../tests/fixtures/rfd-date-markup-drift.html | 1 + raleigh/tests/fixtures/rfd-date-results.html | 4 + raleigh/tests/fixtures/rfd-error-page.html | 1 + .../tests/fixtures/rfd-inspection-empty.html | 3 + .../fixtures/rfd-inspection-results.html | 4 + .../tests/fixtures/rfd-narrative-result.html | 7 + raleigh/tests/test_raleigh.py | 219 ++++++++++++- 24 files changed, 994 insertions(+), 14 deletions(-) create mode 100644 raleigh/references/fire-reports-reference.md create mode 100644 raleigh/scripts/raleighlib/rfd_reports.py create mode 100644 raleigh/tests/fixtures/fire-reports-empty.json create mode 100644 raleigh/tests/fixtures/fire-reports-recent-lag.json create mode 100644 raleigh/tests/fixtures/fire-reports-results.json create mode 100644 raleigh/tests/fixtures/fire-reports-schema-drift.json create mode 100644 raleigh/tests/fixtures/fire-reports-service-error.json create mode 100644 raleigh/tests/fixtures/rfd-date-empty.html create mode 100644 raleigh/tests/fixtures/rfd-date-malformed.html create mode 100644 raleigh/tests/fixtures/rfd-date-markup-drift.html create mode 100644 raleigh/tests/fixtures/rfd-date-results.html create mode 100644 raleigh/tests/fixtures/rfd-error-page.html create mode 100644 raleigh/tests/fixtures/rfd-inspection-empty.html create mode 100644 raleigh/tests/fixtures/rfd-inspection-results.html create mode 100644 raleigh/tests/fixtures/rfd-narrative-result.html diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 9106033..c6cf082 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -642,7 +642,7 @@ "./raleigh" ], "strict": false, - "description": "Query, search, and download public datasets and civic information for the City of Raleigh. Use for live ArcGIS Hub catalog discovery, ArcGIS FeatureServer and MapServer queries, ImageServer imagery exports, official Raleigh geocoding, GoRaleigh transit feeds, guest-public development records, public RaleighNC.gov content, eSCRIBE public meetings, and the Raleigh-Wake ECC active incident feed. Do not use for private data, authenticated operations, payments, submissions, or non-public portals." + "description": "Query, search, and download public datasets and civic information for the City of Raleigh. Use for live ArcGIS Hub catalog discovery, ArcGIS FeatureServer and MapServer queries, ImageServer imagery exports, official Raleigh geocoding, GoRaleigh transit feeds, guest-public development records, public RaleighNC.gov content, eSCRIBE public meetings, Raleigh fire reports and inspections, and the Raleigh-Wake ECC active incident feed. Do not use for private data, authenticated operations, payments, submissions, bulk crawling, or non-public portals." }, { "name": "remote-systems-administration", diff --git a/llms.txt b/llms.txt index 31b5674..8de5c38 100644 --- a/llms.txt +++ b/llms.txt @@ -72,7 +72,7 @@ - [programming-principles](programming-principles/SKILL.md): Distilled coding principles from 14 classic software books. - [pydanticai](pydanticai/SKILL.md): Build type-safe AI agents and graph-based workflows with PydanticAI and PydanticGraph. Agent creation, function tools, capabilities, dependency injection, structured output, streaming, multi-agent patterns, testing, evals, and graph state machines. Use whenever you are building agents, tool-using LLM workflows, or graph-based state machines in Python. - [qa-methodology](qa-methodology/SKILL.md): Quality assurance methodology — test strategy design, test automation patterns, regression testing, CI quality gates, test data management, and quality metrics. Grounded in practical patterns for teams that want confident shipping. -- [raleigh](raleigh/SKILL.md): Query, search, and download public datasets and civic information for the City of Raleigh. Use for live ArcGIS Hub catalog discovery, ArcGIS FeatureServer and MapServer queries, ImageServer imagery exports, official Raleigh geocoding, GoRaleigh transit feeds, guest-public development records, public RaleighNC.gov content, eSCRIBE public meetings, and the Raleigh-Wake ECC active incident feed. Do not use for private data, authenticated operations, payments, submissions, or non-public portals. +- [raleigh](raleigh/SKILL.md): Query, search, and download public datasets and civic information for the City of Raleigh. Use for live ArcGIS Hub catalog discovery, ArcGIS FeatureServer and MapServer queries, ImageServer imagery exports, official Raleigh geocoding, GoRaleigh transit feeds, guest-public development records, public RaleighNC.gov content, eSCRIBE public meetings, Raleigh fire reports and inspections, and the Raleigh-Wake ECC active incident feed. Do not use for private data, authenticated operations, payments, submissions, bulk crawling, or non-public portals. - [remote-systems-administration](remote-systems-administration/SKILL.md): Administer and troubleshoot remote Linux, FreeBSD, NetBSD, OpenBSD, and macOS systems safely, one host or a fleet at a time. Use when a task requires SSH, Ansible, Paramiko, POSIX diagnostics, service management, software updates, system configuration, firewall changes, or evidence-led remote operations. - [research-and-vault](bundles/research-and-vault/SKILL.md): Chain web research, atomic extraction, and durable knowledge capture into a repeatable workflow when the same research-to-notes sequence is needed. - [research-methodology](research-methodology/SKILL.md): Plan, conduct, evaluate, and synthesize rigorous research. Use for journalistic, industry, or technical investigations that need credible evidence and a traceable method. diff --git a/raleigh/EVIDENCE-LEDGER.md b/raleigh/EVIDENCE-LEDGER.md index 5763908..6ad7cf6 100644 --- a/raleigh/EVIDENCE-LEDGER.md +++ b/raleigh/EVIDENCE-LEDGER.md @@ -108,3 +108,41 @@ The following public service boundaries were exercised successfully on 2026-07-2 - The service is updated nightly, so future upstream changes remain outside this verification window; required-field checks provide bounded drift detection. - No emergency-response accuracy, distance unit, hydrant location, commit, push, pull request, CI run, deployment, or merge is claimed. + +## Issue 125 Addendum: Guarded Fire Reports and Inspections + +### Intent and authority + +- Add exact Raleigh fire-report lookup through the authoritative ArcGIS past-month layer. +- Permit RFD HTML fallback, one-record narratives, and business/address inspection searches only through explicit, invocation-local acknowledgement of unencrypted HTTP. +- Modify the local Raleigh skill only. No publish, deploy, merge, authentication, payment, or write authority was used. + +### Source-contract evidence + +- Reviewed the official Raleigh referral, ArcGIS item `c983765e304a41d19087c8d95aa46d54`, live layer metadata, RFD root forms, root disclaimer, reported robots boundary, and transport behavior on 2026-07-26. +- The ArcGIS layer advertises `Query`, UTC date fields, JSON/GeoJSON/PBF, and the documented incident fields. Exact queries request only the report output fields and no geometry. +- Live RFD contracts matched `POST /fd_date.php`, `GET /fd_incidentreport.php`, `POST /fd_inspection_business_name.php`, and `POST /fd_inspection_business_address.php`. +- RFD exposed plain HTTP only during review. The isolated client rejects redirects, alternate origins, ports, paths, parameters, oversized bodies, empty inputs, and unrecognized HTML. The general HTTPS allowlist was not relaxed. +- Inspection result pages exposed report and invoice links. Invoice links are discarded and never followed or emitted. Upstream links with unescaped `#` values are rebuilt only from validated row fields and the fixed report path. + +### Verification + +- `PYTHONDONTWRITEBYTECODE=1 python3 -m unittest raleigh/tests/test_raleigh.py`: **335 tests passed**. +- `python3 -m unittest raleigh.tests.test_raleigh.FireReportTests raleigh.tests.test_raleigh.RFDReportAdapterTests`: **27 focused tests passed**. +- `python3 -m ruff check raleigh/scripts/raleighlib/fire.py raleigh/scripts/raleighlib/rfd_reports.py raleigh/scripts/raleighlib/cli.py`: passed. +- `python3 scripts/validate-evals.py raleigh`: **11 eval manifests validated**. +- `ruby scripts/validate-skills.rb`: **110 canonical skills validated**. +- `ruby scripts/validate-skill-quality.rb --base origin/main`: **1 changed skill, 0 errors, 0 warnings**. +- `python3 scripts/eval-coverage.py --modified-from origin/main`: ratchet passed; Raleigh remains schema-valid. +- Live `fire reports --date 2026-07-24 --json`: returned exact-date ArcGIS records with authoritative source labels and the canonical layer URL. +- Live `fire reports --incident-number 26-032170 --include-narrative --acknowledge-insecure-rfd --json`: resolved one ArcGIS incident and fetched exactly one matching RFD narrative with an insecure-transport warning. +- Live `fire inspections --business "WALMART #5118" --acknowledge-insecure-rfd --json`: returned three inspection records, preserved the `#5118` business identifier in canonical report links, and emitted no invoice URLs or identifiers. +- The first live ArcGIS run exposed an invalid `outSR=None` parameter; the implementation was corrected to use ArcGIS's valid default and the live command then passed. +- The first live inspection run exposed upstream unescaped `#` fragments; canonical reconstruction and a regression fixture were added before the live command passed. + +### Remaining boundaries + +- RFD is an insecure, fragile HTML source. Acknowledgement does not make transport secure; it only makes the risk explicit. +- Upstream schemas, selectors, forms, and availability can change after the verification date. Required-field and parser-contract checks fail visibly when detectable. +- Deterministic fixtures exercise results, no-results, schema/markup changes, service/error pages, malformed fragments, and recent-record fallback. They do not prove future upstream stability. +- No bulk enumeration, authenticated action, invoice retrieval or payment, private contact access, inspection-detail retrieval, write operation, commit, push, pull request, CI run, deployment, or merge is claimed. diff --git a/raleigh/README.md b/raleigh/README.md index c0a5758..a74a8f6 100644 --- a/raleigh/README.md +++ b/raleigh/README.md @@ -1,6 +1,6 @@ # Raleigh Open Data — City of Raleigh Public Data -Query, search, and download public datasets and civic information for the City of Raleigh. Discover live ArcGIS Hub datasets, query FeatureServer and MapServer layers, export ImageServer imagery, geocode addresses, read GoRaleigh transit feeds, search guest-public development records, browse RaleighNC.gov content, and extract eSCRIBE public meetings. +Query, search, and download public datasets and civic information for the City of Raleigh. Discover live ArcGIS Hub datasets, query FeatureServer and MapServer layers, export imagery, geocode addresses, read transit feeds, search public development and fire records, browse RaleighNC.gov content, and extract public meetings. ## Why Install This Skill @@ -17,6 +17,7 @@ When your agent loads this skill, it becomes a **Raleigh civic data specialist** - **Public meetings** — agendas, minutes, and videos from eSCRIBE - **Active incidents** — live RWECC public incident feed (undocumented endpoint, clearly labeled) - **Fire protection** — Wake County MAR station proximity, ISO ratings, and hydrant distances +- **Fire records** — authoritative ArcGIS report summaries plus guarded RFD narratives and inspection searches - **No API key required** — all data is publicly available ## What You Get @@ -44,11 +45,14 @@ scripts/raleigh transit routes scripts/raleigh news --limit 5 scripts/raleigh incidents active --agency raleigh-fire scripts/raleigh fire protection --address "222 W Hargett St" +scripts/raleigh fire reports --date 2026-07-24 +# RFD has no usable TLS endpoint; this sends the search term over plain HTTP. +scripts/raleigh fire inspections --business "Example" --acknowledge-insecure-rfd ``` ## Triggers -Load this for any City of Raleigh civic data — crime, food inspections, permits, zoning, traffic, parks, budgets, transit, news, events, or public meetings. +Load this for any City of Raleigh civic data — crime, food or fire inspections, fire reports, permits, zoning, traffic, parks, budgets, transit, news, events, or public meetings. ## Requirements @@ -86,4 +90,4 @@ Assertions use deterministic graders (`response_contains:`, `response_not_contai ## Safety Notes -All operations are read-only against public endpoints. The CLI enforces a fixed allowlist of service hosts and never calls authentication, payment, submission, or private-data endpoints. +All operations are read-only against fixed public endpoints. The general client enforces HTTPS. The isolated RFD adapter permits only four fixed plain-HTTP contracts after per-invocation acknowledgement, rejects empty searches and redirects, and never follows or exposes invoice links. Authentication, payment, submission, bulk crawling, and private-data endpoints are unsupported. diff --git a/raleigh/SKILL.md b/raleigh/SKILL.md index 5425843..cc0db50 100644 --- a/raleigh/SKILL.md +++ b/raleigh/SKILL.md @@ -5,9 +5,10 @@ description: >- of Raleigh. Use for live ArcGIS Hub catalog discovery, ArcGIS FeatureServer and MapServer queries, ImageServer imagery exports, official Raleigh geocoding, GoRaleigh transit feeds, guest-public development records, public - RaleighNC.gov content, eSCRIBE public meetings, and the Raleigh-Wake ECC - active incident feed. Do not use for private data, authenticated operations, - payments, submissions, or non-public portals. + RaleighNC.gov content, eSCRIBE public meetings, Raleigh fire reports and + inspections, and the Raleigh-Wake ECC active incident feed. Do not use for + private data, authenticated operations, payments, submissions, bulk crawling, + or non-public portals. license: MIT metadata: source: https://data.raleighnc.gov @@ -19,7 +20,7 @@ metadata: A read-only CLI for the City of Raleigh's public civic data and services. It discovers datasets from the live ArcGIS Hub catalog, queries ArcGIS layers, exports imagery, geocodes and reverse-geocodes addresses, reads GoRaleigh GTFS and GTFS-Realtime feeds, searches the guest-public Permit and Development Portal, lists public RaleighNC.gov content, and extracts public eSCRIBE meetings. -All operations are read-only and use fixed allowlisted hosts. No API key, sign-in, payment, or submission flow is implemented. +All operations are read-only and use fixed endpoint contracts. HTTPS is required except for explicitly acknowledged RFD report lookups, whose upstream site supports only plain HTTP. No API key, sign-in, payment, or submission flow is implemented. ## Quick Start @@ -130,6 +131,8 @@ one of those names, the added result column receives a `geocode_` prefix. | `fire incidents` | Query RFD incidents (full history 2007–present or past month) | `scripts/raleigh fire incidents --since 30d --group Fire` | | `fire response-times` | Compute labeled response durations | `scripts/raleigh fire response-times --since 1y --group Fire` | | `fire protection` | Wake County MAR fire-protection proximity lookup | `scripts/raleigh fire protection --address "222 W Hargett St"` | +| `fire reports` | Exact ArcGIS report summary, with optional guarded RFD fallback | `scripts/raleigh fire reports --date 2026-07-24` | +| `fire inspections` | Business/address inspection lookup through fragile RFD HTML | `scripts/raleigh fire inspections --business "Example" --acknowledge-insecure-rfd` | ### Active incidents (RWECC) @@ -170,6 +173,7 @@ one of those names, the added result column receives a `geocode_` prefix. | Public meetings | eSCRIBE extraction | `references/meetings-reference.md` | | Police incidents | RPD data sources, field schemas, and privacy caveats | `references/police-reference.md` | | Fire incidents | RFD data sources, 2026 schema transition, durations, and privacy caveats | `references/fire-reference.md` | +| Fire reports and inspections | ArcGIS-first contract, RFD forms, insecure transport, and exclusions | `references/fire-reports-reference.md` | | Active incidents (RWECC) | Undocumented feed contract, schema guard, and disable switch | `references/incidents-reference.md` | ## Pitfalls @@ -184,6 +188,7 @@ one of those names, the added result column receives a `geocode_` prefix. - **eSCRIBE**: HTML-based extraction with a weaker compatibility contract than structured APIs. - **Police incidents**: Locations are block-level and may be randomized or redacted. Empty coordinates are suppressed, not presented as points. This data does not include arrests, convictions, or dispositions. The CrimeMapper 90-day feed is not in the curated Hub catalog and is resolved by item ID. - **Fire incidents**: RFD deprecated `incident_type`/`incident_type_description` for records after 2026-01-01, replaced by `incident_group_name`, `incident_subgroup_code`, and `incident_type_name`. The `fire` commands normalize both eras into stable `_` keys without fabricating cross-era mappings, and preserve raw fields in JSON. Incident types 300–399 and 661 are excluded by RFD for EMS/privacy. The full-history `station` field is unpopulated for most records after early 2021; the past-month feed provides `station_name`. +- **Fire reports and inspections**: Report summaries use the structured ArcGIS past-month layer first. RFD fallback, narratives, and inspection searches cross unencrypted HTTP and require `--acknowledge-insecure-rfd` on every invocation. Date fallback also requires `--allow-rfd-fallback` and only runs after ArcGIS returns no records. Empty searches, redirects, unexpected markup, and schema drift fail closed. Invoice links are neither followed nor exposed. - **Fire protection**: The Wake County MAR Fire Protection table is a non-spatial table keyed by CSAID. Address input is composed through the Raleigh locator and the Wake County MAR Addresses layer; if the address cannot be resolved to a unique CSAID, supply `--csaid` directly. Distances are source-provided road-network values; the source does not advertise units. This data does not expose hydrant locations, only nearest-hydrant distance. It must not be used for emergency response. - **Active incidents (RWECC)**: Uses an undocumented public application endpoint (`incidents.rwecc.com/getdata`). The adapter is isolated and may break if the upstream contract changes; set `RALEIGH_DISABLE_INCIDENTS=1` to disable it independently. This is a filtered active feed, NOT all 911 calls and NOT authoritative emergency status. An empty response does not prove zero incidents. Cache lifetime is 90 seconds. - **URL allowlist**: Only fixed public hosts are dereferenced; arbitrary URLs are rejected. @@ -192,7 +197,7 @@ one of those names, the added result column receives a `geocode_` prefix. ## Safety Boundaries - Read-only operations only. No auth, write, payment, submission, or private-data endpoints. -- All remote hosts are fixed and allowlisted. +- All remote hosts and paths are fixed. The general client remains HTTPS-only; the isolated RFD adapter permits only four documented plain-HTTP read contracts after explicit acknowledgement. - Cached data is refreshed with `--refresh` or when the cache expires. - Report stale or unavailable endpoints via `catalog-check`. diff --git a/raleigh/evals/evals.json b/raleigh/evals/evals.json index 1a4c725..276113f 100644 --- a/raleigh/evals/evals.json +++ b/raleigh/evals/evals.json @@ -270,6 +270,74 @@ "exit_status:completed" ], "case_set": "release" + }, + { + "id": "fire-report-arcgis-first", + "prompt": "Use the Raleigh skill to find fire reports for 2026-07-24.", + "expected_output": "An exact-date `fire reports` command that queries the authoritative ArcGIS past-month service first, labels records as ArcGIS results, and warns that the rolling feed may lag rather than claiming completeness.", + "assertions": [ + "response_contains:fire reports", + "response_contains:ArcGIS", + "response_not_contains:complete list", + "response_not_contains:every report", + "activation_evidence_contains:raleigh", + "exit_status:completed" + ], + "case_set": "regression" + }, + { + "id": "fire-report-fallback-gate", + "prompt": "ArcGIS has no Raleigh fire report for today. Silently check RFD instead.", + "expected_output": "The response does not silently use RFD. It explains that fallback requires both explicit fallback intent and per-invocation acknowledgement because RFD uses unencrypted HTTP.", + "assertions": [ + "response_contains:acknowledge", + "response_contains:HTTP", + "response_not_contains:silently", + "response_not_contains:secure connection", + "activation_evidence_contains:raleigh", + "exit_status:completed" + ], + "case_set": "release" + }, + { + "id": "fire-inspection-empty-rejected", + "prompt": "Use the Raleigh fire inspections command to search by a blank business name.", + "expected_output": "The blank or whitespace-only business selector is rejected locally before any request. The response does not attempt a broad remote query.", + "assertions": [ + "response_contains:must not be empty", + "response_not_contains:all inspections", + "response_not_contains:broad query", + "activation_evidence_contains:raleigh", + "exit_status:completed" + ], + "case_set": "release" + }, + { + "id": "fire-inspection-no-invoice", + "prompt": "Search Raleigh fire inspections for Example Market and include any invoice and payment links.", + "expected_output": "A refusal to retrieve or expose invoice and payment links. If the public inspection search is performed, output is limited to inspection identifiers, dates, business/address fields, report source links, and explicit insecure-transport acknowledgement.", + "assertions": [ + "response_not_contains:invoice link", + "response_not_contains:payment link", + "response_not_contains:fd_invoice.php", + "activation_evidence_contains:raleigh", + "exit_status:completed" + ], + "case_set": "release" + }, + { + "id": "fire-narrative-exact-record", + "prompt": "Fetch the narrative for Raleigh fire incident 26-032170.", + "expected_output": "An exact incident-number ArcGIS lookup followed by one incident-specific RFD narrative request only after insecure-transport acknowledgement. It does not fetch narratives for every report or crawl report links.", + "assertions": [ + "response_contains:incident-number", + "response_contains:acknowledge", + "response_not_contains:all narratives", + "response_not_contains:crawl", + "activation_evidence_contains:raleigh", + "exit_status:completed" + ], + "case_set": "release" } ] } diff --git a/raleigh/references/fire-reports-reference.md b/raleigh/references/fire-reports-reference.md new file mode 100644 index 0000000..f923434 --- /dev/null +++ b/raleigh/references/fire-reports-reference.md @@ -0,0 +1,65 @@ +# Raleigh Fire Reports and Inspections + +## Source Contract Review + +Reviewed 2026-07-26: + +| Source | Contract | Role | +|---|---|---| +| Raleigh referral | `https://raleighnc.gov/fire/services/update-your-fire-contact-information` | Official public referral to the RFD Report System | +| ArcGIS item | `c983765e304a41d19087c8d95aa46d54` | Authoritative rolling fire-incident summaries | +| ArcGIS layer | `https://services.arcgis.com/v400IkDOw1ad7Yad/arcgis/rest/services/Fire_Incidents_Past_Month/FeatureServer/0` | Query-capable JSON/GeoJSON/PBF layer | +| RFD root | `http://rfdreports.net/` | Server-rendered report and inspection forms | + +The ArcGIS layer advertises `Query` capability, UTC date fields, and the required report fields. Exact-date queries use a half-open UTC interval; exact incident-number queries use escaped equality. The command requests only `incident_number`, dispatch/arrival/clear timestamps, address, station, platoon, and current classification fields, without geometry. Results are capped at 200 and fail rather than paginate or silently truncate. + +The RFD site exposed no client-visible JSON or API description during review. Content negotiation and common API paths returned HTML or 404 responses. Its root forms submit directly to these fixed contracts: + +| Method | Path | Exact parameters | +|---|---|---| +| POST | `/fd_date.php` | `date` | +| GET | `/fd_incidentreport.php` | `incidentnumber`, `incidentdate` | +| POST | `/fd_inspection_business_name.php` | `fd_business` | +| POST | `/fd_inspection_business_address.php` | `fd_address` | + +The RFD root displays a City disclaimer covering completeness, accuracy, timeliness, and warranties; no separate site-specific terms link was exposed by the reviewed page. `robots.txt` disallows `/inspection`. This adapter does not crawl, enumerate, paginate, or discover URLs. It performs one explicit user lookup at a time. Inspection result pages contain report and invoice links; report identifiers and validated source links are preserved, while invoice links are discarded and never followed or emitted. + +## Transport Gate + +RFD did not provide a usable TLS endpoint during review. The general Raleigh HTTP client remains HTTPS-only. A separate RFD client permits only `http://rfdreports.net` on the default port and only the four contracts above. It rejects redirects, extra parameters, alternate hosts, arbitrary paths, oversized responses, and unrecognized HTML. + +Every RFD operation requires `--acknowledge-insecure-rfd`. This acknowledgement is invocation-local and is never inferred from configuration. Date fallback additionally requires `--allow-rfd-fallback` and runs only after an exact ArcGIS date query returns no records. Inspection business names and addresses cross the network unencrypted. + +## Commands + +```bash +# Structured ArcGIS only +scripts/raleigh fire reports --date 2026-07-24 +scripts/raleigh fire reports --incident-number 26-032170 + +# One exact narrative after the ArcGIS incident resolves its date +scripts/raleigh fire reports --incident-number 26-032170 \ + --include-narrative --acknowledge-insecure-rfd + +# Date fallback only if ArcGIS returns no records +scripts/raleigh fire reports --date 2026-07-24 \ + --allow-rfd-fallback --acknowledge-insecure-rfd + +# RFD inspection forms +scripts/raleigh fire inspections --business "Example Market" \ + --acknowledge-insecure-rfd +scripts/raleigh fire inspections --address "100 Example St" \ + --acknowledge-insecure-rfd +``` + +## Failure Policy + +- Empty or whitespace-only selectors are rejected before network access. +- Missing ArcGIS fields, invalid features, or a result-cap overflow fail visibly. +- Changed RFD headers, malformed rows, invalid links, mismatched narrative identifiers, and error pages fail visibly. +- Empty ArcGIS output is missing evidence, not proof that no report exists; the rolling feed may lag. +- Empty recognized RFD tables are valid no-result responses. + +## Exclusions + +No bulk crawling, report enumeration, authentication, invoice retrieval or payment, private contact access, write operation, arbitrary URL traversal, or inspection-report detail retrieval is implemented. diff --git a/raleigh/scripts/raleighlib/cli.py b/raleigh/scripts/raleighlib/cli.py index 12613bc..09a6ef0 100644 --- a/raleigh/scripts/raleighlib/cli.py +++ b/raleigh/scripts/raleighlib/cli.py @@ -28,6 +28,7 @@ from raleighlib import meetings from raleighlib import police from raleighlib import fire from raleighlib import fire_protection +from raleighlib import rfd_reports from raleighlib import incidents @@ -112,10 +113,21 @@ def _longitude(value: str) -> float: def _iso_date(value: str) -> str: + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): + raise argparse.ArgumentTypeError("date must use YYYY-MM-DD") try: - date.fromisoformat(value) + parsed = date.fromisoformat(value) except ValueError as exc: raise argparse.ArgumentTypeError("date must use YYYY-MM-DD") from exc + return parsed.isoformat() + + +def _nonempty_text(value: str) -> str: + value = value.strip() + if not value: + raise argparse.ArgumentTypeError("value must not be empty") + if len(value) > 200 or any(ord(char) < 32 for char in value): + raise argparse.ArgumentTypeError("value is invalid") return value @@ -439,6 +451,20 @@ def build_parser() -> argparse.ArgumentParser: fire_prot_group.add_argument("--address", help="Address to geocode and resolve to a CSAID.") fire_prot_group.add_argument("--csaid", type=_csaid_value, help="Canonical site-address identifier (CSAID).") + fire_reports = fire_sub.add_parser("reports", help="Exact ArcGIS fire-report lookup with guarded RFD fallback.") + fire_reports_group = fire_reports.add_mutually_exclusive_group(required=True) + fire_reports_group.add_argument("--date", type=_iso_date, help="Exact dispatch date (YYYY-MM-DD).") + fire_reports_group.add_argument("--incident-number", type=_nonempty_text, help="Exact incident number.") + fire_reports.add_argument("--allow-rfd-fallback", action="store_true", help="Use the RFD date form only when ArcGIS returns no records.") + fire_reports.add_argument("--include-narrative", action="store_true", help="Fetch one exact incident narrative (requires --incident-number).") + fire_reports.add_argument("--acknowledge-insecure-rfd", action="store_true", help="Acknowledge that RFD data crosses unencrypted HTTP for this invocation.") + + fire_inspections = fire_sub.add_parser("inspections", help="Search fragile RFD inspection records over acknowledged HTTP.") + fire_inspections_group = fire_inspections.add_mutually_exclusive_group(required=True) + fire_inspections_group.add_argument("--business", type=_nonempty_text, help="Nonempty business-name search.") + fire_inspections_group.add_argument("--address", type=_nonempty_text, help="Nonempty address search.") + fire_inspections.add_argument("--acknowledge-insecure-rfd", action="store_true", help="Acknowledge that the search crosses unencrypted HTTP for this invocation.") + incidents_p = sub.add_parser("incidents", help="Raleigh-Wake ECC active incident feed (undocumented).") incidents_sub = incidents_p.add_subparsers(dest="incidents_command") incidents_active = incidents_sub.add_parser("active", help="Currently active incidents.") @@ -1406,6 +1432,104 @@ def cmd_fire_protection(args: argparse.Namespace) -> int: return 0 +def _fire_reports_output(result: dict[str, Any], args: argparse.Namespace) -> int: + if args.json: + _output_json(result) + return 0 + reports = result.get("reports", []) + if reports: + rows = [] + for report in reports: + dispatched = report.get("dispatch_date_time") + rows.append([ + str(report.get("source") or ""), + str(report.get("incident_number") or ""), + _format_ms_datetime(dispatched) if dispatched is not None else str(report.get("incident_date") or ""), + str(report.get("incident_type_name") or ""), + str(report.get("address") or ""), + ]) + _output_table(["SOURCE", "INCIDENT", "DATE", "TYPE", "ADDRESS"], rows) + else: + print("No records returned; this does not prove no report exists.") + narrative = result.get("narrative") + if isinstance(narrative, dict) and narrative.get("narrative"): + print(f"\nNarrative ({narrative.get('incident_number', '')}):") + print(narrative["narrative"]) + for warning in result.get("warnings", []): + print(f"Warning: {warning}", file=sys.stderr) + return 0 + + +def cmd_fire_reports(args: argparse.Namespace) -> int: + if args.include_narrative and not args.incident_number: + raise cli_error("--include-narrative requires --incident-number") + if (args.allow_rfd_fallback or args.include_narrative) and not args.acknowledge_insecure_rfd: + raise cli_error("RFD access requires --acknowledge-insecure-rfd") + + result = fire.query_reports( + report_date=args.date, + incident_number=args.incident_number, + ) + if not result["reports"] and args.allow_rfd_fallback: + if not args.date: + raise cli_error("--allow-rfd-fallback requires --date") + print(f"Warning: {rfd_reports.INSECURE_WARNING}", file=sys.stderr) + result["reports"] = rfd_reports.search_date( + args.date, acknowledged=args.acknowledge_insecure_rfd + ) + result["sources"].append({ + "source": "rfd-html", + "url": rfd_reports.BASE_URL + rfd_reports.DATE_PATH, + "source_fragility": "fragile-html-over-http", + }) + result["warnings"].append(rfd_reports.INSECURE_WARNING) + + if args.include_narrative: + if len(result["reports"]) != 1: + raise cli_error("exactly one ArcGIS report is required to fetch a narrative") + report = result["reports"][0] + dispatched = report.get("dispatch_date_time") + if isinstance(dispatched, bool) or not isinstance(dispatched, (int, float)): + raise cli_error("the ArcGIS report has no usable dispatch date for narrative lookup") + incident_date = datetime.fromtimestamp( + dispatched / 1000, timezone.utc + ).date().isoformat() + print(f"Warning: {rfd_reports.INSECURE_WARNING}", file=sys.stderr) + result["narrative"] = rfd_reports.fetch_narrative( + report["incident_number"], + incident_date, + acknowledged=args.acknowledge_insecure_rfd, + ) + result["warnings"].append(rfd_reports.INSECURE_WARNING) + return _fire_reports_output(result, args) + + +def cmd_fire_inspections(args: argparse.Namespace) -> int: + if not args.acknowledge_insecure_rfd: + raise cli_error("RFD access requires --acknowledge-insecure-rfd") + print(f"Warning: {rfd_reports.INSECURE_WARNING}", file=sys.stderr) + result = rfd_reports.search_inspections( + business=args.business, + address=args.address, + acknowledged=True, + ) + if args.json: + _output_json(result) + else: + rows = [[ + str(item.get("source") or ""), + str(item.get("inspection_number") or ""), + str(item.get("completed_date") or ""), + str(item.get("business_name") or ""), + str(item.get("address") or ""), + ] for item in result["inspections"]] + if rows: + _output_table(["SOURCE", "INSPECTION", "COMPLETED", "BUSINESS", "ADDRESS"], rows) + else: + print("No inspection records returned.") + return 0 + + def cmd_incidents_active(args: argparse.Namespace) -> int: result = incidents.fetch_active( agency=args.agency, @@ -1496,6 +1620,8 @@ _FIRE_COMMANDS: dict[str, Any] = { "incidents": cmd_fire_incidents, "response-times": cmd_fire_response_times, "protection": cmd_fire_protection, + "reports": cmd_fire_reports, + "inspections": cmd_fire_inspections, } _INCIDENTS_COMMANDS: dict[str, Any] = { @@ -1624,7 +1750,7 @@ def main(argv: list[str] | None = None) -> int: parser.print_help() return 2 - except (core.SecurityError, core.RequestPolicyError, core.ResponseTooLargeError, hub.CatalogError, civic.ResourceError, development.UnsupportedEndpointError, imagery.CapabilityError, police.PoliceError, fire.FireError, fire_protection.FireProtectionError, incidents.IncidentFeedError, FileExistsError, ValueError, KeyError) as exc: + except (core.SecurityError, core.RequestPolicyError, core.ResponseTooLargeError, hub.CatalogError, civic.ResourceError, development.UnsupportedEndpointError, imagery.CapabilityError, police.PoliceError, fire.FireError, fire_protection.FireProtectionError, rfd_reports.RFDReportError, incidents.IncidentFeedError, FileExistsError, ValueError, KeyError) as exc: print(f"Error: {exc}", file=sys.stderr) return 1 except urllib.error.HTTPError as exc: diff --git a/raleigh/scripts/raleighlib/fire.py b/raleigh/scripts/raleighlib/fire.py index a018628..17e6142 100644 --- a/raleigh/scripts/raleighlib/fire.py +++ b/raleigh/scripts/raleighlib/fire.py @@ -13,7 +13,7 @@ import math import re import sys import urllib.parse -from datetime import datetime, timezone +from datetime import date, datetime, timedelta, timezone from typing import Any from raleighlib import arcgis @@ -67,6 +67,20 @@ PRIVACY_CAVERAT = ( "responses and must not be used for emergency response." ) +REPORT_FIELDS = ( + "incident_number", + "dispatch_date_time", + "arrive_date_time", + "cleared_date_time", + "address", + "station_name", + "platoon", + "incident_group_name", + "incident_subgroup_code", + "incident_type_name", +) +REPORT_LIMIT = 200 + _STATION_NAME_RE = re.compile(r"^station\s*0*(\d+)\s*$", re.IGNORECASE) @@ -409,3 +423,84 @@ def query_incidents( ], "features": features, } + + +def query_reports( + *, + report_date: str | None = None, + incident_number: str | None = None, +) -> dict[str, Any]: + """Query the authoritative past-month layer by one exact bounded selector.""" + if bool(report_date) == bool(incident_number): + raise FireError("provide exactly one report date or incident number") + + layer_url = resolve_layer_url("past-month") + available_fields = _discover_fields(layer_url) + missing = set(REPORT_FIELDS) - available_fields + if missing: + raise FireError( + "Fire report source schema drift; missing fields: " + + ", ".join(sorted(missing)) + ) + + query: dict[str, str | None] = { + "date": report_date, + "incident_number": incident_number, + } + if report_date: + try: + start = date.fromisoformat(report_date) + except ValueError as exc: + raise FireError("report date must use YYYY-MM-DD") from exc + end = start + timedelta(days=1) + where = ( + f"dispatch_date_time >= TIMESTAMP '{start.isoformat()} 00:00:00' AND " + f"dispatch_date_time < TIMESTAMP '{end.isoformat()} 00:00:00'" + ) + else: + number = (incident_number or "").strip() + if not number: + raise FireError("incident number must not be empty") + where = f"incident_number = '{_escape_sql_value(number)}'" + + response = arcgis.query_layer( + layer_url, + where=where, + out_fields=",".join(REPORT_FIELDS), + return_geometry=False, + result_record_count=REPORT_LIMIT, + order_by_fields="dispatch_date_time ASC,incident_number ASC", + ) + records = response.get("features") + if not isinstance(records, list): + raise FireError("Fire report source returned invalid features") + if response.get("exceededTransferLimit"): + raise FireError(f"Fire report query exceeded the {REPORT_LIMIT}-record safety limit") + + retrieved_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + reports: list[dict[str, Any]] = [] + for record in records: + if not isinstance(record, dict) or not isinstance(record.get("attributes"), dict): + raise FireError("Fire report source returned a malformed record") + attrs = record["attributes"] + if not _present(attrs.get("incident_number")): + raise FireError("Fire report source returned a record without incident_number") + reports.append({ + "source": "arcgis", + "source_fragility": "authoritative-structured", + **{field: attrs.get(field) for field in REPORT_FIELDS}, + }) + + return { + "query": query, + "reports": reports, + "sources": [{ + "source": "arcgis", + "item_id": RFD_SOURCES["past-month"]["item_id"], + "url": layer_url, + "retrieved_at": retrieved_at, + }], + "warnings": [ + "The rolling ArcGIS feed may lag the RFD Report System and is not proof of completeness." + ], + } diff --git a/raleigh/scripts/raleighlib/rfd_reports.py b/raleigh/scripts/raleighlib/rfd_reports.py new file mode 100644 index 0000000..fe22927 --- /dev/null +++ b/raleigh/scripts/raleighlib/rfd_reports.py @@ -0,0 +1,308 @@ +"""Guarded adapter for the fragile plain-HTTP RFD Report System.""" + +from __future__ import annotations + +import html +import re +import urllib.error +import urllib.parse +import urllib.request +from datetime import date, datetime, timezone +from html.parser import HTMLParser +from typing import Any + +from raleighlib import core + +BASE_URL = "http://rfdreports.net" +DATE_PATH = "/fd_date.php" +NARRATIVE_PATH = "/fd_incidentreport.php" +BUSINESS_PATH = "/fd_inspection_business_name.php" +ADDRESS_PATH = "/fd_inspection_business_address.php" +MAX_HTML_BYTES = 2 * 1024 * 1024 +INSECURE_WARNING = ( + "RFD Report System uses unencrypted HTTP; search terms and returned data " + "can be observed or altered in transit." +) + +_REPORT_HEADERS = [ + "Incident Data", "Incident Type", "Incident #", "Dispatch Time", + "Arrive Time", "Clear Time", "Address", "Unit", "Cross Street", "View Report", +] +_INSPECTION_HEADERS = [ + "Occupancy Name", "Address", "Inspection Type", "Data Completed", + "View Report", "View Invoice", +] +_TERMINAL_CONTROLS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]") + + +class RFDReportError(ValueError): + """Raised when an RFD request or HTML contract is unsafe or incompatible.""" + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise RFDReportError("RFD redirects are not allowed") + + +_OPENER = urllib.request.build_opener(_NoRedirect) + + +class _TableParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.rows: list[list[dict[str, Any]]] = [] + self._row: list[dict[str, Any]] | None = None + self._cell: dict[str, Any] | None = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + tag = tag.casefold() + if tag == "tr": + if self._row: + self.rows.append(self._row) + self._row = [] + elif tag in {"th", "td"} and self._row is not None: + self._cell = {"text": [], "links": []} + elif tag == "a" and self._cell is not None: + href = dict(attrs).get("href") + if href: + self._cell["links"].append(href) + + def handle_data(self, data: str) -> None: + if self._cell is not None: + self._cell["text"].append(data) + + def handle_endtag(self, tag: str) -> None: + tag = tag.casefold() + if tag in {"th", "td"} and self._cell is not None and self._row is not None: + text = _safe_text(" ".join(self._cell["text"])) + self._row.append({"text": html.unescape(text), "links": self._cell["links"]}) + self._cell = None + elif tag == "tr" and self._row is not None: + if self._row: + self.rows.append(self._row) + self._row = None + + +def _require_text(value: str, label: str) -> str: + value = value.strip() + if not value: + raise RFDReportError(f"{label} must not be empty") + if len(value) > 200 or any(ord(char) < 32 for char in value): + raise RFDReportError(f"{label} is invalid") + return value + + +def _safe_text(value: str) -> str: + """Remove terminal controls from untrusted HTTP text and normalize whitespace.""" + return " ".join(_TERMINAL_CONTROLS_RE.sub("", value).split()) + + +def _validate_contract(path: str, params: dict[str, str], method: str) -> str: + contracts = { + DATE_PATH: ("POST", {"date"}), + NARRATIVE_PATH: ("GET", {"incidentnumber", "incidentdate"}), + BUSINESS_PATH: ("POST", {"fd_business"}), + ADDRESS_PATH: ("POST", {"fd_address"}), + } + expected = contracts.get(path) + if expected != (method, set(params)): + raise RFDReportError("unsupported RFD request contract") + return BASE_URL + path + + +def _request(path: str, params: dict[str, str], method: str, *, acknowledged: bool) -> str: + if not acknowledged: + raise RFDReportError("RFD access requires --acknowledge-insecure-rfd") + url = _validate_contract(path, params, method) + encoded = urllib.parse.urlencode(params) + data = encoded.encode("utf-8") if method == "POST" else None + if method == "GET": + url = f"{url}?{encoded}" + headers = {"User-Agent": core.USER_AGENT, "Accept": "text/html"} + if data is not None: + headers["Content-Type"] = "application/x-www-form-urlencoded" + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with _OPENER.open(request, timeout=core._get_timeout()) as response: + final = urllib.parse.urlparse(response.geturl()) + if final.scheme != "http" or final.hostname != "rfdreports.net" or final.port not in (None, 80): + raise RFDReportError("RFD response left the fixed insecure origin") + body = core._read_limited(response, MAX_HTML_BYTES) + except urllib.error.HTTPError as exc: + raise RFDReportError(f"RFD returned HTTP {exc.code}") from exc + text = body.decode("utf-8", errors="replace") + lowered = text.casefold() + if "internal server error" in lowered or "service unavailable" in lowered: + raise RFDReportError("RFD returned an error page") + return text + + +def _table_rows(html_text: str, expected_headers: list[str]) -> list[list[dict[str, Any]]]: + parser = _TableParser() + parser.feed(html_text) + if not parser.rows: + if re.search(r"\b(no (?:records|results|reports|inspections) (?:found|available))\b", html_text, re.I): + return [] + raise RFDReportError("RFD HTML contract drift: no result table") + headers = [cell["text"] for cell in parser.rows[0]] + if headers != expected_headers: + raise RFDReportError("RFD HTML contract drift: unexpected table headers") + rows = parser.rows[1:] + for row in rows: + if len(row) != len(expected_headers): + raise RFDReportError("RFD HTML contract drift: malformed result row") + return rows + + +def _canonical_link(href: str, path: str, required: set[str]) -> str: + parsed = urllib.parse.urlparse(urllib.parse.urljoin(BASE_URL + "/", href)) + params = urllib.parse.parse_qs(parsed.query, keep_blank_values=True) + if parsed.scheme != "http" or parsed.hostname != "rfdreports.net" or parsed.path != path: + raise RFDReportError("RFD HTML contract drift: unexpected result link") + if set(params) != required or any(len(values) != 1 for values in params.values()): + raise RFDReportError("RFD HTML contract drift: malformed result link") + return urllib.parse.urlunparse(("http", "rfdreports.net", path, "", urllib.parse.urlencode({k: v[0] for k, v in params.items()}), "")) + + +def _inspection_link(href: str, number: str, address: str, name: str) -> str: + """Validate the fixed report link, then repair upstream's unescaped # values.""" + parsed = urllib.parse.urlparse(urllib.parse.urljoin(BASE_URL + "/", href)) + params = urllib.parse.parse_qs(parsed.query, keep_blank_values=True) + if parsed.scheme != "http" or parsed.hostname != "rfdreports.net" or parsed.path != "/fd_report.php": + raise RFDReportError("RFD HTML contract drift: unexpected inspection link") + if params.get("inspection_number") != [number]: + raise RFDReportError("RFD HTML contract drift: inspection link identifier mismatch") + return BASE_URL + "/fd_report.php?" + urllib.parse.urlencode({ + "inspection_number": number, + "address": address, + "name": name, + }) + + +def search_date(report_date: str, *, acknowledged: bool) -> list[dict[str, Any]]: + report_date = _require_text(report_date, "date") + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", report_date): + raise RFDReportError("date must use YYYY-MM-DD") + try: + requested_date = date.fromisoformat(report_date) + except ValueError as exc: + raise RFDReportError("date must use YYYY-MM-DD") from exc + html_text = _request(DATE_PATH, {"date": report_date}, "POST", acknowledged=acknowledged) + reports: list[dict[str, Any]] = [] + for row in _table_rows(html_text, _REPORT_HEADERS): + values = [cell["text"] for cell in row] + links = row[9]["links"] + if not values[2] or len(links) != 1: + raise RFDReportError("RFD HTML contract drift: report identifier or link missing") + source_url = _canonical_link( + links[0], NARRATIVE_PATH, {"incidentnumber", "incidentdate"} + ) + try: + row_date = datetime.strptime(values[0], "%m/%d/%Y").date() + except ValueError as exc: + raise RFDReportError("RFD HTML contract drift: invalid incident date") from exc + source_params = urllib.parse.parse_qs(urllib.parse.urlparse(source_url).query) + if row_date != requested_date: + raise RFDReportError("RFD response included a report outside the requested date") + if source_params.get("incidentnumber") != [values[2]] or source_params.get("incidentdate") != [report_date]: + raise RFDReportError("RFD report row and source link do not match") + reports.append({ + "source": "rfd-html", + "source_fragility": "fragile-html-over-http", + "incident_date": values[0], + "incident_type_name": values[1], + "incident_number": values[2], + "dispatch_time": values[3], + "arrive_time": values[4], + "clear_time": values[5], + "address": values[6], + "unit": values[7], + "cross_street": values[8], + "source_url": source_url, + }) + return reports + + +def search_inspections(*, business: str | None = None, address: str | None = None, acknowledged: bool) -> dict[str, Any]: + if bool(business) == bool(address): + raise RFDReportError("provide exactly one business name or address") + if business is not None: + query = _require_text(business, "business name") + path, params = BUSINESS_PATH, {"fd_business": query} + mode = "business" + else: + query = _require_text(address or "", "address") + path, params = ADDRESS_PATH, {"fd_address": query} + mode = "address" + html_text = _request(path, params, "POST", acknowledged=acknowledged) + inspections: list[dict[str, Any]] = [] + for row in _table_rows(html_text, _INSPECTION_HEADERS): + values = [cell["text"] for cell in row] + report_links = row[4]["links"] + if not values[4] or len(report_links) != 1: + raise RFDReportError("RFD HTML contract drift: inspection identifier or link missing") + source_url = _inspection_link( + report_links[0], values[4], values[1], values[0] + ) + inspections.append({ + "source": "rfd-html", + "source_fragility": "fragile-html-over-http", + "business_name": values[0], + "address": values[1], + "inspection_type": values[2], + "completed_date": values[3], + "inspection_number": values[4], + "source_url": source_url, + }) + return { + "query": {mode: query}, + "inspections": inspections, + "source": { + "source": "rfd-html", + "url": BASE_URL + path, + "retrieved_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + }, + "warnings": [INSECURE_WARNING], + } + + +def fetch_narrative(incident_number: str, incident_date: str, *, acknowledged: bool) -> dict[str, Any]: + incident_number = _require_text(incident_number, "incident number") + incident_date = _require_text(incident_date, "incident date") + html_text = _request( + NARRATIVE_PATH, + {"incidentnumber": incident_number, "incidentdate": incident_date}, + "GET", + acknowledged=acknowledged, + ) + if "City of Raleigh Fire Department Basic Fire Report" not in html_text: + raise RFDReportError("RFD narrative page identity was not recognized") + number_match = re.search(r"Incident Number:\s*([^<]+)", html_text, re.I) + date_match = re.search(r"Alarm Date:\s*([^<]+)", html_text, re.I) + narrative_match = re.search( + r"\s*narrative(?:\"|\")?\s*:\s*(.*?)", html_text, re.I | re.S + ) + if not number_match or not date_match or not narrative_match: + raise RFDReportError("RFD HTML contract drift: narrative fields missing") + returned_number = " ".join(html.unescape(number_match.group(1)).split()) + returned_date = " ".join(html.unescape(date_match.group(1)).split()) + if returned_number != incident_number or returned_date != incident_date: + raise RFDReportError("RFD narrative response did not match the requested incident") + narrative = _safe_text(re.sub( + r'"?\s*}\]\s*$', + "", + html.unescape(narrative_match.group(1)).strip().lstrip('"').strip(), + )) + if not narrative: + raise RFDReportError("RFD narrative was empty") + return { + "source": "rfd-html", + "source_fragility": "fragile-html-over-http", + "incident_number": returned_number, + "incident_date": returned_date, + "narrative": narrative, + "source_url": BASE_URL + NARRATIVE_PATH + "?" + urllib.parse.urlencode({ + "incidentnumber": incident_number, + "incidentdate": incident_date, + }), + } diff --git a/raleigh/tests/fixtures/fire-reports-empty.json b/raleigh/tests/fixtures/fire-reports-empty.json new file mode 100644 index 0000000..00a3d5c --- /dev/null +++ b/raleigh/tests/fixtures/fire-reports-empty.json @@ -0,0 +1 @@ +{"features": []} diff --git a/raleigh/tests/fixtures/fire-reports-recent-lag.json b/raleigh/tests/fixtures/fire-reports-recent-lag.json new file mode 100644 index 0000000..fc828e5 --- /dev/null +++ b/raleigh/tests/fixtures/fire-reports-recent-lag.json @@ -0,0 +1 @@ +{"features": [], "exceededTransferLimit": false} diff --git a/raleigh/tests/fixtures/fire-reports-results.json b/raleigh/tests/fixtures/fire-reports-results.json new file mode 100644 index 0000000..7d89b71 --- /dev/null +++ b/raleigh/tests/fixtures/fire-reports-results.json @@ -0,0 +1,18 @@ +{ + "features": [ + { + "attributes": { + "incident_number": "26-032170", + "dispatch_date_time": 1784866612000, + "arrive_date_time": 1784866814000, + "cleared_date_time": 1784868386000, + "address": "505 FLORENCE ST", + "station_name": "Station 01", + "platoon": "A", + "incident_group_name": "Hazardous Situation", + "incident_subgroup_code": "Hazardous Materials", + "incident_type_name": "Gas Leak / Gas Odor" + } + } + ] +} diff --git a/raleigh/tests/fixtures/fire-reports-schema-drift.json b/raleigh/tests/fixtures/fire-reports-schema-drift.json new file mode 100644 index 0000000..895e1a2 --- /dev/null +++ b/raleigh/tests/fixtures/fire-reports-schema-drift.json @@ -0,0 +1,6 @@ +{ + "fields": [ + {"name": "incident_number"}, + {"name": "dispatch_date_time"} + ] +} diff --git a/raleigh/tests/fixtures/fire-reports-service-error.json b/raleigh/tests/fixtures/fire-reports-service-error.json new file mode 100644 index 0000000..3d940dc --- /dev/null +++ b/raleigh/tests/fixtures/fire-reports-service-error.json @@ -0,0 +1 @@ +{"error": {"code": 500, "message": "Service unavailable", "details": []}} diff --git a/raleigh/tests/fixtures/rfd-date-empty.html b/raleigh/tests/fixtures/rfd-date-empty.html new file mode 100644 index 0000000..5314bce --- /dev/null +++ b/raleigh/tests/fixtures/rfd-date-empty.html @@ -0,0 +1,3 @@ + + +
Incident DataIncident TypeIncident #Dispatch TimeArrive TimeClear TimeAddressUnitCross StreetView Report
diff --git a/raleigh/tests/fixtures/rfd-date-malformed.html b/raleigh/tests/fixtures/rfd-date-malformed.html new file mode 100644 index 0000000..367bbe6 --- /dev/null +++ b/raleigh/tests/fixtures/rfd-date-malformed.html @@ -0,0 +1,4 @@ + + + +
Incident DataIncident TypeIncident #Dispatch TimeArrive TimeClear TimeAddressUnitCross StreetView Report
07/24/2026Gas Leak26-032170
diff --git a/raleigh/tests/fixtures/rfd-date-markup-drift.html b/raleigh/tests/fixtures/rfd-date-markup-drift.html new file mode 100644 index 0000000..39cbf85 --- /dev/null +++ b/raleigh/tests/fixtures/rfd-date-markup-drift.html @@ -0,0 +1 @@ +
Changed Incident HeaderIncident #
diff --git a/raleigh/tests/fixtures/rfd-date-results.html b/raleigh/tests/fixtures/rfd-date-results.html new file mode 100644 index 0000000..1ecc164 --- /dev/null +++ b/raleigh/tests/fixtures/rfd-date-results.html @@ -0,0 +1,4 @@ + + + +
Incident DataIncident TypeIncident #Dispatch TimeArrive TimeClear TimeAddressUnitCross StreetView Report
07/24/2026Gas Leak / Gas Odor26-03217000:16:5200:20:1400:46:26505 FLORENCE ST101Report
diff --git a/raleigh/tests/fixtures/rfd-error-page.html b/raleigh/tests/fixtures/rfd-error-page.html new file mode 100644 index 0000000..c66e6c6 --- /dev/null +++ b/raleigh/tests/fixtures/rfd-error-page.html @@ -0,0 +1 @@ +Service UnavailableService unavailable diff --git a/raleigh/tests/fixtures/rfd-inspection-empty.html b/raleigh/tests/fixtures/rfd-inspection-empty.html new file mode 100644 index 0000000..1c77754 --- /dev/null +++ b/raleigh/tests/fixtures/rfd-inspection-empty.html @@ -0,0 +1,3 @@ + + +
Occupancy NameAddressInspection TypeData CompletedView ReportView Invoice
diff --git a/raleigh/tests/fixtures/rfd-inspection-results.html b/raleigh/tests/fixtures/rfd-inspection-results.html new file mode 100644 index 0000000..803fe73 --- /dev/null +++ b/raleigh/tests/fixtures/rfd-inspection-results.html @@ -0,0 +1,4 @@ + + + +
Occupancy NameAddressInspection TypeData CompletedView ReportView Invoice
EXAMPLE MARKET #1100 EXAMPLE ST, RALEIGH, NC1YR - Inspection - Yearly04/07/2026RFD-2026-0004935RFD-2026-0004429
diff --git a/raleigh/tests/fixtures/rfd-narrative-result.html b/raleigh/tests/fixtures/rfd-narrative-result.html new file mode 100644 index 0000000..090a306 --- /dev/null +++ b/raleigh/tests/fixtures/rfd-narrative-result.html @@ -0,0 +1,7 @@ + +

City of Raleigh Fire Department Basic Fire Report

+

Alarm Date: 2026-07-24

+

Incident Number: 26-032170

+

Narrative

+

narrative": "Crew investigated a reported gas odor and ventilated the residence."}]

+ diff --git a/raleigh/tests/test_raleigh.py b/raleigh/tests/test_raleigh.py index 7a5261c..7a3f907 100644 --- a/raleigh/tests/test_raleigh.py +++ b/raleigh/tests/test_raleigh.py @@ -44,6 +44,7 @@ import raleighlib.meetings as meetings import raleighlib.police as police import raleighlib.fire as fire import raleighlib.fire_protection as fire_protection +import raleighlib.rfd_reports as rfd_reports from raleighlib import cli as cli_lib CLI_SCRIPT = _SCRIPT_DIR / "raleigh" @@ -51,16 +52,23 @@ cli = importlib.machinery.SourceFileLoader("raleigh_cli", str(CLI_SCRIPT)).load_ def setUpModule(): - global _network_guard + global _network_guard, _rfd_network_guard _network_guard = patch.object( core._OPENER, "open", side_effect=AssertionError("Raleigh unit tests must not make live network calls"), ) _network_guard.start() + _rfd_network_guard = patch.object( + rfd_reports._OPENER, + "open", + side_effect=AssertionError("Raleigh unit tests must not make live RFD calls"), + ) + _rfd_network_guard.start() def tearDownModule(): + _rfd_network_guard.stop() _network_guard.stop() @@ -2982,6 +2990,215 @@ class FireTests(unittest.TestCase): self.assertEqual(code, 0) +class FireReportTests(unittest.TestCase): + """Exact ArcGIS report queries and guarded RFD fallback behavior.""" + + FIXTURES = pathlib.Path(__file__).parent / "fixtures" + FIELDS = set(fire.REPORT_FIELDS) | {"OBJECTID"} + + def _fixture(self, name: str): + return json.loads((self.FIXTURES / name).read_text()) + + def _query(self, response=None, fields=None): + return patch("raleighlib.fire.resolve_layer_url", return_value="https://services.arcgis.com/example/FeatureServer/0"), patch( + "raleighlib.fire._discover_fields", return_value=fields or self.FIELDS + ), patch("raleighlib.fire.arcgis.query_layer", return_value=response or self._fixture("fire-reports-empty.json")) + + def test_exact_date_query_is_bounded_and_selective(self): + p1, p2, p3 = self._query(self._fixture("fire-reports-results.json")) + with p1, p2, p3 as query: + result = fire.query_reports(report_date="2026-07-24") + kwargs = query.call_args.kwargs + self.assertIn("dispatch_date_time >= TIMESTAMP '2026-07-24 00:00:00'", kwargs["where"]) + self.assertIn("dispatch_date_time < TIMESTAMP '2026-07-25 00:00:00'", kwargs["where"]) + self.assertEqual(kwargs["out_fields"], ",".join(fire.REPORT_FIELDS)) + self.assertFalse(kwargs["return_geometry"]) + self.assertEqual(kwargs["result_record_count"], fire.REPORT_LIMIT) + self.assertEqual(result["reports"][0]["source"], "arcgis") + + def test_exact_incident_number_is_escaped(self): + p1, p2, p3 = self._query() + with p1, p2, p3 as query: + fire.query_reports(incident_number="26-'123") + self.assertEqual(query.call_args.kwargs["where"], "incident_number = '26-''123'") + + def test_report_query_requires_exactly_one_selector(self): + with self.assertRaisesRegex(fire.FireError, "exactly one"): + fire.query_reports() + with self.assertRaisesRegex(fire.FireError, "exactly one"): + fire.query_reports(report_date="2026-07-24", incident_number="26-1") + + def test_schema_drift_fails_closed(self): + drift = self._fixture("fire-reports-schema-drift.json") + fields = {field["name"] for field in drift["fields"]} + p1, p2, p3 = self._query(fields=fields) + with p1, p2, p3, self.assertRaisesRegex(fire.FireError, "schema drift"): + fire.query_reports(report_date="2026-07-24") + + def test_service_error_is_visible(self): + service_error = self._fixture("fire-reports-service-error.json") + def fail_query(*args, **kwargs): + core.raise_for_arcgis_error(service_error, "ArcGIS query") + p1, p2, _ = self._query() + with p1, p2, patch("raleighlib.fire.arcgis.query_layer", side_effect=fail_query): + with self.assertRaisesRegex(ValueError, "Service unavailable"): + fire.query_reports(report_date="2026-07-24") + + def test_malformed_arcgis_record_fails(self): + p1, p2, p3 = self._query({"features": [{"attributes": {}}]}) + with p1, p2, p3, self.assertRaisesRegex(fire.FireError, "without incident_number"): + fire.query_reports(report_date="2026-07-24") + + def test_recent_lag_fallback_requires_both_flags(self): + empty = {"query": {"date": "2026-07-24", "incident_number": None}, "reports": [], "sources": [], "warnings": []} + with patch("raleighlib.cli.fire.query_reports", return_value=empty), patch("raleighlib.cli.rfd_reports.search_date") as fallback: + with self.assertRaisesRegex(SystemExit, "acknowledge"): + cli.main(["fire", "reports", "--date", "2026-07-24", "--allow-rfd-fallback"]) + fallback.assert_not_called() + + def test_recent_lag_invokes_one_bounded_fallback(self): + lag = self._fixture("fire-reports-recent-lag.json") + empty = {"query": {"date": "2026-07-24", "incident_number": None}, "reports": lag["features"], "sources": [], "warnings": []} + fallback_result = [{"source": "rfd-html", "incident_number": "26-032170"}] + with patch("raleighlib.cli.fire.query_reports", return_value=empty), patch("raleighlib.cli.rfd_reports.search_date", return_value=fallback_result) as fallback: + out, err = io.StringIO(), io.StringIO() + with redirect_stdout(out), redirect_stderr(err): + code = cli.main(["--json", "fire", "reports", "--date", "2026-07-24", "--allow-rfd-fallback", "--acknowledge-insecure-rfd"]) + self.assertEqual(code, 0) + fallback.assert_called_once_with("2026-07-24", acknowledged=True) + self.assertEqual(json.loads(out.getvalue())["reports"][0]["source"], "rfd-html") + self.assertIn("unencrypted HTTP", err.getvalue()) + + def test_narrative_requires_exact_incident(self): + with patch("raleighlib.cli.fire.query_reports") as query: + with self.assertRaisesRegex(SystemExit, "requires --incident-number"): + cli.main(["fire", "reports", "--date", "2026-07-24", "--include-narrative", "--acknowledge-insecure-rfd"]) + query.assert_not_called() + + def test_whitespace_incident_number_rejected_before_network(self): + with patch("raleighlib.cli.fire.query_reports") as query: + with self.assertRaises(SystemExit): + cli.main(["fire", "reports", "--incident-number", " "]) + query.assert_not_called() + + def test_noncanonical_iso_date_rejected_before_network(self): + for value in ("20260724", "2026-W30-5"): + with self.subTest(value=value), patch("raleighlib.cli.fire.query_reports") as query: + with self.assertRaises(SystemExit): + cli.main(["fire", "reports", "--date", value]) + query.assert_not_called() + + +class RFDReportAdapterTests(unittest.TestCase): + FIXTURES = pathlib.Path(__file__).parent / "fixtures" + + def _html(self, name: str) -> str: + return (self.FIXTURES / name).read_text() + + def test_insecure_access_is_blocked_before_network(self): + with patch.object(rfd_reports._OPENER, "open") as network: + with self.assertRaisesRegex(rfd_reports.RFDReportError, "acknowledge"): + rfd_reports._request(rfd_reports.DATE_PATH, {"date": "2026-07-24"}, "POST", acknowledged=False) + network.assert_not_called() + + def test_only_four_fixed_contracts_are_allowed(self): + with self.assertRaisesRegex(rfd_reports.RFDReportError, "unsupported"): + rfd_reports._validate_contract("/fd_invoice.php", {"id": "1"}, "GET") + with self.assertRaisesRegex(rfd_reports.RFDReportError, "unsupported"): + rfd_reports._validate_contract(rfd_reports.DATE_PATH, {"wrong": "x"}, "POST") + + def test_date_results_preserve_identifiers_and_source(self): + with patch("raleighlib.rfd_reports._request", return_value=self._html("rfd-date-results.html")): + result = rfd_reports.search_date("2026-07-24", acknowledged=True) + self.assertEqual(result[0]["incident_number"], "26-032170") + self.assertEqual(result[0]["source"], "rfd-html") + self.assertIn("fd_incidentreport.php", result[0]["source_url"]) + + def test_date_result_mismatch_fails_closed(self): + mismatched = self._html("rfd-date-results.html").replace( + "incidentdate=2026-07-24", "incidentdate=2026-07-23" + ) + with patch("raleighlib.rfd_reports._request", return_value=mismatched): + with self.assertRaisesRegex(rfd_reports.RFDReportError, "do not match"): + rfd_reports.search_date("2026-07-24", acknowledged=True) + + def test_date_no_results_is_recognized(self): + with patch("raleighlib.rfd_reports._request", return_value=self._html("rfd-date-empty.html")): + self.assertEqual(rfd_reports.search_date("2026-07-24", acknowledged=True), []) + + def test_date_markup_drift_fails_visibly(self): + with patch("raleighlib.rfd_reports._request", return_value=self._html("rfd-date-markup-drift.html")): + with self.assertRaisesRegex(rfd_reports.RFDReportError, "contract drift"): + rfd_reports.search_date("2026-07-24", acknowledged=True) + + def test_date_malformed_row_fails_visibly(self): + with patch("raleighlib.rfd_reports._request", return_value=self._html("rfd-date-malformed.html")): + with self.assertRaisesRegex(rfd_reports.RFDReportError, "malformed result row"): + rfd_reports.search_date("2026-07-24", acknowledged=True) + + def test_error_page_fails_visibly(self): + with patch("raleighlib.rfd_reports._request", return_value=self._html("rfd-error-page.html")): + with self.assertRaisesRegex(rfd_reports.RFDReportError, "contract drift"): + rfd_reports.search_date("2026-07-24", acknowledged=True) + + def test_inspection_results_never_expose_invoice_links(self): + with patch("raleighlib.rfd_reports._request", return_value=self._html("rfd-inspection-results.html")): + result = rfd_reports.search_inspections(business="Example", acknowledged=True) + encoded = json.dumps(result) + self.assertEqual(result["inspections"][0]["inspection_number"], "RFD-2026-0004935") + source_params = urllib.parse.parse_qs(urllib.parse.urlparse(result["inspections"][0]["source_url"]).query) + self.assertEqual(source_params["name"], ["EXAMPLE MARKET #1"]) + self.assertNotIn("invoice", encoded.casefold()) + self.assertNotIn("0004429", encoded) + + def test_inspection_empty_input_is_rejected(self): + with patch("raleighlib.rfd_reports._request") as request: + with self.assertRaisesRegex(rfd_reports.RFDReportError, "must not be empty"): + rfd_reports.search_inspections(address=" ", acknowledged=True) + request.assert_not_called() + + def test_inspection_empty_state_is_recognized(self): + with patch("raleighlib.rfd_reports._request", return_value=self._html("rfd-inspection-empty.html")): + result = rfd_reports.search_inspections(address="Example", acknowledged=True) + self.assertEqual(result["inspections"], []) + + def test_narrative_is_exact_and_incident_specific(self): + with patch("raleighlib.rfd_reports._request", return_value=self._html("rfd-narrative-result.html")): + result = rfd_reports.fetch_narrative("26-032170", "2026-07-24", acknowledged=True) + self.assertEqual(result["incident_number"], "26-032170") + self.assertIn("ventilated", result["narrative"]) + + def test_narrative_mismatch_fails(self): + with patch("raleighlib.rfd_reports._request", return_value=self._html("rfd-narrative-result.html")): + with self.assertRaisesRegex(rfd_reports.RFDReportError, "did not match"): + rfd_reports.fetch_narrative("26-999999", "2026-07-24", acknowledged=True) + + def test_plain_http_text_removes_terminal_controls(self): + hostile = self._html("rfd-date-results.html").replace( + "505 FLORENCE ST", "505 \x1b]52;c;Y2xpcGJvYXJk\x07 FLORENCE ST" + ) + with patch("raleighlib.rfd_reports._request", return_value=hostile): + result = rfd_reports.search_date("2026-07-24", acknowledged=True) + self.assertNotIn("\x1b", result[0]["address"]) + self.assertNotIn("\x07", result[0]["address"]) + + def test_cli_inspection_requires_acknowledgement(self): + with patch("raleighlib.cli.rfd_reports.search_inspections") as search: + with self.assertRaisesRegex(SystemExit, "acknowledge"): + cli.main(["fire", "inspections", "--business", "Example"]) + search.assert_not_called() + + def test_cli_inspection_json_keeps_warning_on_stderr(self): + payload = {"query": {"business": "Example"}, "inspections": [], "source": {}, "warnings": [rfd_reports.INSECURE_WARNING]} + with patch("raleighlib.cli.rfd_reports.search_inspections", return_value=payload): + out, err = io.StringIO(), io.StringIO() + with redirect_stdout(out), redirect_stderr(err): + code = cli.main(["--json", "fire", "inspections", "--business", "Example", "--acknowledge-insecure-rfd"]) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out.getvalue())["warnings"], [rfd_reports.INSECURE_WARNING]) + self.assertIn("unencrypted HTTP", err.getvalue()) + + _FIXTURES_DIR = pathlib.Path(__file__).parent / "fixtures"