From 22f1d52456cca73edd56fccc44a9ca4dfa8645fa Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Sat, 29 Aug 2026 13:12:42 -0400 Subject: [PATCH] fix(raleigh): keep WAF outages blocking (#424) Keep Cloudflare WAF challenges distinct while preserving a blocking canary signal for unavailable civic adapters. --- raleigh/EVIDENCE-LEDGER.md | 10 ++-- raleigh/references/civic-content-reference.md | 2 +- raleigh/scripts/canary.py | 26 +++++----- raleigh/tests/test_raleigh.py | 52 ++++++++++++++++--- 4 files changed, 65 insertions(+), 25 deletions(-) diff --git a/raleigh/EVIDENCE-LEDGER.md b/raleigh/EVIDENCE-LEDGER.md index 9860b38..b02b93e 100644 --- a/raleigh/EVIDENCE-LEDGER.md +++ b/raleigh/EVIDENCE-LEDGER.md @@ -210,16 +210,16 @@ The following public service boundaries were exercised successfully on 2026-07-2 local request, establishing an access-policy boundary rather than a Raleigh schema failure. - Classify this exact marker as `waf_challenge`, preserve source and target in - the report, and keep the observation non-failing. Other 403 responses remain - `auth_regression` and continue to fail the canary. + the report, and keep it as a blocking availability failure. Other 403 + responses remain `auth_regression` and continue to fail the canary. ### Verification target and follow-up - Deterministic tests cover the marker-specific classification and the - non-failing summary accounting. + blocking summary accounting. - The scheduled workflow remains the delivery-boundary check. A later green - run proves the canary no longer treats this known upstream challenge as a - durable failure; it does not prove Raleigh machine access has been restored. + run requires Raleigh machine access to be restored; a challenged civic + endpoint remains a canary failure. - Do not add retries, browser automation, or challenge bypasses. Reclassify only when the provider removes the marker or a new upstream access contract is verified. diff --git a/raleigh/references/civic-content-reference.md b/raleigh/references/civic-content-reference.md index e39298a..917e315 100644 --- a/raleigh/references/civic-content-reference.md +++ b/raleigh/references/civic-content-reference.md @@ -42,7 +42,7 @@ on later `--new-only` runs. - Raleigh's site may present a Cloudflare browser challenge to non-browser clients. The CLI does not attempt to bypass that challenge; the scheduled - canary records it as a visible, non-failing upstream access observation. + canary records it as a visible upstream availability failure. - The CLI preserves canonical page URLs so users can inspect the source presentation. - Rendered HTML is treated as content, not executable markup. - Publication status is requested server-side with `filter[status]=1`; the CLI diff --git a/raleigh/scripts/canary.py b/raleigh/scripts/canary.py index 5a86802..8edf4a0 100644 --- a/raleigh/scripts/canary.py +++ b/raleigh/scripts/canary.py @@ -13,7 +13,7 @@ the report. Exit codes: 0 all probes passed (or only empty-but-valid / restricted observations) - 1 one or more durable contract failures detected + 1 one or more durable contract or availability failures detected 2 script-level error (bad arguments, import failure, etc.) """ @@ -85,10 +85,10 @@ def _classify_exception(exc: Exception) -> str: return "parser_failure" -def _waf_observation(source: str, target: str, err: dict[str, Any] | None) -> dict[str, Any] | None: - """Keep a provider browser challenge visible without treating it as drift.""" +def _waf_failure(source: str, target: str, err: dict[str, Any] | None) -> dict[str, Any] | None: + """Keep provider browser challenges visible as availability failures.""" if err and err.get("failure_class") == "waf_challenge": - return {"source": source, "target": target, "status": "pass", **err} + return {"source": source, "target": target, "status": "fail", **err} return None @@ -295,9 +295,9 @@ def probe_civic_jsonapi() -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] index, err = _probe_with_retry(core.json_request, civic.JSONAPI_ROOT) if err: - observation = _waf_observation("civic", "jsonapi-index", err) - if observation: - return [observation] + failure = _waf_failure("civic", "jsonapi-index", err) + if failure: + return [failure] results.append({"source": "civic", "target": "jsonapi-index", "status": "fail", **err}) return results @@ -317,9 +317,9 @@ def probe_civic_rss() -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] data, err = _probe_with_retry(core.raw_request, civic.RSS_FEED) if err: - observation = _waf_observation("civic", "rss-feed", err) - if observation: - return [observation] + failure = _waf_failure("civic", "rss-feed", err) + if failure: + return [failure] results.append({"source": "civic", "target": "rss-feed", "status": "fail", **err}) return results @@ -524,13 +524,13 @@ def run_canary() -> dict[str, Any]: for r in all_results: if r.get("target") == "summary": continue + if r.get("failure_class") == "waf_challenge": + waf_challenges += 1 if r.get("status") != "fail": if r.get("failure_class") == "empty_but_valid": empty_valid += 1 elif r.get("failure_class") == "restricted_folder": restricted += 1 - elif r.get("failure_class") == "waf_challenge": - waf_challenges += 1 continue fc = r.get("failure_class", "unknown") if _is_transient(fc): @@ -588,7 +588,7 @@ def write_github_summary(report: dict[str, Any]) -> None: challenges = [r for r in report["results"] if r.get("failure_class") == "waf_challenge"] if challenges: - lines.append("### Upstream WAF challenges (blocked machine access; non-failing)") + lines.append("### Upstream WAF challenges (blocked machine access; failing)") lines.append("") lines.append("| Source | Target | Evidence |") lines.append("|--------|--------|----------|") diff --git a/raleigh/tests/test_raleigh.py b/raleigh/tests/test_raleigh.py index 48d837b..4ab8e6a 100644 --- a/raleigh/tests/test_raleigh.py +++ b/raleigh/tests/test_raleigh.py @@ -2726,7 +2726,7 @@ class PoliceTests(unittest.TestCase): self.assertFalse(report["passed"]) self.assertEqual(report["summary"]["transient_failures"], 1) - def test_canary_classifies_cloudflare_challenge_as_visible_non_failing_observation(self): + def test_canary_classifies_cloudflare_challenge_as_visible_availability_failure(self): headers = Message() headers["Server"] = "cloudflare" headers["cf-mitigated"] = "challenge" @@ -2736,10 +2736,50 @@ class PoliceTests(unittest.TestCase): with patch("canary.core.json_request", side_effect=error): results = canary_lib.probe_civic_jsonapi() self.assertEqual(len(results), 1) - self.assertEqual(results[0]["status"], "pass") + self.assertEqual(results[0]["status"], "fail") self.assertEqual(results[0]["failure_class"], "waf_challenge") self.assertIn("Cloudflare", results[0]["error"]) + def test_canary_classifies_rss_cloudflare_challenge_as_availability_failure(self): + headers = Message() + headers["cf-mitigated"] = "challenge" + error = urllib.error.HTTPError( + "https://raleighnc.gov/rss.xml", 403, "Forbidden", headers, None + ) + with patch("canary.core.raw_request", side_effect=error): + results = canary_lib.probe_civic_rss() + self.assertEqual(results[0]["status"], "fail") + self.assertEqual(results[0]["failure_class"], "waf_challenge") + + def test_canary_summary_renders_waf_challenges_as_failures(self): + report = { + "passed": False, + "summary": { + "total_results": 1, + "durable_failures": 1, + "transient_failures": 0, + "empty_but_valid": 0, + "restricted_folders": 0, + "waf_challenges": 1, + }, + "results": [{ + "source": "civic", + "target": "jsonapi-index", + "status": "fail", + "failure_class": "waf_challenge", + "error": "Cloudflare managed challenge", + }], + } + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as summary: + summary_path = summary.name + self.addCleanup(os.unlink, summary_path) + with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": summary_path}): + canary_lib.write_github_summary(report) + contents = pathlib.Path(summary_path).read_text() + self.assertIn("WAF challenges | 1", contents) + self.assertIn("blocked machine access; failing", contents) + self.assertIn("| civic | jsonapi-index | waf_challenge |", contents) + def test_canary_keeps_plain_forbidden_as_auth_failure(self): headers = Message() error = urllib.error.HTTPError( @@ -2750,20 +2790,20 @@ class PoliceTests(unittest.TestCase): self.assertEqual(results[0]["status"], "fail") self.assertEqual(results[0]["failure_class"], "auth_regression") - def test_canary_summary_counts_waf_challenges_without_failing(self): + def test_canary_summary_counts_waf_challenges_as_failures(self): observations = [{ "source": "civic", "target": "jsonapi-index", - "status": "pass", + "status": "fail", "failure_class": "waf_challenge", "error": "Cloudflare managed challenge", "attempt": 1, }] with patch.object(canary_lib, "ALL_PROBES", [("civic", lambda: observations)]): report = canary_lib.run_canary() - self.assertTrue(report["passed"]) + self.assertFalse(report["passed"]) self.assertEqual(report["summary"]["waf_challenges"], 1) - self.assertEqual(report["summary"]["durable_failures"], 0) + self.assertEqual(report["summary"]["durable_failures"], 1) def test_canary_imagery_probe_reports_restricted_folders_as_non_failing(self): with patch("canary.imagery.list_services", return_value=(