fix(raleigh): keep WAF outages blocking (#424)

Keep Cloudflare WAF challenges distinct while preserving a blocking canary signal for unavailable civic adapters.

raleigh/EVIDENCE-LEDGER.md

@@ -210,16 +210,16 @@ The following public service boundaries were exercised successfully on 2026-07-2
   local request, establishing an access-policy boundary rather than a Raleigh
   schema failure.
 - Classify this exact marker as `waf_challenge`, preserve source and target in
-  the report, and keep the observation non-failing. Other 403 responses remain
+  the report, and keep it as a blocking availability failure. Other 403
-  `auth_regression` and continue to fail the canary.
+  responses remain `auth_regression` and continue to fail the canary.
 
 ### Verification target and follow-up
 
 - Deterministic tests cover the marker-specific classification and the
-  non-failing summary accounting.
+  blocking summary accounting.
 - The scheduled workflow remains the delivery-boundary check. A later green
-  run proves the canary no longer treats this known upstream challenge as a
+  run requires Raleigh machine access to be restored; a challenged civic
-  durable failure; it does not prove Raleigh machine access has been restored.
+  endpoint remains a canary failure.
 - Do not add retries, browser automation, or challenge bypasses. Reclassify only
   when the provider removes the marker or a new upstream access contract is
   verified.

raleigh/references/civic-content-reference.md

@@ -42,7 +42,7 @@ on later `--new-only` runs.
 
 - Raleigh's site may present a Cloudflare browser challenge to non-browser
   clients. The CLI does not attempt to bypass that challenge; the scheduled
-  canary records it as a visible, non-failing upstream access observation.
+  canary records it as a visible upstream availability failure.
 - The CLI preserves canonical page URLs so users can inspect the source presentation.
 - Rendered HTML is treated as content, not executable markup.
 - Publication status is requested server-side with `filter[status]=1`; the CLI

raleigh/scripts/canary.py

@@ -13,7 +13,7 @@ the report.
 
 Exit codes:
   0  all probes passed (or only empty-but-valid / restricted observations)
-  1  one or more durable contract failures detected
+  1  one or more durable contract or availability failures detected
   2  script-level error (bad arguments, import failure, etc.)
 """
 
@@ -85,10 +85,10 @@ def _classify_exception(exc: Exception) -> str:
     return "parser_failure"
 
 
-def _waf_observation(source: str, target: str, err: dict[str, Any] | None) -> dict[str, Any] | None:
+def _waf_failure(source: str, target: str, err: dict[str, Any] | None) -> dict[str, Any] | None:
-    """Keep a provider browser challenge visible without treating it as drift."""
+    """Keep provider browser challenges visible as availability failures."""
     if err and err.get("failure_class") == "waf_challenge":
-        return {"source": source, "target": target, "status": "pass", **err}
+        return {"source": source, "target": target, "status": "fail", **err}
     return None
 
 
@@ -295,9 +295,9 @@ def probe_civic_jsonapi() -> list[dict[str, Any]]:
     results: list[dict[str, Any]] = []
     index, err = _probe_with_retry(core.json_request, civic.JSONAPI_ROOT)
     if err:
-        observation = _waf_observation("civic", "jsonapi-index", err)
+        failure = _waf_failure("civic", "jsonapi-index", err)
-        if observation:
+        if failure:
-            return [observation]
+            return [failure]
         results.append({"source": "civic", "target": "jsonapi-index", "status": "fail", **err})
         return results
 
@@ -317,9 +317,9 @@ def probe_civic_rss() -> list[dict[str, Any]]:
     results: list[dict[str, Any]] = []
     data, err = _probe_with_retry(core.raw_request, civic.RSS_FEED)
     if err:
-        observation = _waf_observation("civic", "rss-feed", err)
+        failure = _waf_failure("civic", "rss-feed", err)
-        if observation:
+        if failure:
-            return [observation]
+            return [failure]
         results.append({"source": "civic", "target": "rss-feed", "status": "fail", **err})
         return results
 
@@ -524,13 +524,13 @@ def run_canary() -> dict[str, Any]:
     for r in all_results:
         if r.get("target") == "summary":
             continue
+        if r.get("failure_class") == "waf_challenge":
+            waf_challenges += 1
         if r.get("status") != "fail":
             if r.get("failure_class") == "empty_but_valid":
                 empty_valid += 1
             elif r.get("failure_class") == "restricted_folder":
                 restricted += 1
-            elif r.get("failure_class") == "waf_challenge":
-                waf_challenges += 1
             continue
         fc = r.get("failure_class", "unknown")
         if _is_transient(fc):
@@ -588,7 +588,7 @@ def write_github_summary(report: dict[str, Any]) -> None:
 
     challenges = [r for r in report["results"] if r.get("failure_class") == "waf_challenge"]
     if challenges:
-        lines.append("### Upstream WAF challenges (blocked machine access; non-failing)")
+        lines.append("### Upstream WAF challenges (blocked machine access; failing)")
         lines.append("")
         lines.append("| Source | Target | Evidence |")
         lines.append("|--------|--------|----------|")

raleigh/tests/test_raleigh.py

@@ -2726,7 +2726,7 @@ class PoliceTests(unittest.TestCase):
         self.assertFalse(report["passed"])
         self.assertEqual(report["summary"]["transient_failures"], 1)
 
-    def test_canary_classifies_cloudflare_challenge_as_visible_non_failing_observation(self):
+    def test_canary_classifies_cloudflare_challenge_as_visible_availability_failure(self):
         headers = Message()
         headers["Server"] = "cloudflare"
         headers["cf-mitigated"] = "challenge"
@@ -2736,10 +2736,50 @@ class PoliceTests(unittest.TestCase):
         with patch("canary.core.json_request", side_effect=error):
             results = canary_lib.probe_civic_jsonapi()
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0]["status"], "pass")
+        self.assertEqual(results[0]["status"], "fail")
         self.assertEqual(results[0]["failure_class"], "waf_challenge")
         self.assertIn("Cloudflare", results[0]["error"])
 
+    def test_canary_classifies_rss_cloudflare_challenge_as_availability_failure(self):
+        headers = Message()
+        headers["cf-mitigated"] = "challenge"
+        error = urllib.error.HTTPError(
+            "https://raleighnc.gov/rss.xml", 403, "Forbidden", headers, None
+        )
+        with patch("canary.core.raw_request", side_effect=error):
+            results = canary_lib.probe_civic_rss()
+        self.assertEqual(results[0]["status"], "fail")
+        self.assertEqual(results[0]["failure_class"], "waf_challenge")
+
+    def test_canary_summary_renders_waf_challenges_as_failures(self):
+        report = {
+            "passed": False,
+            "summary": {
+                "total_results": 1,
+                "durable_failures": 1,
+                "transient_failures": 0,
+                "empty_but_valid": 0,
+                "restricted_folders": 0,
+                "waf_challenges": 1,
+            },
+            "results": [{
+                "source": "civic",
+                "target": "jsonapi-index",
+                "status": "fail",
+                "failure_class": "waf_challenge",
+                "error": "Cloudflare managed challenge",
+            }],
+        }
+        with tempfile.NamedTemporaryFile(mode="w+", delete=False) as summary:
+            summary_path = summary.name
+        self.addCleanup(os.unlink, summary_path)
+        with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": summary_path}):
+            canary_lib.write_github_summary(report)
+        contents = pathlib.Path(summary_path).read_text()
+        self.assertIn("WAF challenges | 1", contents)
+        self.assertIn("blocked machine access; failing", contents)
+        self.assertIn("| civic | jsonapi-index | waf_challenge |", contents)
+
     def test_canary_keeps_plain_forbidden_as_auth_failure(self):
         headers = Message()
         error = urllib.error.HTTPError(
@@ -2750,20 +2790,20 @@ class PoliceTests(unittest.TestCase):
         self.assertEqual(results[0]["status"], "fail")
         self.assertEqual(results[0]["failure_class"], "auth_regression")
 
-    def test_canary_summary_counts_waf_challenges_without_failing(self):
+    def test_canary_summary_counts_waf_challenges_as_failures(self):
         observations = [{
             "source": "civic",
             "target": "jsonapi-index",
-            "status": "pass",
+            "status": "fail",
             "failure_class": "waf_challenge",
             "error": "Cloudflare managed challenge",
             "attempt": 1,
         }]
         with patch.object(canary_lib, "ALL_PROBES", [("civic", lambda: observations)]):
             report = canary_lib.run_canary()
-        self.assertTrue(report["passed"])
+        self.assertFalse(report["passed"])
         self.assertEqual(report["summary"]["waf_challenges"], 1)
-        self.assertEqual(report["summary"]["durable_failures"], 0)
+        self.assertEqual(report["summary"]["durable_failures"], 1)
 
     def test_canary_imagery_probe_reports_restricted_folders_as_non_failing(self):
         with patch("canary.imagery.list_services", return_value=(