magnus919_agent-skills/qa-methodology/scripts/check-ac-testability.py

#!/usr/bin/env python3
"""Acceptance-criteria testability checker for qa-methodology.

Scans a markdown spec or acceptance-criteria block for vague, untestable
language and flags criteria that lack an observable outcome.

Testable criteria have concrete, verifiable outcomes (status codes, return
values, observable side effects, numeric thresholds). Untestable criteria
use vague verbs ("should handle", "should be efficient", "should work
properly") with no observable verification path.

Input: a markdown file path argument, or stdin if no path given.

Exit codes:
  0  all detected acceptance criteria are testable
  1  one or more acceptance criteria are untestable
  2  malformed or missing input (bad path, empty input, usage error)
"""

import argparse
import re
import sys

# Patterns that indicate vague, untestable language.
VAGUE_PATTERNS = [
    re.compile(r"\bshould\s+handle\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+efficient\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+fast\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+robust\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+secure\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+reliable\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+scalable\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+user.friendly\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+intuitive\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+performant\b", re.IGNORECASE),
    re.compile(r"\bshould\s+work\s+(correctly|properly|well)\b", re.IGNORECASE),
    re.compile(r"\bshould\s+gracefully\b", re.IGNORECASE),
    re.compile(r"\bhandle\s+(errors?\s+)?gracefully\b", re.IGNORECASE),
    re.compile(r"\bshould\s+support\b(?!\s+\d)", re.IGNORECASE),
    re.compile(r"\bshould\s+appropriately\b", re.IGNORECASE),
    re.compile(r"\bas\s+(needed|required|appropriate)\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+easy\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+clean\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+maintainable\b", re.IGNORECASE),
    re.compile(r"\bshould\s+be\s+readable\b", re.IGNORECASE),
]

# Patterns that indicate a concrete, testable outcome.
TESTABLE_PATTERNS = [
    re.compile(r"\breturns?\s+\d{3}\b", re.IGNORECASE),
    re.compile(r"\breturns?\s+(true|false|null|nil|none)\b", re.IGNORECASE),
    re.compile(r"\breturns?\s+\{", re.IGNORECASE),
    re.compile(r"\breturns?\s+\[", re.IGNORECASE),
    re.compile(r"\breturns?\s+[\"']", re.IGNORECASE),
    re.compile(r"\bexit\s+code\s+\d", re.IGNORECASE),
    re.compile(r"\bexits?\s+(with\s+)?\d", re.IGNORECASE),
    re.compile(r"\bthrows?\s+\w*Error\b", re.IGNORECASE),
    re.compile(r"\braises?\s+\w*Error\b", re.IGNORECASE),
    re.compile(r"\bstatus\s+code\s+\d{3}\b", re.IGNORECASE),
    re.compile(r"\bHTTP\s+\d{3}\b", re.IGNORECASE),
    re.compile(r"\b\d{3}\s+(OK|Created|Bad Request|Not Found|Error)\b", re.IGNORECASE),
    re.compile(r"\bwithin\s+\d+\s*(ms|s|seconds|milliseconds|minutes)\b", re.IGNORECASE),
    re.compile(r"\bless\s+than\s+\d", re.IGNORECASE),
    re.compile(r"\bat\s+most\s+\d", re.IGNORECASE),
    re.compile(r"\bat\s+least\s+\d", re.IGNORECASE),
    re.compile(r"\bexactly\s+\d", re.IGNORECASE),
    re.compile(r"\b\d+(\.\d+)?%", re.IGNORECASE),
    re.compile(r"\bdisplays?\b", re.IGNORECASE),
    re.compile(r"\brenders?\b", re.IGNORECASE),
    re.compile(r"\bnavigates?\s+to\b", re.IGNORECASE),
    re.compile(r"\bredir(e)?cts?\b", re.IGNORECASE),
    re.compile(r"\blog(s|ged|ging)?\s+(the|a|an|this)\b", re.IGNORECASE),
    re.compile(r"\bsends?\s+(an?\s+)?(email|notification|request|event)\b", re.IGNORECASE),
    re.compile(r"\bcreates?\s+(a|an|the)\b", re.IGNORECASE),
    re.compile(r"\bdeletes?\s+(a|an|the)\b", re.IGNORECASE),
    re.compile(r"\bupdates?\s+(a|an|the)\b", re.IGNORECASE),
    re.compile(r"\bstores?\s+(a|an|the|in)\b", re.IGNORECASE),
    re.compile(r"\bcontains?\b", re.IGNORECASE),
    re.compile(r"\bequals?\b", re.IGNORECASE),
    re.compile(r"\bmatches?\s+(the\s+)?(pattern|regex|schema)\b", re.IGNORECASE),
    re.compile(r"\bis\s+(empty|non-empty|present|absent)\b", re.IGNORECASE),
    re.compile(r"\bwith\s+\{[^}]+\}", re.IGNORECASE),
    re.compile(r"\bwhen\s+\w+", re.IGNORECASE),
]

# AC line patterns: markdown list items or Given/When/Then or numbered criteria.
AC_LINE_RE = re.compile(
    r"^\s*(?:[-*+]\s+|\d+[.)]\s+|AC[-_]?\d*[:.]\s*|Given\s|When\s|Then\s)",
    re.IGNORECASE,
)

# Also match lines that start with "should" or "shall" or "must" (common AC phrasing)
SHOULD_LINE_RE = re.compile(
    r"^\s*(?:[-*+]\s+)?(?:the\s+)?(?:system|api|app|application|service|server|user|it)\s+"
    r"(?:should|shall|must)\b",
    re.IGNORECASE,
)


def parse_args(argv=None):
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(
        prog="check-ac-testability.py",
        description=(
            "Check acceptance criteria in a markdown spec for testability. "
            "Flags vague criteria (unobservable verbs, no measurable outcome) "
            "and passes concrete ones (observable outcomes with verification path)."
        ),
        epilog=(
            "Exit codes: 0 all testable, 1 untestable criteria found, "
            "2 malformed/missing input."
        ),
    )
    parser.add_argument(
        "input",
        nargs="?",
        default="-",
        metavar="SPEC_FILE",
        help=(
            "Path to a markdown spec file containing acceptance criteria "
            "(default: stdin)."
        ),
    )
    return parser.parse_args(argv)


def extract_ac_lines(text):
    """Extract lines that look like acceptance criteria from markdown text."""
    ac_lines = []
    for line in text.splitlines():
        stripped = line.strip()
        if not stripped:
            continue
        if AC_LINE_RE.match(line) or SHOULD_LINE_RE.match(line):
            ac_lines.append(stripped)
    return ac_lines


def classify_ac(text):
    """Classify a single AC line as testable or untestable.

    Returns (verdict, reasons) where verdict is 'testable' or 'untestable'
    and reasons is a list of strings explaining why.
    """
    vague_hits = []
    for pat in VAGUE_PATTERNS:
        m = pat.search(text)
        if m:
            vague_hits.append(m.group(0))

    testable_hits = []
    for pat in TESTABLE_PATTERNS:
        m = pat.search(text)
        if m:
            testable_hits.append(m.group(0))

    # If there are concrete observable outcomes, it's testable
    # even if it also has a vague phrase (the concrete part dominates)
    if testable_hits:
        return "testable", ["has observable outcome: {}".format(testable_hits[0])]

    if vague_hits:
        reasons = ["vague language: {}".format(h) for h in vague_hits]
        reasons.append("no observable outcome specified")
        return "untestable", reasons

    # No vague pattern AND no testable pattern: could be a non-criterion line
    # If it has no "should/shall/must" verb, treat as not-an-AC
    if re.search(r"\b(should|shall|must)\b", text, re.IGNORECASE):
        # Has requirement language but no observable outcome
        return "untestable", ["requirement stated but no observable outcome or verification path"]

    # Not a requirement statement at all
    return "testable", ["no requirement language detected; treated as context"]


def check_testability(text):
    """Run testability check on the full text.

    Returns (results, exit_code) where results is a list of dicts.
    """
    ac_lines = extract_ac_lines(text)

    if not ac_lines:
        # No ACs found — nothing to flag
        return [], 0

    results = []
    has_untestable = False

    for line in ac_lines:
        verdict, reasons = classify_ac(line)
        results.append({
            "criterion": line,
            "verdict": verdict,
            "reasons": reasons,
        })
        if verdict == "untestable":
            has_untestable = True

    exit_code = 1 if has_untestable else 0
    return results, exit_code


def format_results(results):
    """Format results as human-readable output."""
    if not results:
        return "No acceptance criteria detected in input."

    lines = []
    for r in results:
        icon = "PASS" if r["verdict"] == "testable" else "FAIL"
        lines.append("[{}] {}".format(icon, r["criterion"]))
        for reason in r["reasons"]:
            lines.append("       {}".format(reason))
        lines.append("")
    return "\n".join(lines)


def main(argv=None):
    """Entry point."""
    args = parse_args(argv)

    # Read input
    try:
        if args.input == "-":
            source_text = sys.stdin.read()
        else:
            with open(args.input, "r", encoding="utf-8") as fh:
                source_text = fh.read()
    except OSError as exc:
        print("error: cannot read input: {}".format(exc), file=sys.stderr)
        return 2

    if not source_text.strip():
        print("error: input is empty", file=sys.stderr)
        return 2

    results, exit_code = check_testability(source_text)
    print(format_results(results))

    return exit_code


if __name__ == "__main__":
    sys.exit(main())