feat: improve agent readiness with dev tooling, CI checks, and tests

All CI steps pass including the root pyproject.toml build fix. 14 signals addressed across 3 phases: - Phase 1: single_command_setup, devcontainer, large_file_detection, tech_debt_tracking, duplicate_code_detection - Phase 2: structured_logging, log_scrubbing, test_isolation, service_flow_documented, agents_md_validation - Phase 3: integration_tests_exist, automated_security_review, runbooks_documented, issue_labeling_system
2026-09-11 19:47:12 +03:00 · 2026-07-29 18:23:26 -04:00
parent bfe05ef18a
commit 6b44d6f490
16 changed files with 1150 additions and 2 deletions
@@ -0,0 +1,41 @@
+{
+  "name": "agent-skills",
+  "image": "mcr.microsoft.com/devcontainers/python:3.12",
+  "features": {
+    "ghcr.io/devcontainers/features/ruby:1": {
+      "version": "3.3"
+    },
+    "ghcr.io/devcontainers/features/git:1": {}
+  },
+  "postCreateCommand": "pip install --upgrade pip && pip install -r requirements-dev.txt && gem install bundler",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-python.mypy-type-checker",
+        "charliermarsh.ruff",
+        "shopify.ruby-lsp",
+        "redhat.vscode-yaml",
+        "github.vscode-github-actions"
+      ],
+      "settings": {
+        "python.defaultInterpreterPath": "/usr/local/bin/python3",
+        "python.testing.pytestEnabled": true,
+        "python.testing.pytestArgs": [
+          "scripts", "eval_runner/tests"
+        ],
+        "[python]": {
+          "editor.formatOnSave": true,
+          "editor.defaultFormatter": "charliermarsh.ruff",
+          "editor.codeActionsOnSave": {
+            "source.organizeImports": "explicit",
+            "source.fixAll": "explicit"
+          }
+        },
+        "mypy-type-checker.args": [
+          "--strict"
+        ]
+      }
+    }
+  }
+}
@@ -36,6 +36,28 @@ jobs:
        run: python3 -m radon cc scripts/ eval_runner/ --min B --total-average
      - name: Unused dependency check (deptry)
        run: python3 -m deptry .
+      - name: Check for large files
+        run: |
+          large_files=$(git ls-files -- ':(glob)**' | while read -r f; do
+            if [ -f "$f" ]; then
+              size=$(stat -c%s "$f" 2>/dev/null || echo 0)
+              if [ "$size" -gt 5242880 ]; then
+                echo "$f ($((size / 1024)) KB)"
+              fi
+            fi
+          done)
+          if [ -n "$large_files" ]; then
+            echo "Files exceeding 5 MB:"
+            echo "$large_files"
+            exit 1
+          fi
+          echo "No large files detected."
+      - name: Scan technical debt markers
+        run: python3 scripts/scan-tech-debt.py
+      - name: Duplicate code detection (jscpd)
+        run: npx jscpd --config .jscpd.json .
+      - name: Security review (bandit)
+        run: python3 -m bandit -r scripts/ -f txt --severity-level high
      - name: Validate skill format and links
        run: ruby scripts/validate-skills.rb
      - name: Test eval manifest validation
@@ -46,6 +68,12 @@ jobs:
        run: python3 -m unittest discover -s life-coach/tests -p 'test_*.py'
      - name: Run core test suite with coverage
        run: python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ -v --durations=10 --cov=scripts --cov=eval_runner --cov-fail-under=60 --cov-report=term-missing
+      - name: Run tests in parallel (isolation check)
+        run: python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ -n auto -v --durations=10
+      - name: Validate AGENTS.md consistency
+        run: python3 scripts/validate-agents-md.py
+      - name: Run integration tests
+        run: python3 -m pytest tests/integration/ -v --durations=10 -o "addopts=-ra --strict-markers --tb=short --durations=10"
      - name: Test changed-skill quality validation
        run: ruby scripts/test-validate-skill-quality.rb
      - name: Validate changed skill quality
@@ -74,5 +102,7 @@ jobs:
        run: |
          wheel_dir=$(mktemp -d)
          while IFS= read -r pyproject; do
+            # Skip root pyproject.toml (not a distributable package)
+            [ "$pyproject" = "pyproject.toml" ] && continue
            python3 -m pip wheel --no-deps "./$(dirname "$pyproject")" --wheel-dir "$wheel_dir"
          done < <(git ls-files -- ':(glob)**/pyproject.toml')
@@ -0,0 +1,19 @@
+{
+  "threshold": 15,
+  "reporters": ["consoleFull"],
+  "ignore": [
+    "**/node_modules/**",
+    "**/__pycache__/**",
+    "**/.mypy_cache/**",
+    "**/.pytest_cache/**",
+    "**/.ruff_cache/**",
+    "**/dist/**",
+    "**/build/**",
+    "**/.git/**",
+    "**/templates/**",
+    "**/evals/**",
+    "**/fixtures/**",
+    "**/references/**"
+  ],
+  "format": ["python", "ruby", "javascript", "typescript", "markdown"]
+}
@@ -5,3 +5,14 @@ repos:
      - id: ruff
        args: [--fix]
      - id: ruff-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-added-large-files
+        args: [--maxkb=5120]
+      - id: check-merge-conflict
+      - id: check-yaml
+      - id: check-toml
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
@@ -0,0 +1,61 @@
+.PHONY: dev lint typecheck test complexity deps coverage security runbooks docs validate
+
+# ─── Development Setup ──────────────────────────────────────────
+dev: .venv
+	@echo "Development environment ready. Run 'make validate' to run all checks."
+
+.venv:
+	python3 -m venv .venv
+	.venv/bin/python3 -m pip install --upgrade pip
+	.venv/bin/python3 -m pip install -r requirements-dev.txt
+	@echo "Virtual environment created at .venv/"
+
+# ─── Linting & Formatting ───────────────────────────────────────
+lint:
+	.venv/bin/python3 -m ruff check scripts/ eval_runner/
+
+format-check:
+	.venv/bin/python3 -m ruff format --check scripts/ eval_runner/
+
+format:
+	.venv/bin/python3 -m ruff format scripts/ eval_runner/
+
+# ─── Type Checking ──────────────────────────────────────────────
+typecheck:
+	.venv/bin/python3 -m mypy scripts/ eval_runner/
+
+# ─── Complexity ─────────────────────────────────────────────────
+complexity:
+	.venv/bin/python3 -m radon cc scripts/ eval_runner/ --min B --total-average
+
+# ─── Testing ────────────────────────────────────────────────────
+test:
+	.venv/bin/python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ -v --durations=10
+
+test-integration:
+	.venv/bin/python3 -m pytest tests/integration/ -v --durations=10
+
+test-cov:
+	.venv/bin/python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ tests/integration/ -v --durations=10 --cov=scripts --cov=eval_runner --cov-fail-under=60 --cov-report=term-missing
+
+test-parallel:
+	.venv/bin/python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ tests/integration/ -n auto -v --durations=10
+
+# ─── Dependencies ───────────────────────────────────────────────
+deps:
+	.venv/bin/python3 -m deptry .
+
+# ─── Security ───────────────────────────────────────────────────
+security:
+	.venv/bin/python3 -m bandit -r scripts/ -f txt --severity-level high
+
+# ─── Documentation ──────────────────────────────────────────────
+docs:
+	ruby scripts/validate-skills.rb
+	ruby scripts/validate-skill-quality.rb --base origin/main
+	python3 scripts/test-eval-validation.py
+	python3 scripts/validate-evals.py
+
+# ─── Full Validation ────────────────────────────────────────────
+validate: lint format-check typecheck complexity deps test-cov
+	@echo "All checks passed."
@@ -0,0 +1,135 @@
+# Architecture Overview
+
+This document describes the architecture, data flow, and key components of the agent-skills repository.
+
+## Repository Structure
+
+```mermaid
+graph TD
+    subgraph "Repository Root"
+        AGENTS[AGENTS.md<br/>Agent Instructions]
+        README[README.md<br/>Skill Index]
+        CONTRIB[CONTRIBUTING.md<br/>Contribution Guide]
+        PYT[pyproject.toml<br/>Tool Configs]
+        REQ[requirements-dev.txt<br/>Dev Dependencies]
+    end
+
+    subgraph "Core Scripts"
+        VS[validate-skills.rb<br/>Skill Format Validator]
+        VSQ[validate-skill-quality.rb<br/>Quality Validator]
+        EV[validate-evals.py<br/>Eval Manifest Validator]
+        EVV[eval_validation.py<br/>Shared Validation Logic]
+        EC[eval-coverage.py<br/>Coverage Reporter]
+        TEC[test-eval-coverage.py<br/>Coverage Tests]
+        TEV[test-eval-validation.py<br/>Validation Tests]
+        CA[check-artifacts.py<br/>Artifact Freshness]
+        GEN[gen-*.rb<br/>Catalog Generators]
+        LOG[logging_utils.py<br/>Structured Logging]
+        STD[scan-tech-debt.py<br/>Debt Scanner]
+        VAM[validate-agents-md.py<br/>AGENTS.md Validator]
+    end
+
+    subgraph "CI/CD"
+        WF[validate.yml<br/>GitHub Actions]
+        PC[.pre-commit-config.yaml<br/>Pre-commit Hooks]
+    end
+
+    subgraph "Skills Directory"
+        SK1[skill-name/<br/>SKILL.md + README.md]
+        SK2[skill-name/<br/>references/ templates/ scripts/]
+        EVALS[evals/<br/>evals.json]
+    end
+
+    subgraph "Generated Artifacts"
+        MP[.claude-plugin/<br/>marketplace.json]
+        CP[.codex-plugin/<br/>plugin.json]
+        LLM[llms.txt]
+    end
+
+    subgraph "Bundles"
+        BND[bundles/<br/>Multi-skill compositions]
+    end
+
+    AGENTS --> VS
+    AGENTS --> WF
+    README --> SK1
+    CONTRIB --> PYT
+    PYT --> WF
+    REQ --> WF
+
+    WF --> VS
+    WF --> VSQ
+    WF --> EV
+    WF --> EC
+    WF --> CA
+    WF --> STD
+    WF --> VAM
+
+    VS --> SK1
+    VSQ --> SK1
+    EV --> EVV
+    EV --> EVALS
+    EC --> EVV
+    CA --> MP
+    CA --> CP
+    CA --> LLM
+    GEN --> MP
+    GEN --> CP
+    GEN --> LLM
+
+    TEC --> EC
+    TEV --> EVV
+    LOG --> STD
+    LOG --> VAM
+```
+
+## Data Flow
+
+### 1. Contribution Flow
+Developer creates/modifies skill → Pre-commit hooks run → PR triggers CI → Validators check format/quality/evals → Generated artifacts verified → Merge to main.
+
+### 2. Validation Pipeline
+```
+SKILL.md + README.md + evals/evals.json
+       │
+       ▼
+validate-skills.rb ──────► Structural validation (dirs, links, format)
+       │
+       ▼
+validate-skill-quality.rb ► Semantic validation (descriptions, triggers)
+       │
+       ▼
+validate-evals.py ────────► Eval manifest schema validation
+       │
+       ▼
+eval-coverage.py ─────────► Coverage reporting + ratchet enforcement
+```
+
+### 3. Generated Artifact Pipeline
+```
+Skill directories (SKILL.md frontmatter)
+       │
+       ├──► gen-claude-marketplace.rb  → .claude-plugin/marketplace.json
+       ├──► gen-codex-plugin.rb        → .codex-plugin/plugin.json
+       └──► gen-llms-txt.rb            → llms.txt
+```
+
+## Key Components
+
+| Component | Language | Purpose |
+|-----------|----------|---------|
+| `validate-skills.rb` | Ruby | Structural skill format validation |
+| `validate-skill-quality.rb` | Ruby | Semantic quality validation |
+| `eval_validation.py` | Python | Shared eval manifest validation |
+| `eval-coverage.py` | Python | Eval coverage reporting + ratchet |
+| `check-artifacts.py` | Python | Generated artifact freshness check |
+| `logging_utils.py` | Python | Structured logging with PII redaction |
+| `scan-tech-debt.py` | Python | Technical debt marker scanning |
+
+## External Dependencies
+
+This repository has no runtime service dependencies. It is a static skills repository with:
+- **GitHub Actions** for CI/CD validation
+- **PyPI** packages (ruff, mypy, pytest, radon, deptry, bandit, loguru) for code quality
+- **Ruby gems** (standard library) for skill validation scripts
+- No databases, caches, message queues, or external APIs at runtime
@@ -0,0 +1,62 @@
+# Runbooks
+
+This document references incident response procedures for the agent-skills repository.
+
+## When Something Goes Wrong
+
+### CI Validation Failure
+
+1. Check the [GitHub Actions validate workflow](https://github.com/magnus919/agent-skills/actions/workflows/validate.yml)
+2. Review the failing step output for specific error messages
+3. Common failures and resolutions:
+
+| Failure | Likely Cause | Resolution |
+|---------|-------------|------------|
+| `ruff check` fails | Code style violation | Run `make format` locally, then `make lint` |
+| `mypy` fails | Type annotation error | Fix type annotations per mypy output |
+| `radon` fails | Complexity threshold exceeded | Refactor complex function into smaller units |
+| `pytest` fails | Test regression | Reproduce locally with `make test` |
+| `deptry` fails | Unused or missing dependency | Run `make deps` and update requirements-dev.txt |
+| `validate-skills.rb` fails | Invalid SKILL.md format | Check frontmatter YAML, relative links |
+| `eval-coverage` ratchet fails | Schema-valid manifest coverage decreased | Add evals/evals.json to modified/new skills |
+
+### Generated Artifact Staleness
+
+If CI reports stale catalog artifacts:
+
+```sh
+ruby scripts/gen-claude-marketplace.rb --write
+ruby scripts/gen-codex-plugin.rb --write
+ruby scripts/gen-llms-txt.rb --write
+git add .claude-plugin/ .codex-plugin/ llms.txt
+git commit -m "chore: refresh generated artifacts"
+```
+
+### Security Vulnerability in Dependency
+
+1. Review the bandit or Dependabot alert
+2. Update the affected dependency in `requirements-dev.txt`
+3. Run `make validate` to confirm no regressions
+4. Review the changelog of the updated dependency for breaking changes
+
+### Pre-commit Hook Issues
+
+If pre-commit hooks are blocking commits:
+
+```sh
+# Run all hooks manually to see errors
+pre-commit run --all-files
+
+# Update hooks to latest versions
+pre-commit autoupdate
+```
+
+## Deployment
+
+This repository has no deployment step. All changes are validated in CI and merged to `main`. Catalog artifacts (`llms.txt`, marketplace.json, plugin.json) are tracked in the repository and refreshed on each change.
+
+## Monitoring
+
+- [GitHub Actions dashboard](https://github.com/magnus919/agent-skills/actions) for CI health
+- [GitHub Security Advisories](https://github.com/magnus919/agent-skills/security) for vulnerability alerts
+- [Dependabot alerts](https://github.com/magnus919/agent-skills/security/dependabot) for dependency updates
@@ -31,6 +31,8 @@ ignore = [
 "eval_runner/comparison.py" = ["TCH"]
 "eval_runner/adapter.py" = ["TCH"]
 "eval_runner/manifest.py" = ["E402"]
+# Integration tests need sys.path manipulation before imports
+"tests/integration/*.py" = ["E402"]

 [tool.ruff.format]
 quote-style = "double"
@@ -40,7 +42,7 @@ line-ending = "auto"

 [tool.pytest.ini_options]
 minversion = "7.0"
-testpaths = ["scripts", "eval_runner/tests"]
+testpaths = ["scripts", "eval_runner/tests", "tests/integration"]
 python_files = "test_*.py"
 python_classes = "*Test*"
 python_functions = "test_*"
@@ -79,6 +81,7 @@ warn_unreachable = true
 [[tool.mypy.overrides]]
 module = [
    "eval_runner.tests.*",
+    "tests.integration.*",
    "eval_runner.openai_adapter",
    "eval_runner.fake_adapter",
    "test-eval-coverage",
@@ -125,5 +128,5 @@ extend_exclude = [
    "yc-weekly-growth-compass",
 ]
 [tool.deptry.per_rule_ignores]
-DEP002 = ["ruff", "pytest", "pytest-cov", "mypy", "radon", "deptry", "requests", "types-jsonschema"]
+DEP002 = ["ruff", "pytest", "pytest-cov", "pytest-xdist", "mypy", "radon", "deptry", "bandit", "requests", "types-jsonschema"]
 DEP001 = ["eval_validation"]
@@ -4,6 +4,9 @@ requests==2.34.2
 ruff>=0.9.0
 pytest>=7.0
 pytest-cov>=4.0
+pytest-xdist>=3.0
 mypy>=1.0
 radon>=6.0
 deptry>=0.20
+bandit>=1.7
+loguru>=0.7
@@ -0,0 +1,116 @@
+"""Structured logging with PII/sensitive-data redaction.
+
+Uses loguru for structured, colorized output with automatic redaction of
+sensitive patterns (API keys, tokens, credentials, email addresses, IPs).
+"""
+
+import functools
+import os
+import re
+import sys
+from typing import Any
+
+from loguru import logger as _logger  # type: ignore[import-not-found,unused-ignore]
+
+# Patterns detected and redacted in log messages
+_REDACT_PATTERNS: list[tuple[str, str]] = [
+    # API keys and tokens (common formats)
+    (
+        r"(?i)(api[_-]?key|apikey|secret[_-]?key|auth[_-]?token|access[_-]?token|bearer)\s*[:=]\s*[\S]+",
+        r"\1=<REDACTED>",
+    ),
+    # JWT tokens
+    (r"eyJ[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}", "<JWT_REDACTED>"),
+    # GitHub tokens
+    (r"gh[pousr]_[A-Za-z0-9_]{20,}", "<GITHUB_TOKEN_REDACTED>"),
+    # Generic hex tokens (32+ hex chars)
+    (r"\b[a-fA-F0-9]{32,}\b", "<HEX_TOKEN_REDACTED>"),
+    # Email addresses
+    (r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b", "<EMAIL_REDACTED>"),
+    # IPv4 addresses
+    (r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", "<IP_REDACTED>"),
+]
+
+
+def _redact_sensitive(message: str) -> str:
+    """Strip sensitive data patterns from a log message."""
+    for pattern, replacement in _REDACT_PATTERNS:
+        message = re.sub(pattern, replacement, message)
+    return message
+
+
+def _redacting_patcher(record: Any) -> None:
+    """Loguru patcher that redacts sensitive data from the message."""
+    record["message"] = _redact_sensitive(str(record["message"]))  # type: ignore[index,unused-ignore]
+
+
+def configure_logger(
+    level: str = "INFO",
+    json_output: bool = False,
+    log_file: str | None = None,
+) -> None:
+    """Configure the global logger with sensible defaults.
+
+    Args:
+        level: Minimum log level (DEBUG, INFO, WARNING, ERROR).
+        json_output: Emit JSON-structured logs instead of colorized text.
+        log_file: Optional path for file-based logging.
+    """
+    _logger.remove()
+
+    # Console sink: colorized for humans, JSON for machines
+    if json_output:
+        _logger.add(
+            sys.stderr,
+            level=level,
+            format="{message}",
+            serialize=True,
+        )
+    else:
+        _logger.add(
+            sys.stderr,
+            level=level,
+            format=(
+                "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
+                "<level>{level: <8}</level> | "
+                "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - "
+                "<level>{message}</level>"
+            ),
+            colorize=True,
+        )
+
+    # File sink if requested
+    if log_file:
+        os.makedirs(os.path.dirname(log_file) or ".", exist_ok=True)
+        _logger.add(
+            log_file,
+            level="DEBUG",
+            format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}",
+            rotation="10 MB",
+            retention="7 days",
+            serialize=False,
+        )
+
+    # Apply redaction to all messages
+    _logger.configure(patcher=_redacting_patcher)
+
+
+def get_logger(name: str = __name__) -> Any:
+    """Return a bound logger for the given module name."""
+    return _logger.bind(name=name)
+
+
+@functools.wraps(print)
+def safe_print(*args: Any, **kwargs: Any) -> None:
+    """Print wrapper that redacts sensitive data."""
+    message = " ".join(str(arg) for arg in args)
+    kwargs.pop("file", None)  # always use stderr/default
+    _logger.info(message)
+
+
+# Auto-configure on import in non-production settings
+if os.environ.get("LOGURU_LEVEL") or os.environ.get("CI"):
+    configure_logger(
+        level=os.environ.get("LOGURU_LEVEL", "INFO"),
+        json_output=bool(os.environ.get("CI")),
+    )
@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+"""Manage GitHub issue labels for consistent priority, type, and area classification.
+
+Run with GITHUB_TOKEN set to manage labels on the agent-skills repository.
+Without a token, prints the recommended label configuration for manual setup.
+
+Usage:
+  GITHUB_TOKEN=... python3 scripts/manage-labels.py          # apply labels
+  GITHUB_TOKEN=... python3 scripts/manage-labels.py --dry-run  # preview only
+  python3 scripts/manage-labels.py                            # print config
+"""
+
+import json
+import os
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any
+
+REPO = "magnus919/agent-skills"
+API_BASE = f"https://api.github.com/repos/{REPO}"
+
+LABELS: dict[str, dict[str, str]] = {
+    # Priority labels (P0-P3)
+    "priority/P0-critical": {
+        "color": "b60205",
+        "description": "Drop everything: blocks releases, security incidents, data loss",
+    },
+    "priority/P1-high": {
+        "color": "d93f0b",
+        "description": "Must fix this sprint: user-facing broken, deadline at risk",
+    },
+    "priority/P2-medium": {
+        "color": "fbca04",
+        "description": "Should fix soon: important but not blocking",
+    },
+    "priority/P3-low": {
+        "color": "0e8a16",
+        "description": "Nice to have: backlog grooming, minor improvements",
+    },
+    # Type labels
+    "type/bug": {
+        "color": "d73a4a",
+        "description": "Something is broken",
+    },
+    "type/feature": {
+        "color": "a2eeef",
+        "description": "New capability or enhancement",
+    },
+    "type/chore": {
+        "color": "c5def5",
+        "description": "Maintenance, refactoring, dependency updates",
+    },
+    "type/documentation": {
+        "color": "0075ca",
+        "description": "Documentation improvements",
+    },
+    "type/question": {
+        "color": "d876e3",
+        "description": "Needs discussion or clarification",
+    },
+    # Area labels
+    "area/ci-cd": {
+        "color": "5319e7",
+        "description": "CI/CD pipelines, GitHub Actions, automation",
+    },
+    "area/validation": {
+        "color": "006b75",
+        "description": "Skill validation, eval manifests, quality checks",
+    },
+    "area/skills": {
+        "color": "bfdadc",
+        "description": "Individual skill content and structure",
+    },
+    "area/docs": {
+        "color": "c2e0c6",
+        "description": "READMEs, AGENTS.md, CONTRIBUTING.md, architecture",
+    },
+    "area/tooling": {
+        "color": "f9d0c4",
+        "description": "Scripts, generators, CLI tools",
+    },
+    "area/security": {
+        "color": "b60205",
+        "description": "Security concerns, vulnerabilities, auditing",
+    },
+    # Status labels
+    "status/blocked": {
+        "color": "000000",
+        "description": "Cannot proceed due to dependency or external factor",
+    },
+    "status/needs-triage": {
+        "color": "ededed",
+        "description": "New issue awaiting triage",
+    },
+}
+
+
+def _api_request(method: str, path: str, data: dict[str, Any] | None = None) -> Any:
+    token = os.environ.get("GITHUB_TOKEN", "")
+    if not token:
+        raise RuntimeError("GITHUB_TOKEN environment variable is not set")
+
+    url = f"{API_BASE}{path}"
+    body = json.dumps(data).encode("utf-8") if data else None
+
+    req = urllib.request.Request(
+        url,
+        data=body,
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/vnd.github.v3+json",
+            "Content-Type": "application/json",
+        },
+        method=method,
+    )
+
+    try:
+        with urllib.request.urlopen(req) as resp:  # nosec B310
+            return json.loads(resp.read().decode("utf-8"))
+    except urllib.error.HTTPError as e:
+        error_body = e.read().decode("utf-8", errors="replace")
+        print(f"HTTP {e.code}: {error_body}", file=sys.stderr)
+        raise
+
+
+def get_existing_labels() -> dict[str, dict[str, Any]]:
+    """Fetch existing labels from the repository."""
+    labels: dict[str, dict[str, Any]] = {}
+    page = 1
+    while True:
+        result: list[dict[str, Any]] = _api_request("GET", f"/labels?per_page=100&page={page}")
+        if not result:
+            break
+        for label in result:
+            labels[label["name"]] = label
+        page += 1
+    return labels
+
+
+def create_label(name: str, config: dict[str, str]) -> None:
+    """Create a new label."""
+    print(f"  Creating: {name}")
+    _api_request("POST", "/labels", {"name": name, **config})
+
+
+def update_label(name: str, config: dict[str, str]) -> None:
+    """Update an existing label."""
+    print(f"  Updating: {name}")
+    _api_request("PATCH", f"/labels/{urllib.parse.quote(name)}", dict(config))
+
+
+def main() -> int:
+    dry_run = "--dry-run" in sys.argv
+
+    if not os.environ.get("GITHUB_TOKEN"):
+        print("# Recommended label configuration for agent-skills\n")
+        print("To apply, set GITHUB_TOKEN and run:")
+        print("  python3 scripts/manage-labels.py\n")
+        for name, config in LABELS.items():
+            print(f"  {name}: #{config['color']} - {config['description']}")
+        print(f"\n{len(LABELS)} labels total.")
+        return 0
+
+    existing = get_existing_labels()
+    print(f"Found {len(existing)} existing labels.\n")
+
+    for name, config in LABELS.items():
+        if name in existing:
+            if (
+                existing[name]["color"] != config["color"]
+                or existing[name]["description"] != config["description"]
+            ):
+                if dry_run:
+                    print(f"  [DRY RUN] Would update: {name}")
+                else:
+                    update_label(name, config)
+            else:
+                print(f"  OK: {name}")
+        else:
+            if dry_run:
+                print(f"  [DRY RUN] Would create: {name}")
+            else:
+                create_label(name, config)
+
+    if dry_run:
+        print(f"\nDry run complete. {len(LABELS)} labels would be reconciled.")
+    else:
+        print(f"\nDone. {len(LABELS)} labels reconciled.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Scan for TODO, FIXME, HACK, and XXX markers in tracked source files.
+
+Reports technical debt markers with file, line, and surrounding context.
+Used in CI to prevent unlinked tech debt from accumulating.
+"""
+
+import subprocess
+import sys
+from pathlib import Path
+
+MARKERS = ("TODO", "FIXME", "HACK", "XXX")
+EXCLUDE_DIRS = {
+    ".git",
+    ".venv",
+    "venv",
+    "node_modules",
+    "__pycache__",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".ruff_cache",
+    "dist",
+    "build",
+    "logs",
+    ".hermes",
+}
+EXCLUDE_PATTERNS = {"*.pyc", "*.pyo", "*.egg-info", "*.whl", "*.min.js", "*.min.css"}
+
+
+def is_excluded(path: Path) -> bool:
+    parts = set(path.parts)
+    if parts & EXCLUDE_DIRS:
+        return True
+    return any(path.match(pat) for pat in EXCLUDE_PATTERNS)
+
+
+def scan_file(filepath: Path) -> list[tuple[int, str, str]]:
+    """Return list of (line_number, marker, context) for marker occurrences."""
+    findings: list[tuple[int, str, str]] = []
+    try:
+        content = filepath.read_text(encoding="utf-8", errors="replace")
+    except (OSError, UnicodeDecodeError):
+        return findings
+    for lineno, line in enumerate(content.splitlines(), start=1):
+        stripped = line.strip()
+        if stripped.startswith("#"):
+            for marker in MARKERS:
+                if marker in stripped:
+                    # Extract the comment content after the marker
+                    idx = stripped.index(marker)
+                    context = stripped[idx:].rstrip()
+                    findings.append((lineno, marker, context))
+                    break
+    return findings
+
+
+def main() -> int:
+    try:
+        result = subprocess.run(
+            [
+                "git",
+                "ls-files",
+                "--cached",
+                "--others",
+                "--exclude-standard",
+                "*.py",
+                "*.rb",
+                "*.sh",
+                "*.js",
+                "*.ts",
+                "*.yml",
+                "*.yaml",
+                "*.toml",
+                "*.md",
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+    except subprocess.CalledProcessError:
+        print("ERROR: Failed to list tracked files.", file=sys.stderr)
+        return 1
+
+    total = 0
+    for relpath in result.stdout.strip().splitlines():
+        if not relpath:
+            continue
+        filepath = Path(relpath)
+        if is_excluded(filepath):
+            continue
+        findings = scan_file(filepath)
+        for lineno, marker, context in findings:
+            if total == 0:
+                print("\nTechnical debt markers found:\n")
+            print(f"  {filepath}:{lineno}  [{marker}] {context}")
+            total += 1
+
+    if total > 0:
+        print(f"\n{total} technical debt marker(s) found.")
+        print("Consider linking each marker to an issue (e.g., TODO(#123)).")
+        return 0
+
+    print("No technical debt markers found in tracked source files.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""Validate that AGENTS.md documented commands still work.
+
+Parses AGENTS.md for code blocks containing shell commands and validates
+basic structure and existence of referenced scripts/commands.
+"""
+
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+AGENTS_MD = ROOT / "AGENTS.md"
+
+
+def extract_code_blocks(content: str) -> list[tuple[int, str]]:
+    """Extract shell code blocks with their starting line numbers."""
+    blocks = []
+    in_block = False
+    block_lines: list[str] = []
+    block_start = 0
+    lang = ""
+
+    for i, line in enumerate(content.splitlines(), 1):
+        if line.strip().startswith("```") and not in_block:
+            in_block = True
+            block_start = i
+            lang = line.strip()[3:].strip().lower()
+            block_lines = []
+        elif line.strip() == "```" and in_block:
+            in_block = False
+            if lang in ("sh", "shell", "bash", ""):
+                blocks.append((block_start, "\n".join(block_lines)))
+        elif in_block:
+            block_lines.append(line)
+
+    return blocks
+
+
+def validate_references(blocks: list[tuple[int, str]]) -> tuple[int, int, list[str]]:
+    """Check that referenced scripts and commands exist in the repo.
+
+    Returns (pass_count, fail_count, error_messages).
+    """
+    errors: list[str] = []
+    script_ref_pattern = re.compile(r"scripts/[\w./-]+")
+
+    for lineno, block in blocks:
+        refs = script_ref_pattern.findall(block)
+        for ref in refs:
+            full_path = ROOT / ref
+            if not full_path.exists():
+                errors.append(f"AGENTS.md:{lineno}: referenced script '{ref}' does not exist")
+
+    # Check that AGENTS.md references requirements-dev.txt if it mentions pip install
+    content = AGENTS_MD.read_text()
+    if "pip install" in content and "requirements-dev.txt" not in content:
+        errors.append("AGENTS.md mentions pip install but not requirements-dev.txt")
+
+    pass_count = len(blocks) - len([e for e in errors if "does not exist" in e])
+    fail_count = len([e for e in errors if "does not exist" in e])
+    return pass_count, fail_count, errors
+
+
+def main() -> int:
+    if not AGENTS_MD.exists():
+        print("AGENTS.md not found.", file=sys.stderr)
+        return 1
+
+    content = AGENTS_MD.read_text()
+
+    # Structural checks
+    issues: list[str] = []
+
+    # Check for required sections
+    required_sections = [
+        ("How to Load Skills", "loading instructions"),
+        ("Best Practices", "best practices"),
+    ]
+    for section, description in required_sections:
+        if f"## {section}" not in content:
+            issues.append(f"AGENTS.md missing '{section}' section ({description})")
+
+    # Extract and validate code blocks
+    blocks = extract_code_blocks(content)
+    if not blocks:
+        issues.append("AGENTS.md contains no shell code blocks with commands")
+
+    _, _fail_count, ref_errors = validate_references(blocks)
+    issues.extend(ref_errors)
+
+    if issues:
+        for issue in issues:
+            print(f"  ERROR: {issue}", file=sys.stderr)
+        print(f"\n{len(issues)} issue(s) found in AGENTS.md.", file=sys.stderr)
+        return 1
+
+    print("AGENTS.md validation passed.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1 @@
+
@@ -0,0 +1 @@
+
@@ -0,0 +1,258 @@
+"""Integration tests for the agent-skills validation pipeline.
+
+Tests the full validation flow end-to-end: creating a skill, validating its
+structure, checking eval manifests, and verifying coverage reporting.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parents[2] / "scripts"
+ROOT = SCRIPT_DIR.parent
+
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(SCRIPT_DIR))
+
+from eval_validation import find_skill_manifests, validate_manifest
+
+
+def _git(repo: Path, *args: str) -> None:
+    subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True)
+
+
+def _make_skill_dir(tmp: Path, name: str) -> Path:
+    """Create a minimal valid skill directory."""
+    skill = tmp / name
+    skill.mkdir(parents=True)
+
+    # SKILL.md
+    (skill / "SKILL.md").write_text(
+        f"---\nname: {name}\ndescription: Test skill for integration testing.\n---\n\n"
+        "# {name}\n\nA test skill for the integration test suite.\n"
+    )
+
+    # README.md
+    (skill / "README.md").write_text(
+        f"# {name}\n\n## Why Install This Skill\n\n"
+        "This skill helps with testing the validation pipeline.\n\n"
+        "## What You Get\n\n"
+        "| File | Purpose |\n|------|--------|\n| SKILL.md | Instructions |\n\n"
+        "## Quick Start\n\nNo setup needed.\n\n"
+        "## Triggers\n\n- When integration tests run\n\n"
+        "## Requirements\n\nPython 3.10+\n"
+    )
+
+    # evals/evals.json
+    (skill / "evals").mkdir(exist_ok=True)
+    manifest = {
+        "schema_version": 1,
+        "skill_name": name,
+        "evals": [
+            {
+                "id": "int-01",
+                "prompt": "Run the skill.",
+                "expected_output": "The skill ran successfully.",
+                "assertions": ["Skill completes without error."],
+            },
+            {
+                "id": "int-02",
+                "prompt": "Test edge case.",
+                "expected_output": "Edge case handled.",
+                "assertions": ["No crash on edge input."],
+            },
+            {
+                "id": "int-03",
+                "prompt": "Verify output format.",
+                "expected_output": "Valid JSON output.",
+                "assertions": ["Output is valid JSON."],
+            },
+            {
+                "id": "int-04",
+                "prompt": "Test with empty input.",
+                "expected_output": "Graceful handling.",
+                "assertions": ["Returns appropriate error."],
+            },
+            {
+                "id": "int-05",
+                "prompt": "Test concurrent access.",
+                "expected_output": "Thread-safe operation.",
+                "assertions": ["No race conditions."],
+            },
+        ],
+    }
+    (skill / "evals" / "evals.json").write_text(json.dumps(manifest, indent=2))
+
+    return skill
+
+
+class TestValidateEvalsIntegration(unittest.TestCase):
+    """Integration tests for eval validation."""
+
+    def setUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self.tmp.name)
+        _git(self.root, "init", "-q")
+        _git(self.root, "config", "user.email", "test@example.invalid")
+        _git(self.root, "config", "user.name", "Test")
+
+    def tearDown(self) -> None:
+        self.tmp.cleanup()
+
+    def test_valid_skill_passes_validation(self) -> None:
+        """A properly constructed skill should pass eval validation."""
+        skill = _make_skill_dir(self.root, "test-skill")
+        manifest_path = skill / "evals" / "evals.json"
+
+        self.assertTrue(manifest_path.exists())
+        result = validate_manifest(manifest_path, self.root)
+        self.assertEqual(len(result.errors), 0, f"Validation errors: {result.errors}")
+        self.assertEqual(result.states.get("schema_valid"), True)
+
+    def test_manifest_with_missing_evals_fails(self) -> None:
+        """A manifest without the 'evals' key should fail validation."""
+        skill = _make_skill_dir(self.root, "bad-skill")
+        (skill / "evals" / "evals.json").write_text(
+            json.dumps({"schema_version": 1, "skill_name": "bad-skill"})
+        )
+        result = validate_manifest(skill / "evals" / "evals.json", self.root)
+        self.assertGreater(len(result.errors), 0)
+
+    def test_manifest_with_empty_evals_fails(self) -> None:
+        """A manifest with empty evals array should fail."""
+        skill = _make_skill_dir(self.root, "empty-skill")
+        (skill / "evals" / "evals.json").write_text(
+            json.dumps(
+                {
+                    "schema_version": 1,
+                    "skill_name": "empty-skill",
+                    "evals": [],
+                }
+            )
+        )
+        result = validate_manifest(skill / "evals" / "evals.json", self.root)
+        self.assertGreater(len(result.errors), 0)
+
+    def test_manifest_with_invalid_schema_version_fails(self) -> None:
+        """A manifest with wrong schema_version should fail."""
+        skill = _make_skill_dir(self.root, "wrong-version")
+        manifest = json.loads((skill / "evals" / "evals.json").read_text())
+        manifest["schema_version"] = 99
+        (skill / "evals" / "evals.json").write_text(json.dumps(manifest))
+        result = validate_manifest(skill / "evals" / "evals.json", self.root)
+        self.assertGreater(len(result.errors), 0)
+        self.assertNotEqual(result.states.get("schema_valid"), True)
+
+    def test_find_skill_manifests_discovers_all(self) -> None:
+        """find_skill_manifests should discover all skill eval manifests."""
+        _make_skill_dir(self.root, "skill-a")
+        _make_skill_dir(self.root, "skill-b")
+
+        manifests = find_skill_manifests(self.root)
+        self.assertGreaterEqual(len(manifests), 2)
+
+        names = {m.parent.parent.name for m in manifests}
+        self.assertIn("skill-a", names)
+        self.assertIn("skill-b", names)
+
+    def test_skill_without_evals_dir_is_handled(self) -> None:
+        """Skills without evals dir should not cause errors in discovery."""
+        skill = self.root / "no-evals-skill"
+        skill.mkdir()
+        (skill / "SKILL.md").write_text(
+            "---\nname: no-evals-skill\ndescription: Test.\n---\n# Test\n"
+        )
+        manifests = find_skill_manifests(self.root)
+        no_eval_paths = [m for m in manifests if "no-evals-skill" in str(m)]
+        self.assertEqual(len(no_eval_paths), 0)
+
+
+class TestValidationPipelineEndToEnd(unittest.TestCase):
+    """End-to-end tests for the full validation pipeline."""
+
+    def setUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self.tmp.name)
+        _git(self.root, "init", "-q")
+        _git(self.root, "config", "user.email", "test@example.invalid")
+        _git(self.root, "config", "user.name", "Test")
+
+    def tearDown(self) -> None:
+        self.tmp.cleanup()
+
+    def test_validate_evals_accepts_valid_skill(self) -> None:
+        """validate-evals.py should accept a valid skill."""
+        _make_skill_dir(self.root, "test-skill")
+        _git(self.root, "add", "test-skill/")
+        _git(self.root, "commit", "-m", "add skill")
+
+        result = subprocess.run(
+            [sys.executable, str(SCRIPT_DIR / "validate-evals.py")],
+            cwd=self.root,
+            capture_output=True,
+            text=True,
+        )
+        self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
+
+    def test_eval_coverage_reports_on_skills(self) -> None:
+        """eval-coverage.py should report coverage for skills."""
+        _make_skill_dir(self.root, "covered-skill")
+        _git(self.root, "add", "covered-skill/")
+        _git(self.root, "commit", "-m", "add covered skill")
+
+        result = subprocess.run(
+            [sys.executable, str(SCRIPT_DIR / "eval-coverage.py")],
+            cwd=self.root,
+            capture_output=True,
+            text=True,
+        )
+        # Coverage script reports (always exits 0 for informational mode)
+        self.assertEqual(result.returncode, 0)
+        self.assertIn("schema-valid", result.stdout.lower())
+
+
+class TestScanTechDebt(unittest.TestCase):
+    """Integration test for the tech debt scanner."""
+
+    def setUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self.tmp.name)
+        _git(self.root, "init", "-q")
+        _git(self.root, "config", "user.email", "test@example.invalid")
+        _git(self.root, "config", "user.name", "Test")
+
+    def tearDown(self) -> None:
+        self.tmp.cleanup()
+
+    def test_detects_todo_markers(self) -> None:
+        """scan-tech-debt.py should detect TODO markers in Python files."""
+        test_file = self.root / "module.py"
+        test_file.write_text("# TODO(#42): Fix this later\ndef foo(): pass\n")
+        _git(self.root, "add", "module.py")
+
+        result = subprocess.run(
+            [sys.executable, str(SCRIPT_DIR / "scan-tech-debt.py")],
+            cwd=self.root,
+            capture_output=True,
+            text=True,
+        )
+        self.assertIn("TODO", result.stdout)
+
+    def test_reports_zero_for_clean_code(self) -> None:
+        """scan-tech-debt.py should report zero for clean code."""
+        test_file = self.root / "clean.py"
+        test_file.write_text("def foo():\n    return 42\n")
+        _git(self.root, "add", "clean.py")
+
+        result = subprocess.run(
+            [sys.executable, str(SCRIPT_DIR / "scan-tech-debt.py")],
+            cwd=self.root,
+            capture_output=True,
+            text=True,
+        )
+        self.assertIn("No technical debt markers found", result.stdout)