mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-11 19:47:12 +03:00
feat: improve agent readiness with dev tooling, CI checks, and tests
All CI steps pass including the root pyproject.toml build fix. 14 signals addressed across 3 phases: - Phase 1: single_command_setup, devcontainer, large_file_detection, tech_debt_tracking, duplicate_code_detection - Phase 2: structured_logging, log_scrubbing, test_isolation, service_flow_documented, agents_md_validation - Phase 3: integration_tests_exist, automated_security_review, runbooks_documented, issue_labeling_system
This commit is contained in:
Magnus Hedemark
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"name": "agent-skills",
|
||||||
|
"image": "mcr.microsoft.com/devcontainers/python:3.12",
|
||||||
|
"features": {
|
||||||
|
"ghcr.io/devcontainers/features/ruby:1": {
|
||||||
|
"version": "3.3"
|
||||||
|
},
|
||||||
|
"ghcr.io/devcontainers/features/git:1": {}
|
||||||
|
},
|
||||||
|
"postCreateCommand": "pip install --upgrade pip && pip install -r requirements-dev.txt && gem install bundler",
|
||||||
|
"customizations": {
|
||||||
|
"vscode": {
|
||||||
|
"extensions": [
|
||||||
|
"ms-python.python",
|
||||||
|
"ms-python.mypy-type-checker",
|
||||||
|
"charliermarsh.ruff",
|
||||||
|
"shopify.ruby-lsp",
|
||||||
|
"redhat.vscode-yaml",
|
||||||
|
"github.vscode-github-actions"
|
||||||
|
],
|
||||||
|
"settings": {
|
||||||
|
"python.defaultInterpreterPath": "/usr/local/bin/python3",
|
||||||
|
"python.testing.pytestEnabled": true,
|
||||||
|
"python.testing.pytestArgs": [
|
||||||
|
"scripts", "eval_runner/tests"
|
||||||
|
],
|
||||||
|
"[python]": {
|
||||||
|
"editor.formatOnSave": true,
|
||||||
|
"editor.defaultFormatter": "charliermarsh.ruff",
|
||||||
|
"editor.codeActionsOnSave": {
|
||||||
|
"source.organizeImports": "explicit",
|
||||||
|
"source.fixAll": "explicit"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"mypy-type-checker.args": [
|
||||||
|
"--strict"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -36,6 +36,28 @@ jobs:
|
|||||||
run: python3 -m radon cc scripts/ eval_runner/ --min B --total-average
|
run: python3 -m radon cc scripts/ eval_runner/ --min B --total-average
|
||||||
- name: Unused dependency check (deptry)
|
- name: Unused dependency check (deptry)
|
||||||
run: python3 -m deptry .
|
run: python3 -m deptry .
|
||||||
|
- name: Check for large files
|
||||||
|
run: |
|
||||||
|
large_files=$(git ls-files -- ':(glob)**' | while read -r f; do
|
||||||
|
if [ -f "$f" ]; then
|
||||||
|
size=$(stat -c%s "$f" 2>/dev/null || echo 0)
|
||||||
|
if [ "$size" -gt 5242880 ]; then
|
||||||
|
echo "$f ($((size / 1024)) KB)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done)
|
||||||
|
if [ -n "$large_files" ]; then
|
||||||
|
echo "Files exceeding 5 MB:"
|
||||||
|
echo "$large_files"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "No large files detected."
|
||||||
|
- name: Scan technical debt markers
|
||||||
|
run: python3 scripts/scan-tech-debt.py
|
||||||
|
- name: Duplicate code detection (jscpd)
|
||||||
|
run: npx jscpd --config .jscpd.json .
|
||||||
|
- name: Security review (bandit)
|
||||||
|
run: python3 -m bandit -r scripts/ -f txt --severity-level high
|
||||||
- name: Validate skill format and links
|
- name: Validate skill format and links
|
||||||
run: ruby scripts/validate-skills.rb
|
run: ruby scripts/validate-skills.rb
|
||||||
- name: Test eval manifest validation
|
- name: Test eval manifest validation
|
||||||
@@ -46,6 +68,12 @@ jobs:
|
|||||||
run: python3 -m unittest discover -s life-coach/tests -p 'test_*.py'
|
run: python3 -m unittest discover -s life-coach/tests -p 'test_*.py'
|
||||||
- name: Run core test suite with coverage
|
- name: Run core test suite with coverage
|
||||||
run: python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ -v --durations=10 --cov=scripts --cov=eval_runner --cov-fail-under=60 --cov-report=term-missing
|
run: python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ -v --durations=10 --cov=scripts --cov=eval_runner --cov-fail-under=60 --cov-report=term-missing
|
||||||
|
- name: Run tests in parallel (isolation check)
|
||||||
|
run: python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ -n auto -v --durations=10
|
||||||
|
- name: Validate AGENTS.md consistency
|
||||||
|
run: python3 scripts/validate-agents-md.py
|
||||||
|
- name: Run integration tests
|
||||||
|
run: python3 -m pytest tests/integration/ -v --durations=10 -o "addopts=-ra --strict-markers --tb=short --durations=10"
|
||||||
- name: Test changed-skill quality validation
|
- name: Test changed-skill quality validation
|
||||||
run: ruby scripts/test-validate-skill-quality.rb
|
run: ruby scripts/test-validate-skill-quality.rb
|
||||||
- name: Validate changed skill quality
|
- name: Validate changed skill quality
|
||||||
@@ -74,5 +102,7 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
wheel_dir=$(mktemp -d)
|
wheel_dir=$(mktemp -d)
|
||||||
while IFS= read -r pyproject; do
|
while IFS= read -r pyproject; do
|
||||||
|
# Skip root pyproject.toml (not a distributable package)
|
||||||
|
[ "$pyproject" = "pyproject.toml" ] && continue
|
||||||
python3 -m pip wheel --no-deps "./$(dirname "$pyproject")" --wheel-dir "$wheel_dir"
|
python3 -m pip wheel --no-deps "./$(dirname "$pyproject")" --wheel-dir "$wheel_dir"
|
||||||
done < <(git ls-files -- ':(glob)**/pyproject.toml')
|
done < <(git ls-files -- ':(glob)**/pyproject.toml')
|
||||||
|
|||||||
+19
@@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"threshold": 15,
|
||||||
|
"reporters": ["consoleFull"],
|
||||||
|
"ignore": [
|
||||||
|
"**/node_modules/**",
|
||||||
|
"**/__pycache__/**",
|
||||||
|
"**/.mypy_cache/**",
|
||||||
|
"**/.pytest_cache/**",
|
||||||
|
"**/.ruff_cache/**",
|
||||||
|
"**/dist/**",
|
||||||
|
"**/build/**",
|
||||||
|
"**/.git/**",
|
||||||
|
"**/templates/**",
|
||||||
|
"**/evals/**",
|
||||||
|
"**/fixtures/**",
|
||||||
|
"**/references/**"
|
||||||
|
],
|
||||||
|
"format": ["python", "ruby", "javascript", "typescript", "markdown"]
|
||||||
|
}
|
||||||
@@ -5,3 +5,14 @@ repos:
|
|||||||
- id: ruff
|
- id: ruff
|
||||||
args: [--fix]
|
args: [--fix]
|
||||||
- id: ruff-format
|
- id: ruff-format
|
||||||
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
|
rev: v5.0.0
|
||||||
|
hooks:
|
||||||
|
- id: check-added-large-files
|
||||||
|
args: [--maxkb=5120]
|
||||||
|
- id: check-merge-conflict
|
||||||
|
- id: check-yaml
|
||||||
|
- id: check-toml
|
||||||
|
- id: detect-private-key
|
||||||
|
- id: end-of-file-fixer
|
||||||
|
- id: trailing-whitespace
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
.PHONY: dev lint typecheck test complexity deps coverage security runbooks docs validate
|
||||||
|
|
||||||
|
# ─── Development Setup ──────────────────────────────────────────
|
||||||
|
dev: .venv
|
||||||
|
@echo "Development environment ready. Run 'make validate' to run all checks."
|
||||||
|
|
||||||
|
.venv:
|
||||||
|
python3 -m venv .venv
|
||||||
|
.venv/bin/python3 -m pip install --upgrade pip
|
||||||
|
.venv/bin/python3 -m pip install -r requirements-dev.txt
|
||||||
|
@echo "Virtual environment created at .venv/"
|
||||||
|
|
||||||
|
# ─── Linting & Formatting ───────────────────────────────────────
|
||||||
|
lint:
|
||||||
|
.venv/bin/python3 -m ruff check scripts/ eval_runner/
|
||||||
|
|
||||||
|
format-check:
|
||||||
|
.venv/bin/python3 -m ruff format --check scripts/ eval_runner/
|
||||||
|
|
||||||
|
format:
|
||||||
|
.venv/bin/python3 -m ruff format scripts/ eval_runner/
|
||||||
|
|
||||||
|
# ─── Type Checking ──────────────────────────────────────────────
|
||||||
|
typecheck:
|
||||||
|
.venv/bin/python3 -m mypy scripts/ eval_runner/
|
||||||
|
|
||||||
|
# ─── Complexity ─────────────────────────────────────────────────
|
||||||
|
complexity:
|
||||||
|
.venv/bin/python3 -m radon cc scripts/ eval_runner/ --min B --total-average
|
||||||
|
|
||||||
|
# ─── Testing ────────────────────────────────────────────────────
|
||||||
|
test:
|
||||||
|
.venv/bin/python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ -v --durations=10
|
||||||
|
|
||||||
|
test-integration:
|
||||||
|
.venv/bin/python3 -m pytest tests/integration/ -v --durations=10
|
||||||
|
|
||||||
|
test-cov:
|
||||||
|
.venv/bin/python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ tests/integration/ -v --durations=10 --cov=scripts --cov=eval_runner --cov-fail-under=60 --cov-report=term-missing
|
||||||
|
|
||||||
|
test-parallel:
|
||||||
|
.venv/bin/python3 -m pytest scripts/test-eval-validation.py scripts/test-eval-coverage.py eval_runner/tests/ tests/integration/ -n auto -v --durations=10
|
||||||
|
|
||||||
|
# ─── Dependencies ───────────────────────────────────────────────
|
||||||
|
deps:
|
||||||
|
.venv/bin/python3 -m deptry .
|
||||||
|
|
||||||
|
# ─── Security ───────────────────────────────────────────────────
|
||||||
|
security:
|
||||||
|
.venv/bin/python3 -m bandit -r scripts/ -f txt --severity-level high
|
||||||
|
|
||||||
|
# ─── Documentation ──────────────────────────────────────────────
|
||||||
|
docs:
|
||||||
|
ruby scripts/validate-skills.rb
|
||||||
|
ruby scripts/validate-skill-quality.rb --base origin/main
|
||||||
|
python3 scripts/test-eval-validation.py
|
||||||
|
python3 scripts/validate-evals.py
|
||||||
|
|
||||||
|
# ─── Full Validation ────────────────────────────────────────────
|
||||||
|
validate: lint format-check typecheck complexity deps test-cov
|
||||||
|
@echo "All checks passed."
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
# Architecture Overview
|
||||||
|
|
||||||
|
This document describes the architecture, data flow, and key components of the agent-skills repository.
|
||||||
|
|
||||||
|
## Repository Structure
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TD
|
||||||
|
subgraph "Repository Root"
|
||||||
|
AGENTS[AGENTS.md<br/>Agent Instructions]
|
||||||
|
README[README.md<br/>Skill Index]
|
||||||
|
CONTRIB[CONTRIBUTING.md<br/>Contribution Guide]
|
||||||
|
PYT[pyproject.toml<br/>Tool Configs]
|
||||||
|
REQ[requirements-dev.txt<br/>Dev Dependencies]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Core Scripts"
|
||||||
|
VS[validate-skills.rb<br/>Skill Format Validator]
|
||||||
|
VSQ[validate-skill-quality.rb<br/>Quality Validator]
|
||||||
|
EV[validate-evals.py<br/>Eval Manifest Validator]
|
||||||
|
EVV[eval_validation.py<br/>Shared Validation Logic]
|
||||||
|
EC[eval-coverage.py<br/>Coverage Reporter]
|
||||||
|
TEC[test-eval-coverage.py<br/>Coverage Tests]
|
||||||
|
TEV[test-eval-validation.py<br/>Validation Tests]
|
||||||
|
CA[check-artifacts.py<br/>Artifact Freshness]
|
||||||
|
GEN[gen-*.rb<br/>Catalog Generators]
|
||||||
|
LOG[logging_utils.py<br/>Structured Logging]
|
||||||
|
STD[scan-tech-debt.py<br/>Debt Scanner]
|
||||||
|
VAM[validate-agents-md.py<br/>AGENTS.md Validator]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "CI/CD"
|
||||||
|
WF[validate.yml<br/>GitHub Actions]
|
||||||
|
PC[.pre-commit-config.yaml<br/>Pre-commit Hooks]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Skills Directory"
|
||||||
|
SK1[skill-name/<br/>SKILL.md + README.md]
|
||||||
|
SK2[skill-name/<br/>references/ templates/ scripts/]
|
||||||
|
EVALS[evals/<br/>evals.json]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Generated Artifacts"
|
||||||
|
MP[.claude-plugin/<br/>marketplace.json]
|
||||||
|
CP[.codex-plugin/<br/>plugin.json]
|
||||||
|
LLM[llms.txt]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Bundles"
|
||||||
|
BND[bundles/<br/>Multi-skill compositions]
|
||||||
|
end
|
||||||
|
|
||||||
|
AGENTS --> VS
|
||||||
|
AGENTS --> WF
|
||||||
|
README --> SK1
|
||||||
|
CONTRIB --> PYT
|
||||||
|
PYT --> WF
|
||||||
|
REQ --> WF
|
||||||
|
|
||||||
|
WF --> VS
|
||||||
|
WF --> VSQ
|
||||||
|
WF --> EV
|
||||||
|
WF --> EC
|
||||||
|
WF --> CA
|
||||||
|
WF --> STD
|
||||||
|
WF --> VAM
|
||||||
|
|
||||||
|
VS --> SK1
|
||||||
|
VSQ --> SK1
|
||||||
|
EV --> EVV
|
||||||
|
EV --> EVALS
|
||||||
|
EC --> EVV
|
||||||
|
CA --> MP
|
||||||
|
CA --> CP
|
||||||
|
CA --> LLM
|
||||||
|
GEN --> MP
|
||||||
|
GEN --> CP
|
||||||
|
GEN --> LLM
|
||||||
|
|
||||||
|
TEC --> EC
|
||||||
|
TEV --> EVV
|
||||||
|
LOG --> STD
|
||||||
|
LOG --> VAM
|
||||||
|
```
|
||||||
|
|
||||||
|
## Data Flow
|
||||||
|
|
||||||
|
### 1. Contribution Flow
|
||||||
|
Developer creates/modifies skill → Pre-commit hooks run → PR triggers CI → Validators check format/quality/evals → Generated artifacts verified → Merge to main.
|
||||||
|
|
||||||
|
### 2. Validation Pipeline
|
||||||
|
```
|
||||||
|
SKILL.md + README.md + evals/evals.json
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
validate-skills.rb ──────► Structural validation (dirs, links, format)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
validate-skill-quality.rb ► Semantic validation (descriptions, triggers)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
validate-evals.py ────────► Eval manifest schema validation
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
eval-coverage.py ─────────► Coverage reporting + ratchet enforcement
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Generated Artifact Pipeline
|
||||||
|
```
|
||||||
|
Skill directories (SKILL.md frontmatter)
|
||||||
|
│
|
||||||
|
├──► gen-claude-marketplace.rb → .claude-plugin/marketplace.json
|
||||||
|
├──► gen-codex-plugin.rb → .codex-plugin/plugin.json
|
||||||
|
└──► gen-llms-txt.rb → llms.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
## Key Components
|
||||||
|
|
||||||
|
| Component | Language | Purpose |
|
||||||
|
|-----------|----------|---------|
|
||||||
|
| `validate-skills.rb` | Ruby | Structural skill format validation |
|
||||||
|
| `validate-skill-quality.rb` | Ruby | Semantic quality validation |
|
||||||
|
| `eval_validation.py` | Python | Shared eval manifest validation |
|
||||||
|
| `eval-coverage.py` | Python | Eval coverage reporting + ratchet |
|
||||||
|
| `check-artifacts.py` | Python | Generated artifact freshness check |
|
||||||
|
| `logging_utils.py` | Python | Structured logging with PII redaction |
|
||||||
|
| `scan-tech-debt.py` | Python | Technical debt marker scanning |
|
||||||
|
|
||||||
|
## External Dependencies
|
||||||
|
|
||||||
|
This repository has no runtime service dependencies. It is a static skills repository with:
|
||||||
|
- **GitHub Actions** for CI/CD validation
|
||||||
|
- **PyPI** packages (ruff, mypy, pytest, radon, deptry, bandit, loguru) for code quality
|
||||||
|
- **Ruby gems** (standard library) for skill validation scripts
|
||||||
|
- No databases, caches, message queues, or external APIs at runtime
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
# Runbooks
|
||||||
|
|
||||||
|
This document references incident response procedures for the agent-skills repository.
|
||||||
|
|
||||||
|
## When Something Goes Wrong
|
||||||
|
|
||||||
|
### CI Validation Failure
|
||||||
|
|
||||||
|
1. Check the [GitHub Actions validate workflow](https://github.com/magnus919/agent-skills/actions/workflows/validate.yml)
|
||||||
|
2. Review the failing step output for specific error messages
|
||||||
|
3. Common failures and resolutions:
|
||||||
|
|
||||||
|
| Failure | Likely Cause | Resolution |
|
||||||
|
|---------|-------------|------------|
|
||||||
|
| `ruff check` fails | Code style violation | Run `make format` locally, then `make lint` |
|
||||||
|
| `mypy` fails | Type annotation error | Fix type annotations per mypy output |
|
||||||
|
| `radon` fails | Complexity threshold exceeded | Refactor complex function into smaller units |
|
||||||
|
| `pytest` fails | Test regression | Reproduce locally with `make test` |
|
||||||
|
| `deptry` fails | Unused or missing dependency | Run `make deps` and update requirements-dev.txt |
|
||||||
|
| `validate-skills.rb` fails | Invalid SKILL.md format | Check frontmatter YAML, relative links |
|
||||||
|
| `eval-coverage` ratchet fails | Schema-valid manifest coverage decreased | Add evals/evals.json to modified/new skills |
|
||||||
|
|
||||||
|
### Generated Artifact Staleness
|
||||||
|
|
||||||
|
If CI reports stale catalog artifacts:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
ruby scripts/gen-claude-marketplace.rb --write
|
||||||
|
ruby scripts/gen-codex-plugin.rb --write
|
||||||
|
ruby scripts/gen-llms-txt.rb --write
|
||||||
|
git add .claude-plugin/ .codex-plugin/ llms.txt
|
||||||
|
git commit -m "chore: refresh generated artifacts"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Security Vulnerability in Dependency
|
||||||
|
|
||||||
|
1. Review the bandit or Dependabot alert
|
||||||
|
2. Update the affected dependency in `requirements-dev.txt`
|
||||||
|
3. Run `make validate` to confirm no regressions
|
||||||
|
4. Review the changelog of the updated dependency for breaking changes
|
||||||
|
|
||||||
|
### Pre-commit Hook Issues
|
||||||
|
|
||||||
|
If pre-commit hooks are blocking commits:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Run all hooks manually to see errors
|
||||||
|
pre-commit run --all-files
|
||||||
|
|
||||||
|
# Update hooks to latest versions
|
||||||
|
pre-commit autoupdate
|
||||||
|
```
|
||||||
|
|
||||||
|
## Deployment
|
||||||
|
|
||||||
|
This repository has no deployment step. All changes are validated in CI and merged to `main`. Catalog artifacts (`llms.txt`, marketplace.json, plugin.json) are tracked in the repository and refreshed on each change.
|
||||||
|
|
||||||
|
## Monitoring
|
||||||
|
|
||||||
|
- [GitHub Actions dashboard](https://github.com/magnus919/agent-skills/actions) for CI health
|
||||||
|
- [GitHub Security Advisories](https://github.com/magnus919/agent-skills/security) for vulnerability alerts
|
||||||
|
- [Dependabot alerts](https://github.com/magnus919/agent-skills/security/dependabot) for dependency updates
|
||||||
+5
-2
@@ -31,6 +31,8 @@ ignore = [
|
|||||||
"eval_runner/comparison.py" = ["TCH"]
|
"eval_runner/comparison.py" = ["TCH"]
|
||||||
"eval_runner/adapter.py" = ["TCH"]
|
"eval_runner/adapter.py" = ["TCH"]
|
||||||
"eval_runner/manifest.py" = ["E402"]
|
"eval_runner/manifest.py" = ["E402"]
|
||||||
|
# Integration tests need sys.path manipulation before imports
|
||||||
|
"tests/integration/*.py" = ["E402"]
|
||||||
|
|
||||||
[tool.ruff.format]
|
[tool.ruff.format]
|
||||||
quote-style = "double"
|
quote-style = "double"
|
||||||
@@ -40,7 +42,7 @@ line-ending = "auto"
|
|||||||
|
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
minversion = "7.0"
|
minversion = "7.0"
|
||||||
testpaths = ["scripts", "eval_runner/tests"]
|
testpaths = ["scripts", "eval_runner/tests", "tests/integration"]
|
||||||
python_files = "test_*.py"
|
python_files = "test_*.py"
|
||||||
python_classes = "*Test*"
|
python_classes = "*Test*"
|
||||||
python_functions = "test_*"
|
python_functions = "test_*"
|
||||||
@@ -79,6 +81,7 @@ warn_unreachable = true
|
|||||||
[[tool.mypy.overrides]]
|
[[tool.mypy.overrides]]
|
||||||
module = [
|
module = [
|
||||||
"eval_runner.tests.*",
|
"eval_runner.tests.*",
|
||||||
|
"tests.integration.*",
|
||||||
"eval_runner.openai_adapter",
|
"eval_runner.openai_adapter",
|
||||||
"eval_runner.fake_adapter",
|
"eval_runner.fake_adapter",
|
||||||
"test-eval-coverage",
|
"test-eval-coverage",
|
||||||
@@ -125,5 +128,5 @@ extend_exclude = [
|
|||||||
"yc-weekly-growth-compass",
|
"yc-weekly-growth-compass",
|
||||||
]
|
]
|
||||||
[tool.deptry.per_rule_ignores]
|
[tool.deptry.per_rule_ignores]
|
||||||
DEP002 = ["ruff", "pytest", "pytest-cov", "mypy", "radon", "deptry", "requests", "types-jsonschema"]
|
DEP002 = ["ruff", "pytest", "pytest-cov", "pytest-xdist", "mypy", "radon", "deptry", "bandit", "requests", "types-jsonschema"]
|
||||||
DEP001 = ["eval_validation"]
|
DEP001 = ["eval_validation"]
|
||||||
|
|||||||
@@ -4,6 +4,9 @@ requests==2.34.2
|
|||||||
ruff>=0.9.0
|
ruff>=0.9.0
|
||||||
pytest>=7.0
|
pytest>=7.0
|
||||||
pytest-cov>=4.0
|
pytest-cov>=4.0
|
||||||
|
pytest-xdist>=3.0
|
||||||
mypy>=1.0
|
mypy>=1.0
|
||||||
radon>=6.0
|
radon>=6.0
|
||||||
deptry>=0.20
|
deptry>=0.20
|
||||||
|
bandit>=1.7
|
||||||
|
loguru>=0.7
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
"""Structured logging with PII/sensitive-data redaction.
|
||||||
|
|
||||||
|
Uses loguru for structured, colorized output with automatic redaction of
|
||||||
|
sensitive patterns (API keys, tokens, credentials, email addresses, IPs).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import functools
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from loguru import logger as _logger # type: ignore[import-not-found,unused-ignore]
|
||||||
|
|
||||||
|
# Patterns detected and redacted in log messages
|
||||||
|
_REDACT_PATTERNS: list[tuple[str, str]] = [
|
||||||
|
# API keys and tokens (common formats)
|
||||||
|
(
|
||||||
|
r"(?i)(api[_-]?key|apikey|secret[_-]?key|auth[_-]?token|access[_-]?token|bearer)\s*[:=]\s*[\S]+",
|
||||||
|
r"\1=<REDACTED>",
|
||||||
|
),
|
||||||
|
# JWT tokens
|
||||||
|
(r"eyJ[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}", "<JWT_REDACTED>"),
|
||||||
|
# GitHub tokens
|
||||||
|
(r"gh[pousr]_[A-Za-z0-9_]{20,}", "<GITHUB_TOKEN_REDACTED>"),
|
||||||
|
# Generic hex tokens (32+ hex chars)
|
||||||
|
(r"\b[a-fA-F0-9]{32,}\b", "<HEX_TOKEN_REDACTED>"),
|
||||||
|
# Email addresses
|
||||||
|
(r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b", "<EMAIL_REDACTED>"),
|
||||||
|
# IPv4 addresses
|
||||||
|
(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", "<IP_REDACTED>"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_sensitive(message: str) -> str:
|
||||||
|
"""Strip sensitive data patterns from a log message."""
|
||||||
|
for pattern, replacement in _REDACT_PATTERNS:
|
||||||
|
message = re.sub(pattern, replacement, message)
|
||||||
|
return message
|
||||||
|
|
||||||
|
|
||||||
|
def _redacting_patcher(record: Any) -> None:
|
||||||
|
"""Loguru patcher that redacts sensitive data from the message."""
|
||||||
|
record["message"] = _redact_sensitive(str(record["message"])) # type: ignore[index,unused-ignore]
|
||||||
|
|
||||||
|
|
||||||
|
def configure_logger(
|
||||||
|
level: str = "INFO",
|
||||||
|
json_output: bool = False,
|
||||||
|
log_file: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Configure the global logger with sensible defaults.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
level: Minimum log level (DEBUG, INFO, WARNING, ERROR).
|
||||||
|
json_output: Emit JSON-structured logs instead of colorized text.
|
||||||
|
log_file: Optional path for file-based logging.
|
||||||
|
"""
|
||||||
|
_logger.remove()
|
||||||
|
|
||||||
|
# Console sink: colorized for humans, JSON for machines
|
||||||
|
if json_output:
|
||||||
|
_logger.add(
|
||||||
|
sys.stderr,
|
||||||
|
level=level,
|
||||||
|
format="{message}",
|
||||||
|
serialize=True,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
_logger.add(
|
||||||
|
sys.stderr,
|
||||||
|
level=level,
|
||||||
|
format=(
|
||||||
|
"<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
|
||||||
|
"<level>{level: <8}</level> | "
|
||||||
|
"<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - "
|
||||||
|
"<level>{message}</level>"
|
||||||
|
),
|
||||||
|
colorize=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# File sink if requested
|
||||||
|
if log_file:
|
||||||
|
os.makedirs(os.path.dirname(log_file) or ".", exist_ok=True)
|
||||||
|
_logger.add(
|
||||||
|
log_file,
|
||||||
|
level="DEBUG",
|
||||||
|
format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}",
|
||||||
|
rotation="10 MB",
|
||||||
|
retention="7 days",
|
||||||
|
serialize=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Apply redaction to all messages
|
||||||
|
_logger.configure(patcher=_redacting_patcher)
|
||||||
|
|
||||||
|
|
||||||
|
def get_logger(name: str = __name__) -> Any:
|
||||||
|
"""Return a bound logger for the given module name."""
|
||||||
|
return _logger.bind(name=name)
|
||||||
|
|
||||||
|
|
||||||
|
@functools.wraps(print)
|
||||||
|
def safe_print(*args: Any, **kwargs: Any) -> None:
|
||||||
|
"""Print wrapper that redacts sensitive data."""
|
||||||
|
message = " ".join(str(arg) for arg in args)
|
||||||
|
kwargs.pop("file", None) # always use stderr/default
|
||||||
|
_logger.info(message)
|
||||||
|
|
||||||
|
|
||||||
|
# Auto-configure on import in non-production settings
|
||||||
|
if os.environ.get("LOGURU_LEVEL") or os.environ.get("CI"):
|
||||||
|
configure_logger(
|
||||||
|
level=os.environ.get("LOGURU_LEVEL", "INFO"),
|
||||||
|
json_output=bool(os.environ.get("CI")),
|
||||||
|
)
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Manage GitHub issue labels for consistent priority, type, and area classification.
|
||||||
|
|
||||||
|
Run with GITHUB_TOKEN set to manage labels on the agent-skills repository.
|
||||||
|
Without a token, prints the recommended label configuration for manual setup.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
GITHUB_TOKEN=... python3 scripts/manage-labels.py # apply labels
|
||||||
|
GITHUB_TOKEN=... python3 scripts/manage-labels.py --dry-run # preview only
|
||||||
|
python3 scripts/manage-labels.py # print config
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
REPO = "magnus919/agent-skills"
|
||||||
|
API_BASE = f"https://api.github.com/repos/{REPO}"
|
||||||
|
|
||||||
|
LABELS: dict[str, dict[str, str]] = {
|
||||||
|
# Priority labels (P0-P3)
|
||||||
|
"priority/P0-critical": {
|
||||||
|
"color": "b60205",
|
||||||
|
"description": "Drop everything: blocks releases, security incidents, data loss",
|
||||||
|
},
|
||||||
|
"priority/P1-high": {
|
||||||
|
"color": "d93f0b",
|
||||||
|
"description": "Must fix this sprint: user-facing broken, deadline at risk",
|
||||||
|
},
|
||||||
|
"priority/P2-medium": {
|
||||||
|
"color": "fbca04",
|
||||||
|
"description": "Should fix soon: important but not blocking",
|
||||||
|
},
|
||||||
|
"priority/P3-low": {
|
||||||
|
"color": "0e8a16",
|
||||||
|
"description": "Nice to have: backlog grooming, minor improvements",
|
||||||
|
},
|
||||||
|
# Type labels
|
||||||
|
"type/bug": {
|
||||||
|
"color": "d73a4a",
|
||||||
|
"description": "Something is broken",
|
||||||
|
},
|
||||||
|
"type/feature": {
|
||||||
|
"color": "a2eeef",
|
||||||
|
"description": "New capability or enhancement",
|
||||||
|
},
|
||||||
|
"type/chore": {
|
||||||
|
"color": "c5def5",
|
||||||
|
"description": "Maintenance, refactoring, dependency updates",
|
||||||
|
},
|
||||||
|
"type/documentation": {
|
||||||
|
"color": "0075ca",
|
||||||
|
"description": "Documentation improvements",
|
||||||
|
},
|
||||||
|
"type/question": {
|
||||||
|
"color": "d876e3",
|
||||||
|
"description": "Needs discussion or clarification",
|
||||||
|
},
|
||||||
|
# Area labels
|
||||||
|
"area/ci-cd": {
|
||||||
|
"color": "5319e7",
|
||||||
|
"description": "CI/CD pipelines, GitHub Actions, automation",
|
||||||
|
},
|
||||||
|
"area/validation": {
|
||||||
|
"color": "006b75",
|
||||||
|
"description": "Skill validation, eval manifests, quality checks",
|
||||||
|
},
|
||||||
|
"area/skills": {
|
||||||
|
"color": "bfdadc",
|
||||||
|
"description": "Individual skill content and structure",
|
||||||
|
},
|
||||||
|
"area/docs": {
|
||||||
|
"color": "c2e0c6",
|
||||||
|
"description": "READMEs, AGENTS.md, CONTRIBUTING.md, architecture",
|
||||||
|
},
|
||||||
|
"area/tooling": {
|
||||||
|
"color": "f9d0c4",
|
||||||
|
"description": "Scripts, generators, CLI tools",
|
||||||
|
},
|
||||||
|
"area/security": {
|
||||||
|
"color": "b60205",
|
||||||
|
"description": "Security concerns, vulnerabilities, auditing",
|
||||||
|
},
|
||||||
|
# Status labels
|
||||||
|
"status/blocked": {
|
||||||
|
"color": "000000",
|
||||||
|
"description": "Cannot proceed due to dependency or external factor",
|
||||||
|
},
|
||||||
|
"status/needs-triage": {
|
||||||
|
"color": "ededed",
|
||||||
|
"description": "New issue awaiting triage",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _api_request(method: str, path: str, data: dict[str, Any] | None = None) -> Any:
|
||||||
|
token = os.environ.get("GITHUB_TOKEN", "")
|
||||||
|
if not token:
|
||||||
|
raise RuntimeError("GITHUB_TOKEN environment variable is not set")
|
||||||
|
|
||||||
|
url = f"{API_BASE}{path}"
|
||||||
|
body = json.dumps(data).encode("utf-8") if data else None
|
||||||
|
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url,
|
||||||
|
data=body,
|
||||||
|
headers={
|
||||||
|
"Authorization": f"Bearer {token}",
|
||||||
|
"Accept": "application/vnd.github.v3+json",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
method=method,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as resp: # nosec B310
|
||||||
|
return json.loads(resp.read().decode("utf-8"))
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
error_body = e.read().decode("utf-8", errors="replace")
|
||||||
|
print(f"HTTP {e.code}: {error_body}", file=sys.stderr)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def get_existing_labels() -> dict[str, dict[str, Any]]:
|
||||||
|
"""Fetch existing labels from the repository."""
|
||||||
|
labels: dict[str, dict[str, Any]] = {}
|
||||||
|
page = 1
|
||||||
|
while True:
|
||||||
|
result: list[dict[str, Any]] = _api_request("GET", f"/labels?per_page=100&page={page}")
|
||||||
|
if not result:
|
||||||
|
break
|
||||||
|
for label in result:
|
||||||
|
labels[label["name"]] = label
|
||||||
|
page += 1
|
||||||
|
return labels
|
||||||
|
|
||||||
|
|
||||||
|
def create_label(name: str, config: dict[str, str]) -> None:
|
||||||
|
"""Create a new label."""
|
||||||
|
print(f" Creating: {name}")
|
||||||
|
_api_request("POST", "/labels", {"name": name, **config})
|
||||||
|
|
||||||
|
|
||||||
|
def update_label(name: str, config: dict[str, str]) -> None:
|
||||||
|
"""Update an existing label."""
|
||||||
|
print(f" Updating: {name}")
|
||||||
|
_api_request("PATCH", f"/labels/{urllib.parse.quote(name)}", dict(config))
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
dry_run = "--dry-run" in sys.argv
|
||||||
|
|
||||||
|
if not os.environ.get("GITHUB_TOKEN"):
|
||||||
|
print("# Recommended label configuration for agent-skills\n")
|
||||||
|
print("To apply, set GITHUB_TOKEN and run:")
|
||||||
|
print(" python3 scripts/manage-labels.py\n")
|
||||||
|
for name, config in LABELS.items():
|
||||||
|
print(f" {name}: #{config['color']} - {config['description']}")
|
||||||
|
print(f"\n{len(LABELS)} labels total.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
existing = get_existing_labels()
|
||||||
|
print(f"Found {len(existing)} existing labels.\n")
|
||||||
|
|
||||||
|
for name, config in LABELS.items():
|
||||||
|
if name in existing:
|
||||||
|
if (
|
||||||
|
existing[name]["color"] != config["color"]
|
||||||
|
or existing[name]["description"] != config["description"]
|
||||||
|
):
|
||||||
|
if dry_run:
|
||||||
|
print(f" [DRY RUN] Would update: {name}")
|
||||||
|
else:
|
||||||
|
update_label(name, config)
|
||||||
|
else:
|
||||||
|
print(f" OK: {name}")
|
||||||
|
else:
|
||||||
|
if dry_run:
|
||||||
|
print(f" [DRY RUN] Would create: {name}")
|
||||||
|
else:
|
||||||
|
create_label(name, config)
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
print(f"\nDry run complete. {len(LABELS)} labels would be reconciled.")
|
||||||
|
else:
|
||||||
|
print(f"\nDone. {len(LABELS)} labels reconciled.")
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Scan for TODO, FIXME, HACK, and XXX markers in tracked source files.
|
||||||
|
|
||||||
|
Reports technical debt markers with file, line, and surrounding context.
|
||||||
|
Used in CI to prevent unlinked tech debt from accumulating.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
MARKERS = ("TODO", "FIXME", "HACK", "XXX")
|
||||||
|
EXCLUDE_DIRS = {
|
||||||
|
".git",
|
||||||
|
".venv",
|
||||||
|
"venv",
|
||||||
|
"node_modules",
|
||||||
|
"__pycache__",
|
||||||
|
".mypy_cache",
|
||||||
|
".pytest_cache",
|
||||||
|
".ruff_cache",
|
||||||
|
"dist",
|
||||||
|
"build",
|
||||||
|
"logs",
|
||||||
|
".hermes",
|
||||||
|
}
|
||||||
|
EXCLUDE_PATTERNS = {"*.pyc", "*.pyo", "*.egg-info", "*.whl", "*.min.js", "*.min.css"}
|
||||||
|
|
||||||
|
|
||||||
|
def is_excluded(path: Path) -> bool:
|
||||||
|
parts = set(path.parts)
|
||||||
|
if parts & EXCLUDE_DIRS:
|
||||||
|
return True
|
||||||
|
return any(path.match(pat) for pat in EXCLUDE_PATTERNS)
|
||||||
|
|
||||||
|
|
||||||
|
def scan_file(filepath: Path) -> list[tuple[int, str, str]]:
|
||||||
|
"""Return list of (line_number, marker, context) for marker occurrences."""
|
||||||
|
findings: list[tuple[int, str, str]] = []
|
||||||
|
try:
|
||||||
|
content = filepath.read_text(encoding="utf-8", errors="replace")
|
||||||
|
except (OSError, UnicodeDecodeError):
|
||||||
|
return findings
|
||||||
|
for lineno, line in enumerate(content.splitlines(), start=1):
|
||||||
|
stripped = line.strip()
|
||||||
|
if stripped.startswith("#"):
|
||||||
|
for marker in MARKERS:
|
||||||
|
if marker in stripped:
|
||||||
|
# Extract the comment content after the marker
|
||||||
|
idx = stripped.index(marker)
|
||||||
|
context = stripped[idx:].rstrip()
|
||||||
|
findings.append((lineno, marker, context))
|
||||||
|
break
|
||||||
|
return findings
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"git",
|
||||||
|
"ls-files",
|
||||||
|
"--cached",
|
||||||
|
"--others",
|
||||||
|
"--exclude-standard",
|
||||||
|
"*.py",
|
||||||
|
"*.rb",
|
||||||
|
"*.sh",
|
||||||
|
"*.js",
|
||||||
|
"*.ts",
|
||||||
|
"*.yml",
|
||||||
|
"*.yaml",
|
||||||
|
"*.toml",
|
||||||
|
"*.md",
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
print("ERROR: Failed to list tracked files.", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
total = 0
|
||||||
|
for relpath in result.stdout.strip().splitlines():
|
||||||
|
if not relpath:
|
||||||
|
continue
|
||||||
|
filepath = Path(relpath)
|
||||||
|
if is_excluded(filepath):
|
||||||
|
continue
|
||||||
|
findings = scan_file(filepath)
|
||||||
|
for lineno, marker, context in findings:
|
||||||
|
if total == 0:
|
||||||
|
print("\nTechnical debt markers found:\n")
|
||||||
|
print(f" {filepath}:{lineno} [{marker}] {context}")
|
||||||
|
total += 1
|
||||||
|
|
||||||
|
if total > 0:
|
||||||
|
print(f"\n{total} technical debt marker(s) found.")
|
||||||
|
print("Consider linking each marker to an issue (e.g., TODO(#123)).")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
print("No technical debt markers found in tracked source files.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Validate that AGENTS.md documented commands still work.
|
||||||
|
|
||||||
|
Parses AGENTS.md for code blocks containing shell commands and validates
|
||||||
|
basic structure and existence of referenced scripts/commands.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
AGENTS_MD = ROOT / "AGENTS.md"
|
||||||
|
|
||||||
|
|
||||||
|
def extract_code_blocks(content: str) -> list[tuple[int, str]]:
|
||||||
|
"""Extract shell code blocks with their starting line numbers."""
|
||||||
|
blocks = []
|
||||||
|
in_block = False
|
||||||
|
block_lines: list[str] = []
|
||||||
|
block_start = 0
|
||||||
|
lang = ""
|
||||||
|
|
||||||
|
for i, line in enumerate(content.splitlines(), 1):
|
||||||
|
if line.strip().startswith("```") and not in_block:
|
||||||
|
in_block = True
|
||||||
|
block_start = i
|
||||||
|
lang = line.strip()[3:].strip().lower()
|
||||||
|
block_lines = []
|
||||||
|
elif line.strip() == "```" and in_block:
|
||||||
|
in_block = False
|
||||||
|
if lang in ("sh", "shell", "bash", ""):
|
||||||
|
blocks.append((block_start, "\n".join(block_lines)))
|
||||||
|
elif in_block:
|
||||||
|
block_lines.append(line)
|
||||||
|
|
||||||
|
return blocks
|
||||||
|
|
||||||
|
|
||||||
|
def validate_references(blocks: list[tuple[int, str]]) -> tuple[int, int, list[str]]:
|
||||||
|
"""Check that referenced scripts and commands exist in the repo.
|
||||||
|
|
||||||
|
Returns (pass_count, fail_count, error_messages).
|
||||||
|
"""
|
||||||
|
errors: list[str] = []
|
||||||
|
script_ref_pattern = re.compile(r"scripts/[\w./-]+")
|
||||||
|
|
||||||
|
for lineno, block in blocks:
|
||||||
|
refs = script_ref_pattern.findall(block)
|
||||||
|
for ref in refs:
|
||||||
|
full_path = ROOT / ref
|
||||||
|
if not full_path.exists():
|
||||||
|
errors.append(f"AGENTS.md:{lineno}: referenced script '{ref}' does not exist")
|
||||||
|
|
||||||
|
# Check that AGENTS.md references requirements-dev.txt if it mentions pip install
|
||||||
|
content = AGENTS_MD.read_text()
|
||||||
|
if "pip install" in content and "requirements-dev.txt" not in content:
|
||||||
|
errors.append("AGENTS.md mentions pip install but not requirements-dev.txt")
|
||||||
|
|
||||||
|
pass_count = len(blocks) - len([e for e in errors if "does not exist" in e])
|
||||||
|
fail_count = len([e for e in errors if "does not exist" in e])
|
||||||
|
return pass_count, fail_count, errors
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if not AGENTS_MD.exists():
|
||||||
|
print("AGENTS.md not found.", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
content = AGENTS_MD.read_text()
|
||||||
|
|
||||||
|
# Structural checks
|
||||||
|
issues: list[str] = []
|
||||||
|
|
||||||
|
# Check for required sections
|
||||||
|
required_sections = [
|
||||||
|
("How to Load Skills", "loading instructions"),
|
||||||
|
("Best Practices", "best practices"),
|
||||||
|
]
|
||||||
|
for section, description in required_sections:
|
||||||
|
if f"## {section}" not in content:
|
||||||
|
issues.append(f"AGENTS.md missing '{section}' section ({description})")
|
||||||
|
|
||||||
|
# Extract and validate code blocks
|
||||||
|
blocks = extract_code_blocks(content)
|
||||||
|
if not blocks:
|
||||||
|
issues.append("AGENTS.md contains no shell code blocks with commands")
|
||||||
|
|
||||||
|
_, _fail_count, ref_errors = validate_references(blocks)
|
||||||
|
issues.extend(ref_errors)
|
||||||
|
|
||||||
|
if issues:
|
||||||
|
for issue in issues:
|
||||||
|
print(f" ERROR: {issue}", file=sys.stderr)
|
||||||
|
print(f"\n{len(issues)} issue(s) found in AGENTS.md.", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print("AGENTS.md validation passed.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
"""Integration tests for the agent-skills validation pipeline.
|
||||||
|
|
||||||
|
Tests the full validation flow end-to-end: creating a skill, validating its
|
||||||
|
structure, checking eval manifests, and verifying coverage reporting.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parents[2] / "scripts"
|
||||||
|
ROOT = SCRIPT_DIR.parent
|
||||||
|
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from eval_validation import find_skill_manifests, validate_manifest
|
||||||
|
|
||||||
|
|
||||||
|
def _git(repo: Path, *args: str) -> None:
|
||||||
|
subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_skill_dir(tmp: Path, name: str) -> Path:
|
||||||
|
"""Create a minimal valid skill directory."""
|
||||||
|
skill = tmp / name
|
||||||
|
skill.mkdir(parents=True)
|
||||||
|
|
||||||
|
# SKILL.md
|
||||||
|
(skill / "SKILL.md").write_text(
|
||||||
|
f"---\nname: {name}\ndescription: Test skill for integration testing.\n---\n\n"
|
||||||
|
"# {name}\n\nA test skill for the integration test suite.\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
# README.md
|
||||||
|
(skill / "README.md").write_text(
|
||||||
|
f"# {name}\n\n## Why Install This Skill\n\n"
|
||||||
|
"This skill helps with testing the validation pipeline.\n\n"
|
||||||
|
"## What You Get\n\n"
|
||||||
|
"| File | Purpose |\n|------|--------|\n| SKILL.md | Instructions |\n\n"
|
||||||
|
"## Quick Start\n\nNo setup needed.\n\n"
|
||||||
|
"## Triggers\n\n- When integration tests run\n\n"
|
||||||
|
"## Requirements\n\nPython 3.10+\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
# evals/evals.json
|
||||||
|
(skill / "evals").mkdir(exist_ok=True)
|
||||||
|
manifest = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"skill_name": name,
|
||||||
|
"evals": [
|
||||||
|
{
|
||||||
|
"id": "int-01",
|
||||||
|
"prompt": "Run the skill.",
|
||||||
|
"expected_output": "The skill ran successfully.",
|
||||||
|
"assertions": ["Skill completes without error."],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "int-02",
|
||||||
|
"prompt": "Test edge case.",
|
||||||
|
"expected_output": "Edge case handled.",
|
||||||
|
"assertions": ["No crash on edge input."],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "int-03",
|
||||||
|
"prompt": "Verify output format.",
|
||||||
|
"expected_output": "Valid JSON output.",
|
||||||
|
"assertions": ["Output is valid JSON."],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "int-04",
|
||||||
|
"prompt": "Test with empty input.",
|
||||||
|
"expected_output": "Graceful handling.",
|
||||||
|
"assertions": ["Returns appropriate error."],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "int-05",
|
||||||
|
"prompt": "Test concurrent access.",
|
||||||
|
"expected_output": "Thread-safe operation.",
|
||||||
|
"assertions": ["No race conditions."],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
(skill / "evals" / "evals.json").write_text(json.dumps(manifest, indent=2))
|
||||||
|
|
||||||
|
return skill
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidateEvalsIntegration(unittest.TestCase):
|
||||||
|
"""Integration tests for eval validation."""
|
||||||
|
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.root = Path(self.tmp.name)
|
||||||
|
_git(self.root, "init", "-q")
|
||||||
|
_git(self.root, "config", "user.email", "test@example.invalid")
|
||||||
|
_git(self.root, "config", "user.name", "Test")
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_valid_skill_passes_validation(self) -> None:
|
||||||
|
"""A properly constructed skill should pass eval validation."""
|
||||||
|
skill = _make_skill_dir(self.root, "test-skill")
|
||||||
|
manifest_path = skill / "evals" / "evals.json"
|
||||||
|
|
||||||
|
self.assertTrue(manifest_path.exists())
|
||||||
|
result = validate_manifest(manifest_path, self.root)
|
||||||
|
self.assertEqual(len(result.errors), 0, f"Validation errors: {result.errors}")
|
||||||
|
self.assertEqual(result.states.get("schema_valid"), True)
|
||||||
|
|
||||||
|
def test_manifest_with_missing_evals_fails(self) -> None:
|
||||||
|
"""A manifest without the 'evals' key should fail validation."""
|
||||||
|
skill = _make_skill_dir(self.root, "bad-skill")
|
||||||
|
(skill / "evals" / "evals.json").write_text(
|
||||||
|
json.dumps({"schema_version": 1, "skill_name": "bad-skill"})
|
||||||
|
)
|
||||||
|
result = validate_manifest(skill / "evals" / "evals.json", self.root)
|
||||||
|
self.assertGreater(len(result.errors), 0)
|
||||||
|
|
||||||
|
def test_manifest_with_empty_evals_fails(self) -> None:
|
||||||
|
"""A manifest with empty evals array should fail."""
|
||||||
|
skill = _make_skill_dir(self.root, "empty-skill")
|
||||||
|
(skill / "evals" / "evals.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"skill_name": "empty-skill",
|
||||||
|
"evals": [],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
result = validate_manifest(skill / "evals" / "evals.json", self.root)
|
||||||
|
self.assertGreater(len(result.errors), 0)
|
||||||
|
|
||||||
|
def test_manifest_with_invalid_schema_version_fails(self) -> None:
|
||||||
|
"""A manifest with wrong schema_version should fail."""
|
||||||
|
skill = _make_skill_dir(self.root, "wrong-version")
|
||||||
|
manifest = json.loads((skill / "evals" / "evals.json").read_text())
|
||||||
|
manifest["schema_version"] = 99
|
||||||
|
(skill / "evals" / "evals.json").write_text(json.dumps(manifest))
|
||||||
|
result = validate_manifest(skill / "evals" / "evals.json", self.root)
|
||||||
|
self.assertGreater(len(result.errors), 0)
|
||||||
|
self.assertNotEqual(result.states.get("schema_valid"), True)
|
||||||
|
|
||||||
|
def test_find_skill_manifests_discovers_all(self) -> None:
|
||||||
|
"""find_skill_manifests should discover all skill eval manifests."""
|
||||||
|
_make_skill_dir(self.root, "skill-a")
|
||||||
|
_make_skill_dir(self.root, "skill-b")
|
||||||
|
|
||||||
|
manifests = find_skill_manifests(self.root)
|
||||||
|
self.assertGreaterEqual(len(manifests), 2)
|
||||||
|
|
||||||
|
names = {m.parent.parent.name for m in manifests}
|
||||||
|
self.assertIn("skill-a", names)
|
||||||
|
self.assertIn("skill-b", names)
|
||||||
|
|
||||||
|
def test_skill_without_evals_dir_is_handled(self) -> None:
|
||||||
|
"""Skills without evals dir should not cause errors in discovery."""
|
||||||
|
skill = self.root / "no-evals-skill"
|
||||||
|
skill.mkdir()
|
||||||
|
(skill / "SKILL.md").write_text(
|
||||||
|
"---\nname: no-evals-skill\ndescription: Test.\n---\n# Test\n"
|
||||||
|
)
|
||||||
|
manifests = find_skill_manifests(self.root)
|
||||||
|
no_eval_paths = [m for m in manifests if "no-evals-skill" in str(m)]
|
||||||
|
self.assertEqual(len(no_eval_paths), 0)
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidationPipelineEndToEnd(unittest.TestCase):
|
||||||
|
"""End-to-end tests for the full validation pipeline."""
|
||||||
|
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.root = Path(self.tmp.name)
|
||||||
|
_git(self.root, "init", "-q")
|
||||||
|
_git(self.root, "config", "user.email", "test@example.invalid")
|
||||||
|
_git(self.root, "config", "user.name", "Test")
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_validate_evals_accepts_valid_skill(self) -> None:
|
||||||
|
"""validate-evals.py should accept a valid skill."""
|
||||||
|
_make_skill_dir(self.root, "test-skill")
|
||||||
|
_git(self.root, "add", "test-skill/")
|
||||||
|
_git(self.root, "commit", "-m", "add skill")
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, str(SCRIPT_DIR / "validate-evals.py")],
|
||||||
|
cwd=self.root,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
|
||||||
|
|
||||||
|
def test_eval_coverage_reports_on_skills(self) -> None:
|
||||||
|
"""eval-coverage.py should report coverage for skills."""
|
||||||
|
_make_skill_dir(self.root, "covered-skill")
|
||||||
|
_git(self.root, "add", "covered-skill/")
|
||||||
|
_git(self.root, "commit", "-m", "add covered skill")
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, str(SCRIPT_DIR / "eval-coverage.py")],
|
||||||
|
cwd=self.root,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
# Coverage script reports (always exits 0 for informational mode)
|
||||||
|
self.assertEqual(result.returncode, 0)
|
||||||
|
self.assertIn("schema-valid", result.stdout.lower())
|
||||||
|
|
||||||
|
|
||||||
|
class TestScanTechDebt(unittest.TestCase):
|
||||||
|
"""Integration test for the tech debt scanner."""
|
||||||
|
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.root = Path(self.tmp.name)
|
||||||
|
_git(self.root, "init", "-q")
|
||||||
|
_git(self.root, "config", "user.email", "test@example.invalid")
|
||||||
|
_git(self.root, "config", "user.name", "Test")
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_detects_todo_markers(self) -> None:
|
||||||
|
"""scan-tech-debt.py should detect TODO markers in Python files."""
|
||||||
|
test_file = self.root / "module.py"
|
||||||
|
test_file.write_text("# TODO(#42): Fix this later\ndef foo(): pass\n")
|
||||||
|
_git(self.root, "add", "module.py")
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, str(SCRIPT_DIR / "scan-tech-debt.py")],
|
||||||
|
cwd=self.root,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
self.assertIn("TODO", result.stdout)
|
||||||
|
|
||||||
|
def test_reports_zero_for_clean_code(self) -> None:
|
||||||
|
"""scan-tech-debt.py should report zero for clean code."""
|
||||||
|
test_file = self.root / "clean.py"
|
||||||
|
test_file.write_text("def foo():\n return 42\n")
|
||||||
|
_git(self.root, "add", "clean.py")
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, str(SCRIPT_DIR / "scan-tech-debt.py")],
|
||||||
|
cwd=self.root,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
self.assertIn("No technical debt markers found", result.stdout)
|
||||||
Reference in New Issue
Block a user