mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-12 12:06:29 +03:00
Magnus HedemarkandGitHub
be0c8df5e2
Phase 1: New skills (not in grandfathered-skills.txt) must have evals/evals.json with at least 5 test cases. All 107 existing skills are grandfathered. Phase 2: scripts/eval-coverage.py reports coverage (skills with/without evals, case counts, reference-priority sorting). Added as informational CI step. Phase 3: Ratchet thresholds — at 25% coverage, modified skills without evals get a warning; at 50%, they fail CI. Enforced via --modified-from flag for PR-scoped checks. Closes #90
49 lines
1.6 KiB
YAML
49 lines
1.6 KiB
YAML
name: Validate skills
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
pull_request:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
validate:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Check out repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Set up Ruby
|
|
uses: ruby/setup-ruby@v1
|
|
with:
|
|
ruby-version: '3.3'
|
|
- name: Validate skill format and links
|
|
run: ruby scripts/validate-skills.rb
|
|
- name: Test changed-skill quality validation
|
|
run: ruby scripts/test-validate-skill-quality.rb
|
|
- name: Validate changed skill quality
|
|
env:
|
|
SKILL_QUALITY_BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
|
|
run: ruby scripts/validate-skill-quality.rb --base "$SKILL_QUALITY_BASE"
|
|
- name: Check tracked repository artifacts
|
|
run: python3 scripts/check-artifacts.py
|
|
- name: Validate Claude Code marketplace
|
|
run: ruby scripts/gen-claude-marketplace.rb
|
|
- name: Validate Codex plugin packaging
|
|
run: ruby scripts/gen-codex-plugin.rb
|
|
- name: Report eval coverage
|
|
run: python3 scripts/eval-coverage.py
|
|
- name: Test llms.txt generator
|
|
run: ruby scripts/test-gen-llms-txt.rb
|
|
- name: Validate llms.txt catalog
|
|
run: ruby scripts/gen-llms-txt.rb
|
|
- name: Build tracked Python packages
|
|
run: |
|
|
wheel_dir=$(mktemp -d)
|
|
while IFS= read -r pyproject; do
|
|
python3 -m pip wheel --no-deps "./$(dirname "$pyproject")" --wheel-dir "$wheel_dir"
|
|
done < <(git ls-files -- ':(glob)**/pyproject.toml')
|