magnus919_agent-skills/.github/workflows/skill-eval.yml

name: Paired skill evaluation smoke

on:
  pull_request:
    paths:
      - '*/SKILL.md'
      - '*/evals/evals.json'
      - '*/scripts/**'
      - 'eval_runner/**'
      - '.github/workflows/skill-eval.yml'
      - 'schemas/comparison-report-v1.schema.json'
      - 'schemas/release-eval-v1.schema.json'
  push:
    branches:
      - main
    paths:
      - '*/SKILL.md'
      - '*/evals/evals.json'
      - '*/scripts/**'
      - 'eval_runner/**'
      - '.github/workflows/skill-eval.yml'
      - 'schemas/comparison-report-v1.schema.json'
      - 'schemas/release-eval-v1.schema.json'

permissions:
  contents: read

jobs:
  paired-eval-tests:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.12'
      - name: Install dependencies
        run: python3 -m pip install -r requirements-dev.txt
      - name: Run paired evaluation unit tests
        run: python3 eval_runner/tests/test_paired.py
      - name: Run release evaluation unit tests
        run: python3 eval_runner/tests/test_release.py
      - name: Run existing eval runner tests
        run: python3 eval_runner/tests/test_runner.py

  paired-eval-smoke:
    runs-on: ubuntu-latest
    needs: paired-eval-tests
    steps:
      - name: Check out repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.12'
      - name: Install dependencies
        run: python3 -m pip install -r requirements-dev.txt
      - name: Detect changed skills with evals
        id: changed
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
        run: |
          manifests=$(
            git diff --name-only "$BASE_SHA" HEAD -- \
              '*/SKILL.md' '*/evals/evals.json' '*/scripts/**' \
              | awk -F/ 'NF > 1 { print $1 }' \
              | sort -u \
              | while read -r skill; do
                  if [ -f "$skill/evals/evals.json" ]; then
                    printf '%s\n' "$skill/evals/evals.json"
                  fi
                done \
              | head -5 \
              | paste -sd' ' -
          )
          echo "manifests=$manifests" >> "$GITHUB_OUTPUT"
      - name: Run paired evaluation (fake adapter)
        if: steps.changed.outputs.manifests != ''
        env:
          MANIFESTS: ${{ steps.changed.outputs.manifests }}
        run: |
          exit_code=0
          for manifest in $MANIFESTS; do
            echo "=== Paired eval: $manifest ==="
            python3 -m eval_runner.paired "$manifest" \
              --adapter fake \
              --output-dir "eval-output-paired/$(dirname "$(dirname "$manifest")" | xargs basename)" \
              || exit_code=1
          done
          exit $exit_code
      - name: Upload paired eval artifacts
        if: always() && steps.changed.outputs.manifests != ''
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: paired-eval-artifacts
          path: eval-output-paired/
          retention-days: 7

  paired-eval-model:
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    runs-on: [self-hosted, linux, agent-skills-eval]
    needs: paired-eval-tests
    steps:
      - name: Check out repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.12'
      - name: Install dependencies
        run: python3 -m pip install -r requirements-dev.txt
      - name: Check model endpoint
        id: endpoint
        env:
          EVAL_BASE_URL: ${{ vars.EVAL_BASE_URL }}
          EVAL_MODEL: ${{ vars.EVAL_MODEL }}
        run: |
          if [ -z "$EVAL_BASE_URL" ] || [ -z "$EVAL_MODEL" ]; then
            echo "available=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          if curl -sf --connect-timeout 5 "$EVAL_BASE_URL/v1/models" > /dev/null 2>&1; then
            echo "available=true" >> "$GITHUB_OUTPUT"
          else
            echo "available=false" >> "$GITHUB_OUTPUT"
          fi
      - name: Detect changed skills with evals
        id: changed
        if: steps.endpoint.outputs.available == 'true'
        env:
          BASE_SHA: ${{ github.event.before }}
        run: |
          manifests=$(
            git diff --name-only "$BASE_SHA" HEAD -- \
              '*/SKILL.md' '*/evals/evals.json' '*/scripts/**' \
              | awk -F/ 'NF > 1 { print $1 }' \
              | sort -u \
              | while read -r skill; do
                  if [ -f "$skill/evals/evals.json" ]; then
                    printf '%s\n' "$skill/evals/evals.json"
                  fi
                done \
              | head -5 \
              | paste -sd' ' -
          )
          echo "manifests=$manifests" >> "$GITHUB_OUTPUT"
      - name: Run paired evaluation (real model)
        if: steps.endpoint.outputs.available == 'true' && steps.changed.outputs.manifests != ''
        env:
          MANIFESTS: ${{ steps.changed.outputs.manifests }}
          EVAL_MODEL: ${{ vars.EVAL_MODEL }}
          EVAL_BASE_URL: ${{ vars.EVAL_BASE_URL }}
        run: |
          exit_code=0
          for manifest in $MANIFESTS; do
            skill=$(dirname "$(dirname "$manifest")" | xargs basename)
            echo "=== Paired eval (model): $skill ==="
            python3 -m eval_runner.paired "$manifest" \
              --adapter openai \
              --base-url "$EVAL_BASE_URL" \
              --model "$EVAL_MODEL" \
              --model-label configured-model \
              --max-tokens 4096 \
              --no-thinking \
              --timeout 300 \
              --output-dir "eval-output-model/$skill" \
              || exit_code=1
          done
          exit $exit_code
      - name: Upload model eval artifacts
        if: always() && steps.endpoint.outputs.available == 'true'
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: paired-eval-model-artifacts
          path: eval-output-model/
          retention-days: 14