#!/usr/bin/env ruby # frozen_string_literal: true require "yaml" require "set" require "json" require "open3" require_relative "validate-references" ROOT = File.expand_path("..", __dir__) ALLOWED_FIELDS = %w[name description license compatibility metadata allowed-tools].freeze README_HEADINGS = ["Why Install This Skill", "What You Get", "Quick Start", "Triggers", "Requirements"].freeze GRANDFATHER_FILE = File.join(ROOT, "scripts", "grandfathered-skills.txt") MIN_EVAL_CASES = 5 errors = [] eval_stdout, eval_stderr, eval_status = Open3.capture3("python3", File.join(ROOT, "scripts", "validate-evals.py"), chdir: ROOT) errors.concat(eval_stderr.lines(chomp: true)) unless eval_status.success? skills = Dir.glob("#{ROOT}/**/SKILL.md").sort.reject do |skill| skill.include?("/agent-council/profiles/skills/") end catalog_entries = File.read(File.join(ROOT, "README.md")).scan( /^### \[([a-z0-9]+(?:-[a-z0-9]+)*)\]\(([^)]+\/SKILL\.md)\)\s*$/ ) catalog_names = catalog_entries.map(&:first) expected_catalog_names = catalog_names.sort_by { |name| [name.downcase, name] } unless catalog_names == expected_catalog_names mismatch = catalog_names.zip(expected_catalog_names).index { |actual, expected| actual != expected } errors << "README.md: skill catalog headings must be sorted case-insensitively; " \ "position #{mismatch + 1} is #{catalog_names[mismatch].inspect}, " \ "expected #{expected_catalog_names[mismatch].inspect}" end catalog_paths = catalog_entries.map(&:last).sort expected_catalog_paths = Dir.glob("#{ROOT}/*/SKILL.md") .map { |path| path.delete_prefix("#{ROOT}/") } .sort missing_catalog_paths = expected_catalog_paths - catalog_paths unexpected_catalog_paths = catalog_paths - expected_catalog_paths duplicate_catalog_names = catalog_names.group_by { |name| name }.select { |_name, values| values.length > 1 }.keys.sort duplicate_catalog_paths = catalog_paths.group_by { |path| path }.select { |_path, values| values.length > 1 }.keys.sort mislabeled_catalog_entries = catalog_entries.reject do |name, path| name == File.basename(File.dirname(path)) end errors << "README.md: missing catalog path(s): #{missing_catalog_paths.join(', ')}" unless missing_catalog_paths.empty? errors << "README.md: unexpected catalog path(s): #{unexpected_catalog_paths.join(', ')}" unless unexpected_catalog_paths.empty? errors << "README.md: duplicate catalog name(s): #{duplicate_catalog_names.join(', ')}" unless duplicate_catalog_names.empty? errors << "README.md: duplicate catalog path(s): #{duplicate_catalog_paths.join(', ')}" unless duplicate_catalog_paths.empty? unless mislabeled_catalog_entries.empty? labels = mislabeled_catalog_entries.map { |name, path| "#{name.inspect} -> #{path}" } errors << "README.md: catalog label/path mismatch(es): #{labels.join(', ')}" end grandfathered = File.exist?(GRANDFATHER_FILE) ? File.readlines(GRANDFATHER_FILE, chomp: true).reject { |l| l.strip.empty? || l.start_with?("#") }.to_set : Set.new skills.each do |skill| relative = skill.delete_prefix("#{ROOT}/") text = File.read(skill) match = text.match(/\A---\n(.*?)\n---\n/m) unless match errors << "#{relative}: missing YAML frontmatter" next end begin data = YAML.safe_load(match[1], permitted_classes: [], aliases: false) rescue Psych::Exception => e errors << "#{relative}: invalid YAML: #{e.message.lines.first.strip}" next end unless data.is_a?(Hash) errors << "#{relative}: frontmatter must be a mapping" next end name = data["name"] expected_name = File.basename(File.dirname(skill)) errors << "#{relative}: invalid name" unless name.is_a?(String) && name.match?(/\A[a-z0-9]+(?:-[a-z0-9]+)*\z/) && name.length.between?(1, 64) && name == expected_name description = data["description"] errors << "#{relative}: invalid description" unless description.is_a?(String) && description.length.between?(1, 1024) errors << "#{relative}: unsupported field(s): #{(data.keys - ALLOWED_FIELDS).join(', ')}" unless (data.keys - ALLOWED_FIELDS).empty? if data.key?("compatibility") && (!data["compatibility"].is_a?(String) || !data["compatibility"].length.between?(1, 500)) errors << "#{relative}: invalid compatibility" end if data.key?("metadata") && (!data["metadata"].is_a?(Hash) || !data["metadata"].all? { |key, value| key.is_a?(String) && value.is_a?(String) }) errors << "#{relative}: metadata must map strings to strings" end errors << "#{relative}: metadata.hermes is client-specific and not allowed" if data.dig("metadata", "hermes") errors << "#{relative}: must stay under 500 lines" unless text.lines.length < 500 root = File.dirname(skill) text.scan(/\[[^\]]+\]\(([^)]+)\)/).flatten.each do |target| path = target.split("#", 2).first next if path.empty? || path.match?(%r{\A(?:https?:|mailto:|/)}) errors << "#{relative}: missing linked resource #{path}" unless File.exist?(File.expand_path(path, root)) end readme = File.join(root, "README.md") unless File.file?(readme) errors << "#{relative}: missing README.md" next end readme_text = File.read(readme) README_HEADINGS.each do |heading| errors << "#{relative}: README missing #{heading}" unless readme_text.match?(/^#+ #{Regexp.escape(heading)}\s*$/) end # Phase 1: new skills (not grandfathered) must have evals with >= MIN_EVAL_CASES cases skill_dir = File.dirname(skill).delete_prefix("#{ROOT}/") unless grandfathered.include?(skill_dir) evals_path = File.join(root, "evals", "evals.json") unless File.file?(evals_path) errors << "#{relative}: new skill must have evals/evals.json (not in grandfathered-skills.txt)" next end begin evals_data = JSON.parse(File.read(evals_path)) case_count = evals_data.is_a?(Hash) && evals_data["evals"].is_a?(Array) ? evals_data["evals"].length : 0 if case_count < MIN_EVAL_CASES errors << "#{relative}: evals/evals.json has #{case_count} case(s), minimum is #{MIN_EVAL_CASES}" end rescue JSON::ParserError => e errors << "#{relative}: evals/evals.json is invalid JSON: #{e.message.lines.first.strip}" end end # Phase 1: prose backtick references in references/*.md must resolve to a # real file (catches stale framework-file references like the RICE typo). errors.concat(ReferenceFileScan.stale_reference_errors(ROOT, skill_dir)) # Phase 1: references/*.md files must stay within the per-file character cap # (oversized files are split into focused files and re-indexed in SKILL.md). errors.concat(ReferenceFileScan.oversized_reference_errors(ROOT, skill_dir)) # Issue #382: SKILL.md bodies (frontmatter excluded) must stay within the # ~5,000-token character budget; dense prose moves into references/ files. errors.concat(ReferenceFileScan.oversized_skill_md_errors(ROOT, skill_dir)) end if errors.empty? puts "Validated #{skills.length} canonical skills." else warn errors.join("\n") exit 1 end