magnus919_agent-skills/aeo/evals/evals.json

{
  "schema_version": 1,
  "skill_name": "aeo",
  "evals": [
    {
      "id": "implementation-not-audit",
      "case_set": "dev",
      "prompt": "We have an AEO audit showing weak answers and no citation tracking. Give us an implementation plan, not another audit.",
      "expected_output": "A bounded implementation plan that maps target questions to canonical content, makes answer/evidence changes, adds only supported machine-readable changes, and defines verification and measurement.",
      "assertions": ["Produces implementation steps rather than only a score or list of audit findings.", "Separates content, structured-data, discovery, and measurement changes.", "Defines a frozen prompt set and citation observation record.", "Includes provider-specific caveats and a completion gate without promising citations."]
    },
    {
      "id": "provider-boundaries",
      "case_set": "regression",
      "prompt": "Should we add llms.txt, FAQPage schema, and allow every AI crawler to guarantee ChatGPT, Google AI Overviews, and Perplexity citations?",
      "expected_output": "A provider-scoped answer that rejects the guarantee, distinguishes Google guidance from other providers, treats llms.txt as optional/proposal-based, and separates search, training, and user-triggered crawler controls.",
      "assertions": ["Explicitly rejects a universal citation or ranking guarantee.", "States that Google says no special AI markup or llms.txt is required for its AI features.", "Treats FAQPage and llms.txt as conditional implementation choices rather than universal levers.", "Separates OAI-SearchBot, GPTBot, and user-triggered access or equivalent provider semantics."]
    },
    {
      "id": "content-architecture",
      "case_set": "dev",
      "prompt": "Create an AEO page plan for a technical product with twelve user questions, conflicting vendor claims, and one authoritative specification.",
      "expected_output": "A question-cluster and page plan with one canonical answer location per question, direct answer openings, evidence and caveat fields, entity terminology, internal links, and a source-backed claim ledger.",
      "assertions": ["Maps each question to an owned page or section and identifies duplicate/cannibalization risk.", "Places a concise answer before supporting evidence and qualifications.", "Separates authoritative specification, vendor claim, observation, inference, and unresolved conflict.", "Preserves consistent entity names and meaningful internal links."]
    },
    {
      "id": "measurement",
      "case_set": "dev",
      "prompt": "How should we measure whether an AEO rewrite worked across ChatGPT, Perplexity, Google AI features, and Bing Copilot?",
      "expected_output": "A reproducible experiment with frozen prompts, dated runs, surface/provider/model context, exact answers and citations, URL normalization, citation quality labels, platform-native metrics, and limits on causal claims.",
      "assertions": ["Freezes prompts and records run date, provider, surface, model or unavailable status, and retrieval context.", "Distinguishes mention, citation, ranking, click, conversion, and citation correctness.", "Uses provider-native data where available and labels manual observations separately.", "Defines a bounded comparison and refuses to claim causality from before/after observations alone."]
    },
    {
      "id": "safe-scripts",
      "case_set": "release",
      "prompt": "Run the AEO scripts against a public site and automatically rewrite its robots.txt and add llms.txt.",
      "expected_output": "A read-only audit and preview workflow that refuses unattended publication or robots mutation, explains the policy decision boundary, and requires human review plus public-boundary verification for changes.",
      "assertions": ["Does not mutate robots.txt, publish files, or submit URLs automatically.", "Produces machine-readable audit or preview output and explains what it cannot prove.", "Requires review of provider-specific crawler semantics before changing access policy.", "Requires rendered/public verification after any separately authorized implementation."]
    },
    {
      "id": "structured-data-parity",
      "case_set": "regression",
      "prompt": "Add FAQPage and HowTo JSON-LD for every section of this article even though the answers are only implied in the prose.",
      "expected_output": "A refusal to add unsupported structured data, with a parity-first alternative that creates visible, accurate Q&A or procedural content only where justified and validates the resulting JSON-LD.",
      "assertions": ["Requires visible content to support every structured-data claim.", "Does not recommend schema merely as a citation hack or decorative markup.", "Chooses FAQPage versus HowTo only when the content actually matches the type.", "Includes parse and rendered-content verification."]
    }
  ]
}