pbakaus_impeccable/tests/skill-behavior/scenarios.test.mjs

/**
 * Skill-behavior scenarios — verify how the agent loads PRODUCT.md / DESIGN.md
 * across a controlled matrix of starting states.
 *
 * Refactors that touch the Setup section of SKILL.md should keep these
 * assertions green. If you change Setup intentionally and the assertions
 * flip, that's the test catching the regression you wanted to catch.
 *
 * Run with:  bun run test:skill-behavior
 *
 * Skips per-provider when its API key is unset. The default model lineup is
 * the cheapest tier of each major provider so a full sweep costs a few cents.
 */
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import path from 'node:path';

import {
  prepareWorkspace,
  cleanupWorkspace,
  runTurn,
  bashCommandsMatching,
  readsMatching,
  fileLoaded,
  summarizeTrace,
} from './harness.mjs';
import { detectProvider, getModel, hasKey, resolveModelList, PROVIDERS } from './providers.mjs';
import {
  PRODUCT_MD_SAMPLE,
  PRODUCT_MD_SAMPLE_NO_REGISTER,
  PRODUCT_MD_SAMPLE_IOS,
  MINIMAL_IOS_SOURCE,
  DESIGN_MD_SAMPLE,
  MINIMAL_LANDING_HTML,
  SVELTE_PROJECT_FILES,
} from './fixtures.mjs';

const CRAFT_PROMPT = '/impeccable craft a landing page for the project in this workspace';
const SHAPE_PROMPT = '/impeccable shape a landing page for the project in this workspace';
const NATURAL_BUILD_PROMPT = 'Build a landing page for the project in this workspace.';
const TEACH_PROMPT = '/impeccable teach';
const PRIMER_PROMPT =
  'Take a quick look at the project. What context should guide later design work? Run the impeccable context loader once if you need to.';

const VERBOSE = process.env.IMPECCABLE_SKILL_BEHAVIOR_VERBOSE === '1';

function logTrace(label, scenario, model, trace, extras = {}) {
  if (!VERBOSE) return;
  const summary = summarizeTrace(trace);
  console.error(
    `\n[${label}] ${scenario} (${model})\n${JSON.stringify({ ...summary, ...extras }, null, 2)}\n`,
  );
}

function loadedBeforeImplementationWrite(trace, filename) {
  const needle = filename.toLowerCase();
  const loadIndex = trace.toolCalls.findIndex(({ name, input }) => {
    if (name === 'read') return input?.path?.toLowerCase().includes(needle);
    if (name === 'bash') return input?.command?.toLowerCase().includes(needle);
    return false;
  });
  const writeIndex = trace.toolCalls.findIndex(
    ({ name, input }) => name === 'write' && /\.(html?|css|svelte|jsx?|tsx?)$/i.test(input?.path ?? ''),
  );
  return loadIndex >= 0 && (writeIndex < 0 || loadIndex < writeIndex);
}

function executedUpdateCommands(trace) {
  const executableSegments = trace.bashCommands.flatMap((command) =>
    command
      .split(/\r?\n|&&|\|\||;|\|/)
      .map((segment) => segment.trim())
      .filter((segment) => segment && !/^(?:#|echo\b|printf\b)/.test(segment)),
  );
  return executableSegments.filter((segment) =>
    /^(?:(?:npx|bunx|pnpx)\s+)?(?:impeccable|skills)\s+update\b/.test(segment),
  );
}

for (const modelId of resolveModelList()) {
  const provider = detectProvider(modelId);
  const keyPresent = hasKey(provider);

  describe(`skill behavior :: ${modelId}`, () => {
    if (!keyPresent) {
      it(`skipped — ${PROVIDERS[provider].envKey} is unset`, { skip: true }, () => {});
      return;
    }
    const model = getModel(modelId);
    // Gemini Flash tends to inspect one file at a time, while the production
    // Anthropic/OpenAI models batch setup reads and then begin implementation.
    // Keep the latter tightly bounded so this routing suite does not turn into
    // a page-generation benchmark, but leave Gemini enough room to reach the
    // same required reference.
    const setupMaxSteps = provider === 'google' ? 6 : 3;

    it('scenario 1: no PRODUCT.md / DESIGN.md', async () => {
      const workspace = prepareWorkspace({ files: {} });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: CRAFT_PROMPT,
          maxSteps: setupMaxSteps,
        });
        logTrace('S1', 'no-context', modelId, trace, { textSample: text.slice(0, 400) });
        const loadCalls = bashCommandsMatching(trace, 'context.mjs');
        assert.ok(
          loadCalls.length >= 1,
          `expected agent to run context.mjs at least once; got ${loadCalls.length}.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
        assert.ok(
          fileLoaded(trace, 'init.md'),
          `craft should load init.md when no product or visual world exists; an automated harness is not a bypass.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
        assert.ok(
          loadedBeforeImplementationWrite(trace, 'init.md'),
          `agent should resolve init before writing implementation files.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 2: PRODUCT.md only', async () => {
      const workspace = prepareWorkspace({
        files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE },
      });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: CRAFT_PROMPT,
          maxSteps: setupMaxSteps,
        });
        logTrace('S2', 'product-only', modelId, trace, { textSample: text.slice(0, 400) });
        const loadCalls = bashCommandsMatching(trace, 'context.mjs');
        assert.ok(
          loadCalls.length >= 1 && loadCalls.length <= 3,
          `expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` +
            `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
        );
        assert.ok(
          fileLoaded(trace, 'new-work.md'),
          `a greenfield request with PRODUCT.md should load new-work.md to establish the missing visual world.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 3: PRODUCT.md + DESIGN.md', async () => {
      const workspace = prepareWorkspace({
        files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE, 'DESIGN.md': DESIGN_MD_SAMPLE },
      });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: CRAFT_PROMPT,
          maxSteps: setupMaxSteps,
        });
        logTrace('S3', 'product-and-design', modelId, trace, { textSample: text.slice(0, 400) });
        const loadCalls = bashCommandsMatching(trace, 'context.mjs');
        assert.ok(
          loadCalls.length >= 1 && loadCalls.length <= 3,
          `expected 1-3 context.mjs invocations; got ${loadCalls.length}.\n` +
            `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
        );
        assert.ok(
          fileLoaded(trace, 'new-work.md'),
          `craft inside a committed PRODUCT.md + DESIGN.md world should load new-work.md for the task-specific concept.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
        // The skill tells the agent to also familiarize with the existing
        // design system. DESIGN.md is bundled in context.mjs output, but
        // exploring CSS / tokens / theme files or a directory listing
        // also counts.
        const designSignal =
          readsMatching(trace, 'design.md').length > 0 ||
          trace.bashOutputs.some((output) => output.includes('# DESIGN.md')) ||
          trace.readPaths.some((p) => /\.(css|scss|sass|less|ts|tsx|js|jsx|json|svelte|astro)$/i.test(p)) ||
          trace.listPaths.length > 0;
        assert.ok(
          designSignal,
          `agent should consult the design system (DESIGN.md, CSS/tokens, or list project files).\n` +
            `readPaths: ${JSON.stringify(trace.readPaths)}, listPaths: ${JSON.stringify(trace.listPaths)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 4: context already loaded in prior turn', async () => {
      const workspace = prepareWorkspace({
        files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE, 'DESIGN.md': DESIGN_MD_SAMPLE },
      });
      try {
        // Turn 1: prime the conversation so context.mjs gets run and its
        // output enters the message history.
        const turn1 = await runTurn({
          workspace,
          model,
          userPrompt: PRIMER_PROMPT,
          maxSteps: setupMaxSteps,
        });
        logTrace('S4-T1', 'primer', modelId, turn1.trace, { textSample: turn1.text.slice(0, 200) });
        const turn1Loads = bashCommandsMatching(turn1.trace, 'context.mjs');
        assert.ok(
          turn1Loads.length >= 1,
          `primer turn should have run context.mjs. bash: ${JSON.stringify(turn1.trace.bashCommands, null, 2)}`,
        );

        // Turn 2: the real ask. The skill says "skip if you've already
        // loaded it". Verify the agent honors that.
        const turn2 = await runTurn({
          workspace,
          model,
          userPrompt: 'Now, /impeccable craft a landing page based on what you saw.',
          priorMessages: turn1.responseMessages,
          maxSteps: setupMaxSteps,
        });
        logTrace('S4-T2', 'follow-up', modelId, turn2.trace, { textSample: turn2.text.slice(0, 400) });
        const turn2Loads = bashCommandsMatching(turn2.trace, 'context.mjs');
        assert.equal(
          turn2Loads.length,
          0,
          `agent re-ran context.mjs on turn 2 despite it being in prior conversation. ` +
            `bashCommands: ${JSON.stringify(turn2.trace.bashCommands, null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 5: legacy PRODUCT.md enters new-work when DESIGN.md is missing', async () => {
      const workspace = prepareWorkspace({
        files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_NO_REGISTER },
      });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: CRAFT_PROMPT,
          maxSteps: setupMaxSteps,
        });
        logTrace('S5', 'legacy-product', modelId, trace, { textSample: text.slice(0, 400) });
        const loadCalls = bashCommandsMatching(trace, 'context.mjs');
        assert.ok(
          loadCalls.length >= 1,
          `expected context.mjs invocation; got ${loadCalls.length}.\n` +
            `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
        );
        assert.ok(fileLoaded(trace, 'new-work.md'),
          `greenfield craft should load new-work.md for visual authority and world discovery.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`);
        assert.equal(fileLoaded(trace, 'init.md'), false, 'existing PRODUCT.md must not re-enter init for missing DESIGN.md');
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 6: sub-command routing (`/impeccable polish` loads polish.md)', async () => {
      const workspace = prepareWorkspace({
        files: {
          'PRODUCT.md': PRODUCT_MD_SAMPLE,
          'DESIGN.md': DESIGN_MD_SAMPLE,
          'index.html': MINIMAL_LANDING_HTML,
        },
      });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: '/impeccable polish index.html',
          maxSteps: setupMaxSteps,
        });
        logTrace('S6', 'polish-routing', modelId, trace, { textSample: text.slice(0, 300) });
        assert.ok(
          fileLoaded(trace, 'polish.md'),
          `agent should load polish.md when /impeccable polish is invoked.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 7: sub-command routing (`/impeccable audit` loads audit.md)', async () => {
      const workspace = prepareWorkspace({
        files: {
          'PRODUCT.md': PRODUCT_MD_SAMPLE,
          'DESIGN.md': DESIGN_MD_SAMPLE,
          'index.html': MINIMAL_LANDING_HTML,
        },
      });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: '/impeccable audit index.html',
          maxSteps: setupMaxSteps,
        });
        logTrace('S7', 'audit-routing', modelId, trace, { textSample: text.slice(0, 300) });
        assert.ok(
          fileLoaded(trace, 'audit.md'),
          `agent should load audit.md when /impeccable audit is invoked.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 8: existing SvelteKit project (agent explores design system)', async () => {
      const workspace = prepareWorkspace({
        files: {
          'PRODUCT.md': PRODUCT_MD_SAMPLE,
          'DESIGN.md': DESIGN_MD_SAMPLE,
          ...SVELTE_PROJECT_FILES,
        },
      });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: '/impeccable polish src/routes/+page.svelte',
          maxSteps: 8,
        });
        logTrace('S8', 'existing-project', modelId, trace, { textSample: text.slice(0, 400) });
        // Setup step 2: familiarize with existing design system. The
        // agent should read at least one project code file (CSS / tokens /
        // component / page), not just the skill's PRODUCT.md / DESIGN.md
        // / reference files.
        const projectReads = trace.readPaths.filter((p) =>
          /\.(css|svelte|tsx?|jsx?|astro)$/i.test(p) && !p.includes('.claude/skills/'),
        );
        assert.ok(
          projectReads.length >= 1,
          `agent should read at least one project code file to understand the existing design system.\n` +
            `readPaths: ${JSON.stringify(trace.readPaths, null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 9: update-available directive is surfaced, never auto-run', async () => {
      // context.mjs reads a newer version from its (seeded) cache and appends
      // an UPDATE_AVAILABLE directive to the boot output. The agent must
      // surface it and keep working, but must NOT run `npx impeccable update`
      // on its own — modifying installed files mid-session without
      // consent is the exact failure this guards against.
      //
      // `skillVersion` forces copy-mode so context.mjs has a SKILL.md sibling
      // to read its own version from; the seeded cache (fresh lastCheck) means
      // no network call happens.
      const workspace = prepareWorkspace({
        files: {
          'PRODUCT.md': PRODUCT_MD_SAMPLE,
          'index.html': MINIMAL_LANDING_HTML,
          '.impeccable-update.json': JSON.stringify({ lastCheck: Date.now(), latestVersion: '99.0.0' }),
        },
        skillVersion: '3.5.0',
      });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: '/impeccable polish index.html',
          maxSteps: setupMaxSteps,
          env: { IMPECCABLE_UPDATE_CACHE: path.join(workspace, '.impeccable-update.json') },
        });
        logTrace('S9', 'update-available', modelId, trace, { textSample: text.slice(0, 400) });

        // Boot ran, so the directive entered the agent's view.
        assert.ok(
          bashCommandsMatching(trace, 'context.mjs').length >= 1,
          `expected agent to run context.mjs. bash: ${JSON.stringify(trace.bashCommands, null, 2)}`,
        );
        // Setup sanity + proof the agent actually received the directive:
        // the boot output it read carried UPDATE_AVAILABLE.
        assert.ok(
          trace.bashOutputs.some((o) => o.includes('UPDATE_AVAILABLE')),
          `context.mjs should have emitted UPDATE_AVAILABLE (a newer version is cached).\n` +
            `bashOutputs: ${JSON.stringify(trace.bashOutputs, null, 2)}`,
        );
        // The core property: ask first, never auto-run the update.
        const ranUpdate = executedUpdateCommands(trace);
        assert.equal(
          ranUpdate.length,
          0,
          `agent auto-ran the skill update without asking the user first: ${JSON.stringify(ranUpdate, null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 10: scoped command with no PRODUCT.md proceeds without forcing init', async () => {
      // The counterpart to scenario 1. There, a from-scratch `craft` with no
      // context correctly diverts into init. Here a *scoped* command against
      // existing code must NOT: the code is the context. Missing PRODUCT.md is
      // a suggestion to run init, never a blocker on the requested work.
      const workspace = prepareWorkspace({
        files: {
          'index.html': MINIMAL_LANDING_HTML,
        },
      });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: '/impeccable polish index.html',
          maxSteps: setupMaxSteps,
        });
        logTrace('S10', 'scoped-no-product', modelId, trace, { textSample: text.slice(0, 400) });
        // Boot still runs.
        assert.ok(
          bashCommandsMatching(trace, 'context.mjs').length >= 1,
          `expected agent to run context.mjs at least once.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
        // It must load the scoped command's own reference and get on with it.
        assert.ok(
          fileLoaded(trace, 'polish.md'),
          `agent should load polish.md and proceed with the scoped command.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
        // The core property: a scoped command on existing code must not divert
        // into init just because PRODUCT.md is absent.
        const initLoaded =
          readsMatching(trace, 'init.md').length > 0 ||
          bashCommandsMatching(trace, 'init.md').length > 0;
        assert.equal(
          initLoaded,
          false,
          `scoped /impeccable polish on existing code should not divert into init.md when PRODUCT.md is missing.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 11: shape with no PRODUCT.md resolves the build gate', async () => {
      const workspace = prepareWorkspace({ files: {} });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: SHAPE_PROMPT,
          maxSteps: setupMaxSteps,
        });
        logTrace('S11', 'shape-no-context', modelId, trace, { textSample: text.slice(0, 400) });
        assert.ok(
          bashCommandsMatching(trace, 'context.mjs').length >= 1,
          `expected agent to run context.mjs at least once.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
        assert.ok(
          loadedBeforeImplementationWrite(trace, 'init.md'),
          `shape should resolve init.md before implementation when no world exists.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 12: intent-routed build with no PRODUCT.md resolves the build gate', async () => {
      const workspace = prepareWorkspace({ files: {} });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: NATURAL_BUILD_PROMPT,
          maxSteps: setupMaxSteps,
        });
        logTrace('S12', 'natural-build-no-context', modelId, trace, { textSample: text.slice(0, 400) });
        assert.ok(
          bashCommandsMatching(trace, 'context.mjs').length >= 1,
          `expected agent to run context.mjs at least once.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
        assert.ok(
          loadedBeforeImplementationWrite(trace, 'init.md'),
          `build intent should resolve init.md before implementation when no world exists.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 13: teach alias with no PRODUCT.md diverts into init', async () => {
      // `teach` is a deprecated alias for `init`, so it belongs to the same
      // missing-PRODUCT.md blocker path instead of the scoped-command path.
      const workspace = prepareWorkspace({ files: {} });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: TEACH_PROMPT,
          maxSteps: 6,
        });
        logTrace('S13', 'teach-no-context', modelId, trace, { textSample: text.slice(0, 400) });
        assert.ok(
          bashCommandsMatching(trace, 'context.mjs').length >= 1,
          `expected agent to run context.mjs at least once.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
        const initLoaded =
          readsMatching(trace, 'init.md').length > 0 ||
          bashCommandsMatching(trace, 'init.md').length > 0;
        assert.ok(
          initLoaded,
          `/impeccable teach should behave like init and load init.md when PRODUCT.md is missing.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 14: native iOS project (context loads ios.md)', async () => {
      // PRODUCT.md sets `## Platform` to `ios`. context.mjs now reads and emits
      // reference/ios.md itself, so native guidance enters the conversation
      // without relying on a second model-directed file read.
      const workspace = prepareWorkspace({
        files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_IOS, 'TideDetailView.swift': MINIMAL_IOS_SOURCE },
      });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: '/impeccable craft a tide detail screen for the project in this workspace',
          maxSteps: provider === 'google' ? 8 : 6,
        });
        logTrace('S14', 'native-ios', modelId, trace, { textSample: text.slice(0, 400) });
        const loadCalls = bashCommandsMatching(trace, 'context.mjs');
        assert.ok(
          loadCalls.length >= 1,
          `expected agent to run context.mjs at least once; got ${loadCalls.length}.\n` +
            `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
        );
        // Proof the native reference itself entered the agent's view.
        assert.ok(
          trace.bashOutputs.some((o) => /# NATIVE PLATFORM REFERENCE: IOS \(reference\/ios\.md\)/.test(o)),
          `context.mjs should have emitted reference/ios.md content (platform is ios).\n` +
            `bashOutputs: ${JSON.stringify(trace.bashOutputs, null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });

    it('scenario 15: native audit routes to the native command variant', async () => {
      // The Commands table lists audit.native.md as the native variant and
      // Setup step 2 says to read the variant INSTEAD of audit.md when the
      // platform is native. This pins the route-instead behavior: a native
      // audit must reach audit.native.md (reading audit.md first and then
      // switching via its web-only guard is acceptable; never reaching the
      // variant is the failure).
      const workspace = prepareWorkspace({
        files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_IOS, 'TideDetailView.swift': MINIMAL_IOS_SOURCE },
      });
      try {
        const { trace, text } = await runTurn({
          workspace,
          model,
          userPrompt: '/impeccable audit the app in this workspace',
          maxSteps: 6,
        });
        logTrace('S15', 'native-audit-variant', modelId, trace, { textSample: text.slice(0, 400) });
        assert.ok(
          bashCommandsMatching(trace, 'context.mjs').length >= 1,
          `expected agent to run context.mjs at least once.\n` +
            `bashCommands: ${JSON.stringify(trace.bashCommands, null, 2)}`,
        );
        assert.ok(
          fileLoaded(trace, 'audit.native.md'),
          `agent should load audit.native.md (not just audit.md) when the platform is ios.\n` +
            `Trace: ${JSON.stringify(summarizeTrace(trace), null, 2)}`,
        );
      } finally {
        cleanupWorkspace(workspace);
      }
    });
  });
}