Improve critique skill reliability

- add provider-specific block compilation and tests

- bundle detector scripts for skill critique runs

- harden critique orchestration, browser handling, and storage
This commit is contained in:
Paul Bakaus
2026-05-18 15:15:14 -07:00
parent e1d3ea0b6f
commit bc1894889e
285 changed files with 142401 additions and 1251 deletions
+15 -3
View File
@@ -5,8 +5,8 @@
import { describe, it, beforeEach, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { mkdtempSync, rmSync, symlinkSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
@@ -173,6 +173,19 @@ describe('CLI entry point', () => {
assert.match(r.stderr, /no stable slug/);
});
it('runs when invoked through a symlinked harness path', () => {
const linkedScript = join(cwd, 'linked-critique-storage.mjs');
symlinkSync(SCRIPT, linkedScript);
const r = spawnSync(process.execPath, [linkedScript, 'slug', 'index.html'], {
cwd,
encoding: 'utf-8',
});
assert.equal(r.status, 0, `stderr: ${r.stderr}`);
assert.equal(r.stdout.trim(), 'index-html');
});
it('latest subcommand exits 2 when no snapshot exists', () => {
const r = spawnSync(process.execPath, [SCRIPT, 'latest', 'never-written'], {
cwd,
@@ -204,4 +217,3 @@ describe('readTrend', () => {
assert.deepEqual(readTrend('nope', { cwd }), []);
});
});
+27
View File
@@ -0,0 +1,27 @@
import { describe, expect, test } from 'bun:test';
import fs from 'fs';
import path from 'path';
import { readSourceFiles } from '../../scripts/lib/utils.js';
const ROOT = process.cwd();
describe('skill detector bundle', () => {
test('adds the detector wrapper and engine files to skill scripts', () => {
const { skills } = readSourceFiles(ROOT);
const skill = skills.find(s => s.name === 'impeccable');
const scriptNames = new Set(skill.scripts.map(s => s.name));
expect(scriptNames.has('detect.mjs')).toBe(true);
expect(scriptNames.has('detector/detect-antipatterns.mjs')).toBe(true);
expect(scriptNames.has('detector/detect-antipatterns-browser.js')).toBe(true);
expect(scriptNames.has('detector/cli/main.mjs')).toBe(true);
expect(scriptNames.has('detector/engines/static-html/detect-html.mjs')).toBe(true);
});
test('critique references the bundled detector command', () => {
const critique = fs.readFileSync(path.join(ROOT, 'skill/reference/critique.md'), 'utf-8');
expect(critique).toContain('node {{scripts_path}}/detect.mjs --json [--fast] [target]');
expect(critique).not.toContain('npx impeccable detect');
});
});
+86
View File
@@ -0,0 +1,86 @@
import { describe, expect, test } from 'bun:test';
import { PROVIDERS } from '../../scripts/lib/transformers/providers.js';
import { compileProviderBlocks, PROVIDER_BLOCK_TAGS } from '../../scripts/lib/utils.js';
describe('compileProviderBlocks', () => {
test('keeps matching provider block bodies and removes tags', () => {
const content = [
'Before',
'<codex>',
'Codex-only guidance.',
'</codex>',
'After',
].join('\n');
expect(compileProviderBlocks(content, ['codex'])).toBe([
'Before',
'Codex-only guidance.',
'After',
].join('\n'));
});
test('removes non-matching provider blocks', () => {
const content = [
'Before',
'<codex>',
'Codex-only guidance.',
'</codex>',
'After',
].join('\n');
expect(compileProviderBlocks(content, ['claude-code'])).toBe([
'Before',
'',
'After',
].join('\n'));
});
test('does not leave extra blank lines around stripped blocks', () => {
const content = [
'Before',
'',
'<codex>',
'Codex-only guidance.',
'</codex>',
'',
'After',
].join('\n');
expect(compileProviderBlocks(content, ['claude-code'])).toBe([
'Before',
'',
'After',
].join('\n'));
});
test('preserves unknown standalone tags', () => {
const content = [
'Before',
'<aside>',
'Normal markdown HTML.',
'</aside>',
'After',
].join('\n');
expect(compileProviderBlocks(content, ['codex'])).toBe(content);
});
test('keeps codex blocks for targets that opt into the codex tag', () => {
const content = [
'<codex>',
'Codex repo skill guidance.',
'</codex>',
].join('\n');
expect(compileProviderBlocks(content, ['agents', 'codex'])).toBe('Codex repo skill guidance.');
});
test('all provider configs use known provider block tags', () => {
for (const config of Object.values(PROVIDERS)) {
expect(config.providerTags?.length).toBeGreaterThan(0);
for (const tag of config.providerTags) {
expect(PROVIDER_BLOCK_TAGS.has(tag)).toBe(true);
}
}
});
});
@@ -0,0 +1,137 @@
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import fs from 'fs';
import path from 'path';
import { createTransformer } from '../../../scripts/lib/transformers/factory.js';
const TEST_DIR = path.join(process.cwd(), 'test-tmp-provider-block-transformer');
const baseConfig = {
provider: 'cursor',
providerTags: ['cursor'],
configDir: '.test',
displayName: 'Test Provider',
frontmatterFields: [],
};
describe('provider block transformer integration', () => {
beforeEach(() => {
if (fs.existsSync(TEST_DIR)) {
fs.rmSync(TEST_DIR, { recursive: true, force: true });
}
});
afterEach(() => {
if (fs.existsSync(TEST_DIR)) {
fs.rmSync(TEST_DIR, { recursive: true, force: true });
}
});
test('compiles provider blocks in skill bodies', () => {
const transform = createTransformer(baseConfig);
const skills = [{
name: 'test',
description: 'Test',
body: [
'Shared guidance.',
'<cursor>',
'Cursor-only guidance.',
'</cursor>',
'<codex>',
'Codex-only guidance.',
'</codex>',
].join('\n')
}];
transform(skills, TEST_DIR);
const content = fs.readFileSync(path.join(TEST_DIR, 'cursor/.test/skills/test/SKILL.md'), 'utf-8');
expect(content).toContain('Shared guidance.');
expect(content).toContain('Cursor-only guidance.');
expect(content).not.toContain('Codex-only guidance.');
expect(content).not.toContain('<cursor>');
});
test('compiles provider blocks in reference files', () => {
const transform = createTransformer(baseConfig);
const skills = [{
name: 'test',
description: 'Test',
body: 'Body',
references: [
{
name: 'ref',
filePath: '/fake/ref.md',
content: [
'Shared reference.',
'<cursor>',
'Cursor reference.',
'</cursor>',
'<codex>',
'Codex reference.',
'</codex>',
].join('\n')
},
]
}];
transform(skills, TEST_DIR);
const ref = fs.readFileSync(path.join(TEST_DIR, 'cursor/.test/skills/test/reference/ref.md'), 'utf-8');
expect(ref).toContain('Shared reference.');
expect(ref).toContain('Cursor reference.');
expect(ref).not.toContain('Codex reference.');
expect(ref).not.toContain('<cursor>');
});
test('compiles provider blocks in generated agents', () => {
const config = {
...baseConfig,
provider: 'codex',
providerTags: ['codex'],
agentFormat: 'codex-toml',
};
const transform = createTransformer(config);
const skills = [{
name: 'test',
description: 'Test',
body: 'Body',
agents: [{
name: 'review-helper',
codexName: 'review_helper',
description: 'Review helper',
body: [
'Shared agent guidance.',
'<codex>',
'Codex agent guidance.',
'</codex>',
'<claude-code>',
'Claude agent guidance.',
'</claude-code>',
].join('\n')
}]
}];
transform(skills, TEST_DIR);
const agent = fs.readFileSync(path.join(TEST_DIR, 'codex/.test/agents/review_helper.toml'), 'utf-8');
expect(agent).toContain('Shared agent guidance.');
expect(agent).toContain('Codex agent guidance.');
expect(agent).not.toContain('Claude agent guidance.');
expect(agent).not.toContain('<codex>');
});
test('writes nested script artifacts', () => {
const transform = createTransformer(baseConfig);
const skills = [{
name: 'test',
description: 'Test',
body: 'Body',
scripts: [
{ name: 'detect.mjs', content: 'export {};\n' },
{ name: 'detector/detect-antipatterns.mjs', content: 'export const bundled = true;\n' },
],
}];
transform(skills, TEST_DIR);
const detector = path.join(TEST_DIR, 'cursor/.test/skills/test/scripts/detector/detect-antipatterns.mjs');
expect(fs.existsSync(detector)).toBe(true);
expect(fs.readFileSync(detector, 'utf-8')).toContain('bundled = true');
});
});