mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-21 02:26:31 +03:00
Improve critique skill reliability
- add provider-specific block compilation and tests - bundle detector scripts for skill critique runs - harden critique orchestration, browser handling, and storage
This commit is contained in:
@@ -5,8 +5,8 @@
|
||||
|
||||
import { describe, it, beforeEach, afterEach } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { mkdtempSync, rmSync, symlinkSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -173,6 +173,19 @@ describe('CLI entry point', () => {
|
||||
assert.match(r.stderr, /no stable slug/);
|
||||
});
|
||||
|
||||
it('runs when invoked through a symlinked harness path', () => {
|
||||
const linkedScript = join(cwd, 'linked-critique-storage.mjs');
|
||||
symlinkSync(SCRIPT, linkedScript);
|
||||
|
||||
const r = spawnSync(process.execPath, [linkedScript, 'slug', 'index.html'], {
|
||||
cwd,
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
|
||||
assert.equal(r.status, 0, `stderr: ${r.stderr}`);
|
||||
assert.equal(r.stdout.trim(), 'index-html');
|
||||
});
|
||||
|
||||
it('latest subcommand exits 2 when no snapshot exists', () => {
|
||||
const r = spawnSync(process.execPath, [SCRIPT, 'latest', 'never-written'], {
|
||||
cwd,
|
||||
@@ -204,4 +217,3 @@ describe('readTrend', () => {
|
||||
assert.deepEqual(readTrend('nope', { cwd }), []);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { readSourceFiles } from '../../scripts/lib/utils.js';
|
||||
|
||||
const ROOT = process.cwd();
|
||||
|
||||
describe('skill detector bundle', () => {
|
||||
test('adds the detector wrapper and engine files to skill scripts', () => {
|
||||
const { skills } = readSourceFiles(ROOT);
|
||||
const skill = skills.find(s => s.name === 'impeccable');
|
||||
const scriptNames = new Set(skill.scripts.map(s => s.name));
|
||||
|
||||
expect(scriptNames.has('detect.mjs')).toBe(true);
|
||||
expect(scriptNames.has('detector/detect-antipatterns.mjs')).toBe(true);
|
||||
expect(scriptNames.has('detector/detect-antipatterns-browser.js')).toBe(true);
|
||||
expect(scriptNames.has('detector/cli/main.mjs')).toBe(true);
|
||||
expect(scriptNames.has('detector/engines/static-html/detect-html.mjs')).toBe(true);
|
||||
});
|
||||
|
||||
test('critique references the bundled detector command', () => {
|
||||
const critique = fs.readFileSync(path.join(ROOT, 'skill/reference/critique.md'), 'utf-8');
|
||||
|
||||
expect(critique).toContain('node {{scripts_path}}/detect.mjs --json [--fast] [target]');
|
||||
expect(critique).not.toContain('npx impeccable detect');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { PROVIDERS } from '../../scripts/lib/transformers/providers.js';
|
||||
import { compileProviderBlocks, PROVIDER_BLOCK_TAGS } from '../../scripts/lib/utils.js';
|
||||
|
||||
describe('compileProviderBlocks', () => {
|
||||
test('keeps matching provider block bodies and removes tags', () => {
|
||||
const content = [
|
||||
'Before',
|
||||
'<codex>',
|
||||
'Codex-only guidance.',
|
||||
'</codex>',
|
||||
'After',
|
||||
].join('\n');
|
||||
|
||||
expect(compileProviderBlocks(content, ['codex'])).toBe([
|
||||
'Before',
|
||||
'Codex-only guidance.',
|
||||
'After',
|
||||
].join('\n'));
|
||||
});
|
||||
|
||||
test('removes non-matching provider blocks', () => {
|
||||
const content = [
|
||||
'Before',
|
||||
'<codex>',
|
||||
'Codex-only guidance.',
|
||||
'</codex>',
|
||||
'After',
|
||||
].join('\n');
|
||||
|
||||
expect(compileProviderBlocks(content, ['claude-code'])).toBe([
|
||||
'Before',
|
||||
'',
|
||||
'After',
|
||||
].join('\n'));
|
||||
});
|
||||
|
||||
test('does not leave extra blank lines around stripped blocks', () => {
|
||||
const content = [
|
||||
'Before',
|
||||
'',
|
||||
'<codex>',
|
||||
'Codex-only guidance.',
|
||||
'</codex>',
|
||||
'',
|
||||
'After',
|
||||
].join('\n');
|
||||
|
||||
expect(compileProviderBlocks(content, ['claude-code'])).toBe([
|
||||
'Before',
|
||||
'',
|
||||
'After',
|
||||
].join('\n'));
|
||||
});
|
||||
|
||||
test('preserves unknown standalone tags', () => {
|
||||
const content = [
|
||||
'Before',
|
||||
'<aside>',
|
||||
'Normal markdown HTML.',
|
||||
'</aside>',
|
||||
'After',
|
||||
].join('\n');
|
||||
|
||||
expect(compileProviderBlocks(content, ['codex'])).toBe(content);
|
||||
});
|
||||
|
||||
test('keeps codex blocks for targets that opt into the codex tag', () => {
|
||||
const content = [
|
||||
'<codex>',
|
||||
'Codex repo skill guidance.',
|
||||
'</codex>',
|
||||
].join('\n');
|
||||
|
||||
expect(compileProviderBlocks(content, ['agents', 'codex'])).toBe('Codex repo skill guidance.');
|
||||
});
|
||||
|
||||
test('all provider configs use known provider block tags', () => {
|
||||
for (const config of Object.values(PROVIDERS)) {
|
||||
expect(config.providerTags?.length).toBeGreaterThan(0);
|
||||
for (const tag of config.providerTags) {
|
||||
expect(PROVIDER_BLOCK_TAGS.has(tag)).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,137 @@
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { createTransformer } from '../../../scripts/lib/transformers/factory.js';
|
||||
|
||||
const TEST_DIR = path.join(process.cwd(), 'test-tmp-provider-block-transformer');
|
||||
|
||||
const baseConfig = {
|
||||
provider: 'cursor',
|
||||
providerTags: ['cursor'],
|
||||
configDir: '.test',
|
||||
displayName: 'Test Provider',
|
||||
frontmatterFields: [],
|
||||
};
|
||||
|
||||
describe('provider block transformer integration', () => {
|
||||
beforeEach(() => {
|
||||
if (fs.existsSync(TEST_DIR)) {
|
||||
fs.rmSync(TEST_DIR, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (fs.existsSync(TEST_DIR)) {
|
||||
fs.rmSync(TEST_DIR, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('compiles provider blocks in skill bodies', () => {
|
||||
const transform = createTransformer(baseConfig);
|
||||
const skills = [{
|
||||
name: 'test',
|
||||
description: 'Test',
|
||||
body: [
|
||||
'Shared guidance.',
|
||||
'<cursor>',
|
||||
'Cursor-only guidance.',
|
||||
'</cursor>',
|
||||
'<codex>',
|
||||
'Codex-only guidance.',
|
||||
'</codex>',
|
||||
].join('\n')
|
||||
}];
|
||||
transform(skills, TEST_DIR);
|
||||
|
||||
const content = fs.readFileSync(path.join(TEST_DIR, 'cursor/.test/skills/test/SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Shared guidance.');
|
||||
expect(content).toContain('Cursor-only guidance.');
|
||||
expect(content).not.toContain('Codex-only guidance.');
|
||||
expect(content).not.toContain('<cursor>');
|
||||
});
|
||||
|
||||
test('compiles provider blocks in reference files', () => {
|
||||
const transform = createTransformer(baseConfig);
|
||||
const skills = [{
|
||||
name: 'test',
|
||||
description: 'Test',
|
||||
body: 'Body',
|
||||
references: [
|
||||
{
|
||||
name: 'ref',
|
||||
filePath: '/fake/ref.md',
|
||||
content: [
|
||||
'Shared reference.',
|
||||
'<cursor>',
|
||||
'Cursor reference.',
|
||||
'</cursor>',
|
||||
'<codex>',
|
||||
'Codex reference.',
|
||||
'</codex>',
|
||||
].join('\n')
|
||||
},
|
||||
]
|
||||
}];
|
||||
transform(skills, TEST_DIR);
|
||||
|
||||
const ref = fs.readFileSync(path.join(TEST_DIR, 'cursor/.test/skills/test/reference/ref.md'), 'utf-8');
|
||||
expect(ref).toContain('Shared reference.');
|
||||
expect(ref).toContain('Cursor reference.');
|
||||
expect(ref).not.toContain('Codex reference.');
|
||||
expect(ref).not.toContain('<cursor>');
|
||||
});
|
||||
|
||||
test('compiles provider blocks in generated agents', () => {
|
||||
const config = {
|
||||
...baseConfig,
|
||||
provider: 'codex',
|
||||
providerTags: ['codex'],
|
||||
agentFormat: 'codex-toml',
|
||||
};
|
||||
const transform = createTransformer(config);
|
||||
const skills = [{
|
||||
name: 'test',
|
||||
description: 'Test',
|
||||
body: 'Body',
|
||||
agents: [{
|
||||
name: 'review-helper',
|
||||
codexName: 'review_helper',
|
||||
description: 'Review helper',
|
||||
body: [
|
||||
'Shared agent guidance.',
|
||||
'<codex>',
|
||||
'Codex agent guidance.',
|
||||
'</codex>',
|
||||
'<claude-code>',
|
||||
'Claude agent guidance.',
|
||||
'</claude-code>',
|
||||
].join('\n')
|
||||
}]
|
||||
}];
|
||||
transform(skills, TEST_DIR);
|
||||
|
||||
const agent = fs.readFileSync(path.join(TEST_DIR, 'codex/.test/agents/review_helper.toml'), 'utf-8');
|
||||
expect(agent).toContain('Shared agent guidance.');
|
||||
expect(agent).toContain('Codex agent guidance.');
|
||||
expect(agent).not.toContain('Claude agent guidance.');
|
||||
expect(agent).not.toContain('<codex>');
|
||||
});
|
||||
|
||||
test('writes nested script artifacts', () => {
|
||||
const transform = createTransformer(baseConfig);
|
||||
const skills = [{
|
||||
name: 'test',
|
||||
description: 'Test',
|
||||
body: 'Body',
|
||||
scripts: [
|
||||
{ name: 'detect.mjs', content: 'export {};\n' },
|
||||
{ name: 'detector/detect-antipatterns.mjs', content: 'export const bundled = true;\n' },
|
||||
],
|
||||
}];
|
||||
transform(skills, TEST_DIR);
|
||||
|
||||
const detector = path.join(TEST_DIR, 'cursor/.test/skills/test/scripts/detector/detect-antipatterns.mjs');
|
||||
expect(fs.existsSync(detector)).toBe(true);
|
||||
expect(fs.readFileSync(detector, 'utf-8')).toContain('bundled = true');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user