Improve critique skill reliability

- add provider-specific block compilation and tests

- bundle detector scripts for skill critique runs

- harden critique orchestration, browser handling, and storage
This commit is contained in:
Paul Bakaus
2026-05-18 15:15:14 -07:00
parent e1d3ea0b6f
commit bc1894889e
285 changed files with 142401 additions and 1251 deletions
+26 -5
View File
@@ -1,5 +1,13 @@
import path from 'path';
import { cleanDir, ensureDir, writeFile, generateYamlFrontmatter, generateYamlDocument, replacePlaceholders } from '../utils.js';
import {
cleanDir,
ensureDir,
writeFile,
generateYamlFrontmatter,
generateYamlDocument,
replacePlaceholders,
compileProviderBlocks,
} from '../utils.js';
import { SKILL_CATEGORIES, CATEGORY_ORDER } from '../sub-pages-data.js';
/**
@@ -138,7 +146,17 @@ function buildAgentFile(config, agent, body) {
* @returns {Function} transform(skills, distDir, options?)
*/
export function createTransformer(config) {
const { provider, configDir, displayName, frontmatterFields = [], bodyTransform, placeholderProvider, writeOpenAIMetadata = false, includeVersion = true } = config;
const {
provider,
configDir,
displayName,
frontmatterFields = [],
bodyTransform,
placeholderProvider,
providerTags = [provider],
writeOpenAIMetadata = false,
includeVersion = true,
} = config;
const placeholderKey = placeholderProvider || provider;
const activeFields = frontmatterFields
@@ -200,7 +218,8 @@ export function createTransformer(config) {
const frontmatter = generateYamlFrontmatter(frontmatterObj);
// Build body
let skillBody = replacePlaceholders(skill.body, placeholderKey, commandNames, allSkillNames);
let skillBody = compileProviderBlocks(skill.body, providerTags);
skillBody = replacePlaceholders(skillBody, placeholderKey, commandNames, allSkillNames);
// Replace {{scripts_path}} with provider-aware path to skill's scripts directory
const scriptsPath = `${configDir}/skills/${skillName}/scripts`;
@@ -220,7 +239,8 @@ export function createTransformer(config) {
const refDir = path.join(skillDir, 'reference');
ensureDir(refDir);
for (const ref of skill.references) {
let refContent = replacePlaceholders(ref.content, placeholderKey, [], allSkillNames);
let refContent = compileProviderBlocks(ref.content, providerTags);
refContent = replacePlaceholders(refContent, placeholderKey, [], allSkillNames);
refContent = refContent.replace(/\{\{scripts_path\}\}/g, scriptsPath);
writeFile(path.join(refDir, `${ref.name}.md`), refContent);
refCount++;
@@ -245,7 +265,8 @@ export function createTransformer(config) {
// Agents can declare `providers: <list>` to limit which harnesses
// they emit to. Default (no field) ships everywhere with agentFormat.
if (agent.providers && !agent.providers.includes(provider)) continue;
const body = replacePlaceholders(agent.body, placeholderKey, [], allSkillNames);
let body = compileProviderBlocks(agent.body, providerTags);
body = replacePlaceholders(body, placeholderKey, [], allSkillNames);
const agentFile = buildAgentFile(config, agent, body);
if (!agentFile) continue;
ensureDir(agentsDir);
+14
View File
@@ -5,18 +5,21 @@
* - provider: key into PROVIDER_PLACEHOLDERS (e.g. 'claude-code')
* - configDir: dot-directory name (e.g. '.claude')
* - displayName: human-readable name for log output (e.g. 'Claude Code')
* - providerTags: markdown block tags kept for this target (e.g. <codex>...</codex>)
* - frontmatterFields: which optional fields to emit beyond name + description
* - bodyTransform: optional function (body, skill) => transformed body
*/
export const PROVIDERS = {
cursor: {
provider: 'cursor',
providerTags: ['cursor'],
configDir: '.cursor',
displayName: 'Cursor',
frontmatterFields: ['license', 'compatibility', 'metadata'],
},
'claude-code': {
provider: 'claude-code',
providerTags: ['claude-code', 'claude'],
configDir: '.claude',
displayName: 'Claude Code',
frontmatterFields: ['user-invocable', 'argument-hint', 'license', 'compatibility', 'metadata', 'allowed-tools'],
@@ -24,12 +27,14 @@ export const PROVIDERS = {
},
gemini: {
provider: 'gemini',
providerTags: ['gemini'],
configDir: '.gemini',
displayName: 'Gemini',
frontmatterFields: [],
},
codex: {
provider: 'codex',
providerTags: ['codex'],
configDir: '.codex',
displayName: 'Codex',
frontmatterFields: [],
@@ -39,6 +44,7 @@ export const PROVIDERS = {
},
agents: {
provider: 'agents',
providerTags: ['agents', 'codex'],
configDir: '.agents',
displayName: 'Codex Repo Skills',
placeholderProvider: 'codex',
@@ -48,6 +54,7 @@ export const PROVIDERS = {
},
github: {
provider: 'github',
providerTags: ['github'],
configDir: '.github',
displayName: 'GitHub Copilot',
placeholderProvider: 'agents',
@@ -55,30 +62,35 @@ export const PROVIDERS = {
},
kiro: {
provider: 'kiro',
providerTags: ['kiro'],
configDir: '.kiro',
displayName: 'Kiro',
frontmatterFields: ['license', 'compatibility', 'metadata'],
},
opencode: {
provider: 'opencode',
providerTags: ['opencode'],
configDir: '.opencode',
displayName: 'OpenCode',
frontmatterFields: ['user-invocable', 'argument-hint', 'license', 'compatibility', 'metadata', 'allowed-tools'],
},
pi: {
provider: 'pi',
providerTags: ['pi'],
configDir: '.pi',
displayName: 'Pi',
frontmatterFields: ['license', 'compatibility', 'metadata', 'allowed-tools'],
},
qoder: {
provider: 'qoder',
providerTags: ['qoder'],
configDir: '.qoder',
displayName: 'Qoder',
frontmatterFields: ['user-invocable', 'argument-hint', 'license', 'compatibility', 'metadata', 'allowed-tools'],
},
'trae-cn': {
provider: 'trae-cn',
providerTags: ['trae-cn', 'trae'],
configDir: '.trae-cn',
displayName: 'Trae China',
placeholderProvider: 'trae',
@@ -86,12 +98,14 @@ export const PROVIDERS = {
},
trae: {
provider: 'trae',
providerTags: ['trae'],
configDir: '.trae',
displayName: 'Trae',
frontmatterFields: ['user-invocable', 'argument-hint', 'license', 'compatibility', 'metadata'],
},
'rovo-dev': {
provider: 'rovo-dev',
providerTags: ['rovo-dev'],
configDir: '.rovodev',
displayName: 'Rovo Dev',
frontmatterFields: ['user-invocable', 'argument-hint', 'license', 'compatibility', 'metadata', 'allowed-tools'],
+72
View File
@@ -9,6 +9,8 @@ import path from 'path';
// New installs write project config at .impeccable/live/config.json instead.
export const PER_PROJECT_SCRIPT_ARTIFACTS = new Set(['config.json']);
const DETECTOR_BUNDLE_DIR = 'cli/engine';
// Walk the harness-dir skill tree and return any per-project script
// artifacts found, ready for restoration after a full sync rm+recopy.
// Returns [{ relPath, content: Buffer }], where relPath is relative to
@@ -39,6 +41,32 @@ export function restorePerProjectArtifacts(rootDir, stashed) {
}
}
function readDetectorBundleScripts(rootDir) {
const detectorDir = path.join(rootDir, DETECTOR_BUNDLE_DIR);
if (!fs.existsSync(detectorDir)) return [];
const scripts = [];
const walk = (dir) => {
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
const entryPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
walk(entryPath);
continue;
}
if (!entry.isFile()) continue;
const relPath = path.relative(detectorDir, entryPath).split(path.sep).join('/');
scripts.push({
name: `detector/${relPath}`,
content: fs.readFileSync(entryPath, 'utf-8'),
filePath: entryPath,
generated: true,
});
}
};
walk(detectorDir);
return scripts;
}
/**
* Parse frontmatter from markdown content
* Returns { frontmatter: object, body: string }
@@ -202,6 +230,7 @@ export function readSourceFiles(rootDir) {
});
}
}
scripts.push(...readDetectorBundleScripts(rootDir));
const agents = [];
const agentsDir = path.join(skillDir, 'agents');
@@ -551,6 +580,49 @@ export const PROVIDER_PLACEHOLDERS = {
}
};
export const PROVIDER_BLOCK_TAGS = new Set([
'agents',
'claude',
'claude-code',
'codex',
'cursor',
'gemini',
'github',
'kiro',
'opencode',
'pi',
'qoder',
'rovo-dev',
'trae',
'trae-cn',
]);
/**
* Compile harness-conditional markdown blocks.
*
* Known provider blocks must be written as standalone tags:
*
* <codex>
* Codex-only instructions.
* </codex>
*
* Matching blocks keep their body and drop the tags. Non-matching blocks are
* removed. Unknown tags are preserved so ordinary markdown/HTML is untouched.
*/
export function compileProviderBlocks(content, activeTags = []) {
const activeTagSet = new Set(activeTags);
const providerBlockPattern = /(^|\r?\n)[ \t]*<([a-z][a-z0-9-]*)>[ \t]*\r?\n([\s\S]*?)\r?\n[ \t]*<\/\2>[ \t]*(?=\r?\n|$)/g;
let didCompileBlock = false;
const compiled = content.replace(providerBlockPattern, (match, prefix, tag, body) => {
if (!PROVIDER_BLOCK_TAGS.has(tag)) return match;
didCompileBlock = true;
return activeTagSet.has(tag) ? `${prefix}${body}` : prefix;
});
return didCompileBlock ? compiled.replace(/(?:\r?\n){3,}/g, '\n\n') : compiled;
}
/**
* Replace all {{placeholder}} tokens with provider-specific values
*/