mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-12 22:26:38 +03:00
105 lines
4.0 KiB
JavaScript
105 lines
4.0 KiB
JavaScript
/**
|
|
* Multi-provider model factory for the skill-behavior test harness.
|
|
*
|
|
* The default lineup stays on current, economical models so the entire
|
|
* routing suite remains practical to run. Frontier quality is measured by
|
|
* the sibling impeccable-evals harness; this suite measures skill protocol.
|
|
*
|
|
* Anthropic and OpenAI use the Vercel AI SDK providers. Google uses
|
|
* @ai-sdk/google for the same reason — uniform tool-use semantics across all
|
|
* three keeps the harness tiny.
|
|
*
|
|
* .env is loaded from the repo root (copied from impeccable-evals). Tests
|
|
* skip cleanly when the matching key is unset rather than failing CI.
|
|
*/
|
|
import { anthropic, createAnthropic } from '@ai-sdk/anthropic';
|
|
import { google } from '@ai-sdk/google';
|
|
import { openai } from '@ai-sdk/openai';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const REPO_ROOT = path.resolve(__dirname, '..', '..');
|
|
|
|
function loadEnv() {
|
|
const envPath = path.join(REPO_ROOT, '.env');
|
|
if (!fs.existsSync(envPath)) return;
|
|
const text = fs.readFileSync(envPath, 'utf8');
|
|
for (const line of text.split('\n')) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed || trimmed.startsWith('#')) continue;
|
|
const eq = trimmed.indexOf('=');
|
|
if (eq === -1) continue;
|
|
const key = trimmed.slice(0, eq).trim();
|
|
let value = trimmed.slice(eq + 1).trim();
|
|
if (value.startsWith('"') && value.endsWith('"')) value = value.slice(1, -1);
|
|
if (value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
|
|
if (!process.env[key]) process.env[key] = value;
|
|
}
|
|
}
|
|
loadEnv();
|
|
|
|
export const PROVIDERS = {
|
|
anthropic: { envKey: 'ANTHROPIC_API_KEY', label: 'Anthropic' },
|
|
openai: { envKey: 'OPENAI_API_KEY', label: 'OpenAI' },
|
|
google: { envKey: 'GOOGLE_CLOUD_API_KEY', label: 'Google' },
|
|
deepseek: { envKey: 'DEEPSEEK_API_KEY', label: 'DeepSeek' },
|
|
};
|
|
|
|
export function detectProvider(modelId) {
|
|
if (modelId.startsWith('claude-')) return 'anthropic';
|
|
if (modelId.startsWith('gpt-')) return 'openai';
|
|
if (modelId.startsWith('gemini-')) return 'google';
|
|
if (modelId.startsWith('deepseek-')) return 'deepseek';
|
|
throw new Error(`Unsupported model id: "${modelId}"`);
|
|
}
|
|
|
|
export function hasKey(provider) {
|
|
const meta = PROVIDERS[provider];
|
|
if (!meta) return false;
|
|
return Boolean(process.env[meta.envKey]);
|
|
}
|
|
|
|
export function getModel(modelId) {
|
|
const provider = detectProvider(modelId);
|
|
if (provider === 'anthropic') return anthropic(modelId);
|
|
if (provider === 'openai') return openai(modelId);
|
|
if (provider === 'google') {
|
|
// The @ai-sdk/google provider reads GOOGLE_GENERATIVE_AI_API_KEY by
|
|
// default; the evals .env stores the same value under
|
|
// GOOGLE_CLOUD_API_KEY. Mirror it so the SDK picks it up automatically.
|
|
if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY && process.env.GOOGLE_CLOUD_API_KEY) {
|
|
process.env.GOOGLE_GENERATIVE_AI_API_KEY = process.env.GOOGLE_CLOUD_API_KEY;
|
|
}
|
|
return google(modelId);
|
|
}
|
|
if (provider === 'deepseek') {
|
|
// DeepSeek's official Claude Code integration exposes an Anthropic-
|
|
// compatible endpoint authenticated with a Bearer token.
|
|
const deepseek = createAnthropic({
|
|
baseURL: 'https://api.deepseek.com/anthropic',
|
|
authToken: process.env.DEEPSEEK_API_KEY,
|
|
name: 'deepseek.anthropic',
|
|
});
|
|
return deepseek(modelId);
|
|
}
|
|
throw new Error(`Unsupported provider: ${provider}`);
|
|
}
|
|
|
|
/**
|
|
* Default model lineup. These are current models, but intentionally the
|
|
* economical members of each family: this test is about routing/loading
|
|
* behavior, not design-output quality.
|
|
* Override with IMPECCABLE_SKILL_BEHAVIOR_MODELS=claude-foo,gpt-bar.
|
|
*/
|
|
export const DEFAULT_MODELS = ['claude-sonnet-5', 'gpt-5.6-luna', 'gemini-3.5-flash', 'deepseek-v4-flash'];
|
|
|
|
export function resolveModelList() {
|
|
const override = process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS;
|
|
if (override && override.trim()) {
|
|
return override.split(',').map((s) => s.trim()).filter(Boolean);
|
|
}
|
|
return DEFAULT_MODELS;
|
|
}
|