mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-15 23:56:29 +03:00
86 lines
3.1 KiB
TypeScript
86 lines
3.1 KiB
TypeScript
const candidate = (
|
|
firstMedianMs: number | null,
|
|
firstP95Ms: number | null,
|
|
allMedianMs: number | null,
|
|
allP95Ms: number | null,
|
|
passedRuns: number,
|
|
costUsd: number,
|
|
) => ({
|
|
firstMedianMs,
|
|
firstP95Ms,
|
|
allMedianMs,
|
|
allP95Ms,
|
|
passedRuns,
|
|
totalRuns: 5,
|
|
costUsd,
|
|
reliable: passedRuns === 5,
|
|
});
|
|
|
|
export const liveProviderResult = {
|
|
generatedAt: '2026-07-13T01:11:49.054Z',
|
|
fixture: 'vite8-react-brand-fidelity',
|
|
runsPerCandidate: 5,
|
|
statisticallyStable: true,
|
|
strategies: ['atomic-full', 'progressive-full', 'progressive-compact', 'parallel-compact'],
|
|
cleanupControl: {
|
|
providerIndependent: true,
|
|
acceptToCleanPickingMs: 172.19,
|
|
productionBuildMs: 383.51,
|
|
markerFree: true,
|
|
browserClean: true,
|
|
consoleErrors: 0,
|
|
passed: true,
|
|
},
|
|
providers: [
|
|
{
|
|
provider: 'Anthropic',
|
|
model: 'Claude Sonnet 4.6',
|
|
effort: 'low',
|
|
atomic: candidate(57168.08, 64787.41, 57168.08, 64787.41, 4, 0.882555),
|
|
progressiveFull: candidate(18623.04, 21744.52, 40591.04, 57284.19, 3, 1.35405),
|
|
progressiveCompact: candidate(16387.92, 17641.08, 41505.32, 46789.53, 5, 0.386004),
|
|
parallelCompact: candidate(10567.09, 10755.49, 19805.73, 20156.03, 3, 0.424566),
|
|
recommendation: 'Progressive compact is the only 5/5 candidate; use atomic only as a compatibility fallback.',
|
|
},
|
|
{
|
|
provider: 'OpenAI',
|
|
model: 'GPT-5.5',
|
|
effort: 'low',
|
|
atomic: candidate(27266.19, 31780.17, 27266.19, 31780.17, 5, 0.602296),
|
|
progressiveFull: candidate(12681.34, 14283.56, 36102.57, 41957.05, 5, 0.713001),
|
|
progressiveCompact: candidate(11777.98, 13334.92, 28867.61, 35691.27, 5, 0.48768),
|
|
parallelCompact: candidate(11846.01, 13551.87, 16211.64, 19043.63, 5, 0.614395),
|
|
recommendation: 'Progressive compact wins first review and cost; parallel compact wins full-set completion.',
|
|
},
|
|
{
|
|
provider: 'Google',
|
|
model: 'Gemini 3.1 Flash-Lite',
|
|
effort: 'minimal (provider default)',
|
|
atomic: candidate(6579.06, 6579.06, 6579.06, 6579.06, 1, 0.034123),
|
|
progressiveFull: candidate(null, null, null, null, 0, 0.023667),
|
|
progressiveCompact: candidate(3063.57, 3069.15, 5357.76, 5368.13, 3, 0.015316),
|
|
parallelCompact: candidate(1635.05, 1973.83, 2193.7, 3576.19, 5, 0.020824),
|
|
recommendation: 'Parallel compact is the only 5/5 candidate and the clear production path.',
|
|
},
|
|
],
|
|
totals: {
|
|
runs: 60,
|
|
passingRuns: 44,
|
|
apiCalls: 160,
|
|
retries: 47,
|
|
rejectedResponses: 64,
|
|
estimatedCostUsd: 5.558477,
|
|
},
|
|
gate: {
|
|
dimensions: ['brand', 'component', 'token', 'copy', 'source', 'accept-cleanup'],
|
|
minimumOverall: 0.9,
|
|
minimumDimension: 0.75,
|
|
},
|
|
recommendation: {
|
|
codexDefault: 'progressive-compact',
|
|
googleDefault: 'parallel-compact',
|
|
portableFallback: 'atomic-full',
|
|
rationale: 'Choose per provider and harness. Compact progressive is the reliable first-review winner for OpenAI and Anthropic; parallel compact is required for Gemini reliability. Harnesses without measured progressive publication retain atomic delivery.',
|
|
},
|
|
} as const;
|