skills/pbakaus_impeccable
1
0
Fork 0
mirror of https://github.com/pbakaus/impeccable.git synced 2026-09-14 23:26:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

Harden Live production benchmarks and turn failures

Browse Source 
AI-assisted: Codex
This commit is contained in:
Paul Bakaus
2026-07-13 10:19:42 -07:00
parent db63d08168
commit e9121b26fe
3 changed files with 65 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/benchmark-live.mjs
+29 -24
View File
@@ -557,34 +557,39 @@ async function waitForReset(page) {
}
async function installBrowserTimingProbe(page) {
await page.evaluate(() => {
const state = { iteration: 0, goAt: null, generateAt: null };
window.__IMPECCABLE_LIVE_BENCH_TIMING__ = state;
const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.()
|| window.__IMPECCABLE_LIVE_UI_ROOT__
|| document;
root.addEventListener('click', (event) => {
const button = event.composedPath().find((node) =>
node?.getAttribute?.('aria-label') === 'Generate variants'
);
if (button) state.goAt = performance.now();
}, true);
await page.addInitScript(installBrowserTimingProbeInPage);
await page.evaluate(installBrowserTimingProbeInPage);
}
const originalFetch = window.fetch.bind(window);
window.fetch = (input, init) => {
try {
const url = typeof input === 'string' ? input : input?.url;
if (String(url || '').endsWith('/events') && init?.method === 'POST') {
const payload = typeof init.body === 'string' ? JSON.parse(init.body) : null;
if (payload?.type === 'generate') state.generateAt = performance.now();
}
} catch { /* measurement must never affect Live */ }
return originalFetch(input, init);
};
});
function installBrowserTimingProbeInPage() {
if (window.__IMPECCABLE_LIVE_BENCH_TIMING__?.installed === true) return;
const state = { iteration: 0, goAt: null, generateAt: null, installed: true };
window.__IMPECCABLE_LIVE_BENCH_TIMING__ = state;
const root = window.__IMPECCABLE_LIVE_CHROME_CORE__?.root?.()
|| window.__IMPECCABLE_LIVE_UI_ROOT__
|| document;
root.addEventListener('click', (event) => {
const button = event.composedPath().find((node) =>
node?.getAttribute?.('aria-label') === 'Generate variants'
);
if (button) state.goAt = performance.now();
}, true);
const originalFetch = window.fetch.bind(window);
window.fetch = (input, init) => {
try {
const url = typeof input === 'string' ? input : input?.url;
if (String(url || '').endsWith('/events') && init?.method === 'POST') {
const payload = typeof init.body === 'string' ? JSON.parse(init.body) : null;
if (payload?.type === 'generate') state.generateAt = performance.now();
}
} catch { /* measurement must never affect Live */ }
return originalFetch(input, init);
};
}
async function resetBrowserTimingProbe(page, iteration) {
await page.evaluate(installBrowserTimingProbeInPage);
await page.evaluate((nextIteration) => {
const state = window.__IMPECCABLE_LIVE_BENCH_TIMING__;
if (!state) return;
skill/scripts/live/codex-app-server-client.mjs
+9 -1
View File
@@ -453,6 +453,14 @@ export class CodexAppServerClient {
await completionPromise;
await Promise.all(agentMessageCallbacks);
const completedAt = completed?.receivedAt ?? this.clock();
const status = completed?.params?.turn?.status || result.turn?.status || null;
if (status !== 'completed') {
const interrupted = status === 'interrupted' || status === 'cancelled' || status === 'canceled';
throw new CodexAppServerError(`turn ${turnId} completed with status ${status || 'unknown'}`, {
code: interrupted ? 'TURN_INTERRUPTED' : 'TURN_FAILED',
data: completed?.params?.turn || result.turn || null,
});
}
return {
threadId,
turnId,
@@ -461,7 +469,7 @@ export class CodexAppServerClient {
started,
completed,
tokenUsage,
status: completed?.params?.turn?.status || result.turn?.status || null,
status,
agentMessages,
message: agentMessages.at(-1) || null,
requestedAt,
tests/live-codex-app-server-client.test.mjs
+27
View File
@@ -337,6 +337,33 @@ describe('dedicated Codex worker threads', () => {
await client.close();
});
it('rejects failed turn completions instead of treating them as usable output', async () => {
const { client } = await connectClient((message, process) => {
if (message.method === 'thread/start') {
process.respond(message, { thread: { id: 'worker' } });
}
if (message.method === 'turn/start') {
process.respond(message, { turn: { id: 'turn-failed', status: 'inProgress' } });
queueMicrotask(() => {
process.send({
method: 'turn/completed',
params: {
threadId: 'worker',
turn: { id: 'turn-failed', status: 'failed', error: { message: 'model unavailable' } },
},
});
});
}
});
await client.startDedicatedThread({ serviceName: 'impeccable_live_worker' });
await assert.rejects(
client.startTurn({ threadId: 'worker', input: 'work' }),
(error) => error.code === 'TURN_FAILED' && /status failed/.test(error.message),
);
await client.close();
});
it('interrupts, unsubscribes, archives, and cleanly closes', async () => {
const methods = [];
const { client, child } = await connectClient((message, process) => {