Prewarm Codex Live with safe fallback

Return after a durable starting record, overlap app-server initialization with page startup, dynamically reclaim generation after worker failure, and cap hard-crash leases at 15 seconds.\n\nAI-assisted: OpenAI Codex.
This commit is contained in:
Paul Bakaus
2026-07-12 21:11:50 -07:00
parent 6f3076051f
commit 6ed43ca682
9 changed files with 170 additions and 24 deletions
+25 -4
View File
@@ -7,6 +7,8 @@ import { fileURLToPath } from 'node:url';
import { createCodexAppServerClient } from './live/codex-app-server-client.mjs';
import {
CODEX_WORKER_OWNER,
codexWorkerProcessStateIsOwned,
codexWorkerStateIsOwned,
resolveCodexWorkerConfig,
} from './live/codex-worker.mjs';
@@ -24,7 +26,7 @@ const scriptsDir = path.dirname(scriptPath);
const statePath = getLiveCodexWorkerStatePath(cwd);
if (args.includes('--help') || args.includes('-h')) {
console.log(`Usage: node live-codex-worker.mjs [--background | --status | --stop]
console.log(`Usage: node live-codex-worker.mjs [--background [--no-wait] | --status | --stop]
Codex Live generation supervisor. It owns a separate
app-server process and dedicated worker thread; it never attaches to the
@@ -55,7 +57,7 @@ if (args.includes('--status')) {
if (args.includes('--stop')) {
const state = readJson(statePath);
if (state?.pid && !codexWorkerStateIsOwned(state, cwd)) {
if (state?.pid && !codexWorkerProcessStateIsOwned(state, cwd)) {
console.log(JSON.stringify({
ok: false,
status: 'not_stopped',
@@ -95,10 +97,10 @@ if (!config.enabled) {
if (args.includes('--background')) {
const existing = readJson(statePath);
if (codexWorkerStateIsOwned(existing, cwd)
if (codexWorkerProcessStateIsOwned(existing, cwd)
&& existing?.pid
&& pidReachable(existing.pid)
&& ['ready', 'working'].includes(existing.status)) {
&& ['starting', 'ready', 'working'].includes(existing.status)) {
console.log(JSON.stringify({ ...existing, ok: true, reused: true }));
process.exit(0);
}
@@ -113,6 +115,24 @@ if (args.includes('--background')) {
});
child.unref();
fs.closeSync(logFd);
const observed = readJson(statePath);
const starting = observed?.pid === child.pid && ['ready', 'working'].includes(observed.status)
? observed
: writeState({
ok: true,
owner: CODEX_WORKER_OWNER,
pid: child.pid,
status: 'starting',
threadId: null,
model: config.model,
effort: config.effort,
profile: config.profile,
delivery: config.delivery,
});
if (args.includes('--no-wait')) {
console.log(JSON.stringify({ ...starting, ok: true, starting: true, logPath }));
process.exit(0);
}
const ready = await waitFor(() => {
const state = readJson(statePath);
if (state?.pid !== child.pid) return null;
@@ -213,6 +233,7 @@ function writeState(value) {
const temporary = `${statePath}.${process.pid}.${Date.now()}.tmp`;
fs.writeFileSync(temporary, JSON.stringify(state, null, 2) + '\n', 'utf-8');
fs.renameSync(temporary, statePath);
return state;
}
function readJson(file) {
+47 -10
View File
@@ -10,10 +10,12 @@
*/
import { execFileSync } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { completionAckForAcceptResult, completionTypeForAcceptResult } from './live/completion.mjs';
import { readLiveServerInfo } from './lib/impeccable-paths.mjs';
import { getLiveCodexWorkerStatePath, readLiveServerInfo } from './lib/impeccable-paths.mjs';
import { codexWorkerProcessStateIsOwned } from './live/codex-worker.mjs';
// Absolute path to a sibling script in this skill's scripts dir, so runtime
// error hints print a directly-runnable command instead of a placeholder.
@@ -152,7 +154,13 @@ export async function waitForEventAck(base, token, eventId, {
return false;
}
export async function fetchNextEvent(base, token, { totalDeadline, types } = {}) {
export async function fetchNextEvent(base, token, {
totalDeadline,
types,
resolveTypes,
perRequestTimeoutMs = PER_REQUEST_TIMEOUT_MS,
leaseMs = DEFAULT_EVENT_LEASE_MS,
} = {}) {
while (true) {
if (totalDeadline && Date.now() >= totalDeadline) {
return { type: 'timeout' };
@@ -161,13 +169,13 @@ export async function fetchNextEvent(base, token, { totalDeadline, types } = {})
const remaining = totalDeadline
? totalDeadline - Date.now()
: PER_REQUEST_TIMEOUT_MS;
const slice = Math.min(Math.max(remaining, 1000), PER_REQUEST_TIMEOUT_MS);
const slice = Math.min(Math.max(remaining, 1000), perRequestTimeoutMs);
const query = new URLSearchParams({
token,
timeout: String(slice),
leaseMs: String(DEFAULT_EVENT_LEASE_MS),
leaseMs: String(leaseMs),
});
const normalizedTypes = normalizePollTypes(types);
const normalizedTypes = normalizePollTypes(resolveTypes ? await resolveTypes() : types);
if (normalizedTypes.length > 0) query.set('types', normalizedTypes.join(','));
const res = await fetch(`${base}/poll?${query}`);
@@ -253,9 +261,9 @@ export function printPollEvent(event) {
console.log(JSON.stringify(event));
}
export async function runPollOnce(base, token, { totalTimeout = 600_000, types } = {}) {
export async function runPollOnce(base, token, { totalTimeout = 600_000, types, resolveTypes, perRequestTimeoutMs } = {}) {
const deadline = Date.now() + totalTimeout;
const event = await fetchNextEvent(base, token, { totalDeadline: deadline, types });
const event = await fetchNextEvent(base, token, { totalDeadline: deadline, types, resolveTypes, perRequestTimeoutMs });
await augmentEventWithAcceptHandling(event, base, token);
writeCarbonizeBanner(event);
printPollEvent(event);
@@ -267,11 +275,13 @@ export async function runPollStream(base, token, {
ackPollIntervalMs = 400,
shouldContinue = () => true,
types,
resolveTypes,
perRequestTimeoutMs,
} = {}) {
process.stderr.write('[impeccable-poll] stream mode: one JSON object per line on stdout; use --reply while this process stays running\n');
while (shouldContinue()) {
const event = await fetchNextEvent(base, token, { types });
const event = await fetchNextEvent(base, token, { types, resolveTypes, perRequestTimeoutMs });
await augmentEventWithAcceptHandling(event, base, token);
writeCarbonizeBanner(event);
printPollEvent(event);
@@ -332,6 +342,8 @@ Modes:
Options:
--timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode
--types=A,B Lease only these event types (used by partitioned Codex control lane)
--codex-worker-fallback
Add generation events only if the dedicated Codex worker fails or exits
--ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000)
--file PATH Attach a source file path to the reply (generate/steer flow)
--data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON
@@ -372,18 +384,21 @@ Harness note:
const streamMode = args.includes('--stream');
const typesArg = args.find((a) => a.startsWith('--types='));
const types = normalizePollTypes(typesArg ? typesArg.slice('--types='.length) : null);
const workerFallback = args.includes('--codex-worker-fallback');
const resolveTypes = workerFallback ? () => resolveCodexWorkerFallbackTypes(types) : null;
const perRequestTimeoutMs = workerFallback ? 2_000 : undefined;
const ackTimeoutArg = args.find((a) => a.startsWith('--ack-timeout='));
const ackTimeoutMs = ackTimeoutArg ? parseInt(ackTimeoutArg.split('=')[1], 10) : 600_000;
try {
if (streamMode) {
await runPollStream(base, info.token, { ackTimeoutMs, types });
await runPollStream(base, info.token, { ackTimeoutMs, types, resolveTypes, perRequestTimeoutMs });
return;
}
const timeoutArg = args.find((a) => a.startsWith('--timeout='));
const totalTimeout = timeoutArg ? parseInt(timeoutArg.split('=')[1], 10) : 600_000;
await runPollOnce(base, info.token, { totalTimeout, types });
await runPollOnce(base, info.token, { totalTimeout, types, resolveTypes, perRequestTimeoutMs });
} catch (err) {
handlePollError(err);
}
@@ -394,6 +409,28 @@ export function normalizePollTypes(value) {
return [...new Set(values.map((type) => String(type).trim()).filter(Boolean))];
}
export function resolveCodexWorkerFallbackTypes(baseTypes, {
cwd = process.cwd(),
state = readJson(getLiveCodexWorkerStatePath(cwd)),
isPidReachable = pidReachable,
} = {}) {
const base = normalizePollTypes(baseTypes);
const workerOwnsGeneration = codexWorkerProcessStateIsOwned(state, cwd)
&& ['starting', 'ready', 'working'].includes(state?.status)
&& isPidReachable(state?.pid);
if (workerOwnsGeneration) return base;
return normalizePollTypes([...base, 'generate', 'accept', 'discard', 'prefetch']);
}
function readJson(file) {
try { return JSON.parse(fs.readFileSync(file, 'utf-8')); } catch { return null; }
}
function pidReachable(pid) {
if (!Number.isInteger(pid) || pid <= 0) return false;
try { process.kill(pid, 0); return true; } catch { return false; }
}
// Auto-execute when run directly
const _running = process.argv[1];
if (_running?.endsWith('live-poll.mjs') || _running?.endsWith('live-poll.mjs/')) {
+3 -3
View File
@@ -299,7 +299,7 @@ function ensureCodexWorker(cwd, liveConfig) {
if (!config.enabled) {
return { enabled: false, mode: 'foreground', codexOnly: true };
}
const out = runScript('live-codex-worker.mjs', ['--background'], { cwd });
const out = runScript('live-codex-worker.mjs', ['--background', '--no-wait'], { cwd });
const result = safeParse(out);
if (!result?.ok) {
const safeFallback = result?.fallback === 'foreground' && result?.terminated !== false;
@@ -315,7 +315,7 @@ function ensureCodexWorker(cwd, liveConfig) {
}
return {
enabled: true,
mode: 'dedicated-app-server',
mode: result.starting ? 'prewarming-app-server' : 'dedicated-app-server',
codexOnly: true,
pid: result.pid,
threadId: result.threadId,
@@ -324,7 +324,7 @@ function ensureCodexWorker(cwd, liveConfig) {
profile: result.profile,
delivery: result.delivery,
foregroundTypes: ['steer', 'manual_edit_apply', 'carbonize_cleanup', 'exit'],
foregroundPoll: 'live-poll.mjs --types=steer,manual_edit_apply,carbonize_cleanup,exit',
foregroundPoll: 'live-poll.mjs --types=steer,manual_edit_apply,carbonize_cleanup,exit --codex-worker-fallback',
logPath: result.logPath || null,
};
}
@@ -32,6 +32,7 @@ import {
} from '../live-poll.mjs';
export const CODEX_WORKER_EVENT_TYPES = Object.freeze(['generate', 'accept', 'discard', 'prefetch']);
export const CODEX_WORKER_EVENT_LEASE_MS = 15_000;
export class CodexLiveWorkerSupervisor {
constructor({
@@ -117,7 +118,10 @@ export class CodexLiveWorkerSupervisor {
if (!this.thread) await this.initialize();
this.running = true;
while (this.running) {
const event = await this.fetchEvent(this.base, this.token, { types: CODEX_WORKER_EVENT_TYPES });
const event = await this.fetchEvent(this.base, this.token, {
types: CODEX_WORKER_EVENT_TYPES,
leaseMs: CODEX_WORKER_EVENT_LEASE_MS,
});
if (!event || event.type === 'timeout') continue;
if (event.type === 'exit') {
await this.cancelActive('live_exit');
+12 -2
View File
@@ -296,12 +296,22 @@ export function generationIsCanceled(eventId, { cwd = process.cwd() } = {}) {
}
export function codexWorkerStateIsOwned(state, cwd) {
return state?.owner === CODEX_WORKER_OWNER
&& canonicalPath(state?.cwd) === canonicalPath(cwd)
return codexWorkerOwnerMatches(state, cwd)
&& typeof state?.threadId === 'string'
&& state.threadId.length > 0;
}
export function codexWorkerProcessStateIsOwned(state, cwd) {
return codexWorkerOwnerMatches(state, cwd)
&& Number.isInteger(state?.pid)
&& state.pid > 0;
}
function codexWorkerOwnerMatches(state, cwd) {
return state?.owner === CODEX_WORKER_OWNER
&& canonicalPath(state?.cwd) === canonicalPath(cwd);
}
function canonicalPath(value) {
if (!value || typeof value !== 'string') return null;
const resolved = path.resolve(value);