Prevent duplicate long-running Live turns

Keep short crash-recovery leases without allowing a healthy worker to queue its own generation twice, and surface non-monotonic benchmark journals as errors.\n\nAI-assisted: OpenAI Codex.
This commit is contained in:
Paul Bakaus
2026-07-12 21:18:20 -07:00
parent 960f4b725b
commit 1e4927d86a
4 changed files with 52 additions and 6 deletions
+10 -5
View File
@@ -148,17 +148,22 @@ export function buildInteractionRun(events, { iteration, scenario, goStartedAt,
export function deriveJournalGenerationMetrics(snapshot = {}) {
const timings = snapshot.generationTimings || {};
const at = (phase) => Number(timings[phase]?.at);
const delta = (start, end) => (
Number.isFinite(at(start)) && Number.isFinite(at(end))
? roundMs(Math.max(0, at(end) - at(start)))
: null
);
const timingErrors = [];
const delta = (start, end) => {
if (!Number.isFinite(at(start)) || !Number.isFinite(at(end))) return null;
if (at(end) < at(start)) {
timingErrors.push(`${end}_before_${start}`);
return null;
}
return roundMs(at(end) - at(start));
};
return {
workerPickupToSourceReadyMs: delta('picked_up', 'source_ready'),
workerFirstGenerationToReviewableMs: delta('first_variant_generating', 'first_reviewable'),
workerFirstValidationToReviewableMs: delta('first_variant_validating', 'first_reviewable'),
workerRemainingGenerationToReadyMs: delta('remaining_variants_generating', 'all_variants_ready'),
workerRemainingValidationToReadyMs: delta('remaining_variants_validating', 'all_variants_ready'),
journalTimingErrors: timingErrors,
journalGenerationTimings: timings,
};
}
@@ -69,6 +69,7 @@ export class CodexLiveWorkerSupervisor {
this.queue = Promise.resolve();
this.active = null;
this.canceled = new Set();
this.queuedGenerationIds = new Set();
this.thread = null;
this.model = null;
this.liveSpec = '';
@@ -143,9 +144,12 @@ export class CodexLiveWorkerSupervisor {
continue;
}
if (event.type === 'generate') {
if (this.queuedGenerationIds.has(event.id)) continue;
this.queuedGenerationIds.add(event.id);
this.queue = this.queue
.then(() => this.processGeneration(event))
.catch((error) => this.handleGenerationFailure(event, error));
.catch((error) => this.handleGenerationFailure(event, error))
.finally(() => this.queuedGenerationIds.delete(event.id));
continue;
}
if (event.type === 'prefetch') continue;
+12
View File
@@ -30,6 +30,18 @@ describe('live benchmark metrics', () => {
assert.equal(metrics.workerFirstValidationToReviewableMs, 30);
assert.equal(metrics.workerRemainingGenerationToReadyMs, 185);
assert.equal(metrics.workerRemainingValidationToReadyMs, 30);
assert.deepEqual(metrics.journalTimingErrors, []);
});
it('surfaces non-monotonic worker phases instead of reporting a false zero', () => {
const metrics = deriveJournalGenerationMetrics({
generationTimings: {
first_variant_generating: { at: 300 },
first_reviewable: { at: 200 },
},
});
assert.equal(metrics.workerFirstGenerationToReviewableMs, null);
assert.deepEqual(metrics.journalTimingErrors, ['first_reviewable_before_first_variant_generating']);
});
it('keeps published progressive CSS byte-stable and carries deferred params', () => {
@@ -160,6 +160,31 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
}]);
});
it('renews but never queues the same long-running generation twice', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-duplicate-lease-'));
const client = fakeClient();
const supervisor = createSupervisor({
cwd,
statePath: path.join(cwd, 'state.json'),
client,
});
supervisor.thread = { id: 'live-worker-thread' };
let generations = 0;
supervisor.processGeneration = async () => {
generations += 1;
await new Promise((resolve) => setImmediate(resolve));
};
const events = [
{ type: 'generate', id: 'generation-1', count: 3 },
{ type: 'generate', id: 'generation-1', count: 3 },
{ type: 'exit' },
];
supervisor.fetchEvent = async () => events.shift();
await supervisor.run();
assert.equal(generations, 1);
assert.equal(supervisor.queuedGenerationIds.size, 0);
});
it('reconnects and resumes the owned worker once after app-server loss', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-reconnect-'));
const client = fakeClient();