mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-17 16:46:31 +03:00
Prevent duplicate long-running Live turns
Keep short crash-recovery leases without allowing a healthy worker to queue its own generation twice, and surface non-monotonic benchmark journals as errors.\n\nAI-assisted: OpenAI Codex.
This commit is contained in:
@@ -148,17 +148,22 @@ export function buildInteractionRun(events, { iteration, scenario, goStartedAt,
|
||||
export function deriveJournalGenerationMetrics(snapshot = {}) {
|
||||
const timings = snapshot.generationTimings || {};
|
||||
const at = (phase) => Number(timings[phase]?.at);
|
||||
const delta = (start, end) => (
|
||||
Number.isFinite(at(start)) && Number.isFinite(at(end))
|
||||
? roundMs(Math.max(0, at(end) - at(start)))
|
||||
: null
|
||||
);
|
||||
const timingErrors = [];
|
||||
const delta = (start, end) => {
|
||||
if (!Number.isFinite(at(start)) || !Number.isFinite(at(end))) return null;
|
||||
if (at(end) < at(start)) {
|
||||
timingErrors.push(`${end}_before_${start}`);
|
||||
return null;
|
||||
}
|
||||
return roundMs(at(end) - at(start));
|
||||
};
|
||||
return {
|
||||
workerPickupToSourceReadyMs: delta('picked_up', 'source_ready'),
|
||||
workerFirstGenerationToReviewableMs: delta('first_variant_generating', 'first_reviewable'),
|
||||
workerFirstValidationToReviewableMs: delta('first_variant_validating', 'first_reviewable'),
|
||||
workerRemainingGenerationToReadyMs: delta('remaining_variants_generating', 'all_variants_ready'),
|
||||
workerRemainingValidationToReadyMs: delta('remaining_variants_validating', 'all_variants_ready'),
|
||||
journalTimingErrors: timingErrors,
|
||||
journalGenerationTimings: timings,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -69,6 +69,7 @@ export class CodexLiveWorkerSupervisor {
|
||||
this.queue = Promise.resolve();
|
||||
this.active = null;
|
||||
this.canceled = new Set();
|
||||
this.queuedGenerationIds = new Set();
|
||||
this.thread = null;
|
||||
this.model = null;
|
||||
this.liveSpec = '';
|
||||
@@ -143,9 +144,12 @@ export class CodexLiveWorkerSupervisor {
|
||||
continue;
|
||||
}
|
||||
if (event.type === 'generate') {
|
||||
if (this.queuedGenerationIds.has(event.id)) continue;
|
||||
this.queuedGenerationIds.add(event.id);
|
||||
this.queue = this.queue
|
||||
.then(() => this.processGeneration(event))
|
||||
.catch((error) => this.handleGenerationFailure(event, error));
|
||||
.catch((error) => this.handleGenerationFailure(event, error))
|
||||
.finally(() => this.queuedGenerationIds.delete(event.id));
|
||||
continue;
|
||||
}
|
||||
if (event.type === 'prefetch') continue;
|
||||
|
||||
@@ -30,6 +30,18 @@ describe('live benchmark metrics', () => {
|
||||
assert.equal(metrics.workerFirstValidationToReviewableMs, 30);
|
||||
assert.equal(metrics.workerRemainingGenerationToReadyMs, 185);
|
||||
assert.equal(metrics.workerRemainingValidationToReadyMs, 30);
|
||||
assert.deepEqual(metrics.journalTimingErrors, []);
|
||||
});
|
||||
|
||||
it('surfaces non-monotonic worker phases instead of reporting a false zero', () => {
|
||||
const metrics = deriveJournalGenerationMetrics({
|
||||
generationTimings: {
|
||||
first_variant_generating: { at: 300 },
|
||||
first_reviewable: { at: 200 },
|
||||
},
|
||||
});
|
||||
assert.equal(metrics.workerFirstGenerationToReviewableMs, null);
|
||||
assert.deepEqual(metrics.journalTimingErrors, ['first_reviewable_before_first_variant_generating']);
|
||||
});
|
||||
|
||||
it('keeps published progressive CSS byte-stable and carries deferred params', () => {
|
||||
|
||||
@@ -160,6 +160,31 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
|
||||
}]);
|
||||
});
|
||||
|
||||
it('renews but never queues the same long-running generation twice', async () => {
|
||||
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-duplicate-lease-'));
|
||||
const client = fakeClient();
|
||||
const supervisor = createSupervisor({
|
||||
cwd,
|
||||
statePath: path.join(cwd, 'state.json'),
|
||||
client,
|
||||
});
|
||||
supervisor.thread = { id: 'live-worker-thread' };
|
||||
let generations = 0;
|
||||
supervisor.processGeneration = async () => {
|
||||
generations += 1;
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
};
|
||||
const events = [
|
||||
{ type: 'generate', id: 'generation-1', count: 3 },
|
||||
{ type: 'generate', id: 'generation-1', count: 3 },
|
||||
{ type: 'exit' },
|
||||
];
|
||||
supervisor.fetchEvent = async () => events.shift();
|
||||
await supervisor.run();
|
||||
assert.equal(generations, 1);
|
||||
assert.equal(supervisor.queuedGenerationIds.size, 0);
|
||||
});
|
||||
|
||||
it('reconnects and resumes the owned worker once after app-server loss', async () => {
|
||||
const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-reconnect-'));
|
||||
const client = fakeClient();
|
||||
|
||||
Reference in New Issue
Block a user