Harden Live worker recovery

AI-assisted: Codex
2026-09-20 10:06:54 +03:00 · 2026-07-13 10:44:37 -07:00
parent a274f93c4e
commit c6dfd22329
7 changed files with 296 additions and 21 deletions
@@ -74,6 +74,33 @@ describe('live-accept — style-element edge cases', () => {
    assert.ok(!after.includes('original text'), 'original content dropped');
  });

+  it('replays a durable receipt when Accept is retried after source was already written', () => {
+    const html = `<body>
+  <!-- impeccable-variants-start RECEIPT1 -->
+  <div data-impeccable-variants="RECEIPT1" data-impeccable-variant-count="2" style="display: contents">
+    <div data-impeccable-variant="original"><p>original</p></div>
+    <style data-impeccable-css="RECEIPT1" />
+    <div data-impeccable-variant="1"><p>accepted once</p></div>
+    <div data-impeccable-variant="2" style="display: none"><p>other</p></div>
+  </div>
+  <!-- impeccable-variants-end RECEIPT1 -->
+</body>`;
+    writeFileSync(join(tmp, 'page.html'), html);
+
+    const first = runAccept(tmp, ['--id', 'RECEIPT1', '--variant', '1']);
+    const afterFirst = readFileSync(join(tmp, 'page.html'), 'utf-8');
+    const replay = runAccept(tmp, ['--id', 'RECEIPT1', '--variant', '1']);
+
+    assert.equal(first.handled, true);
+    assert.equal(replay.handled, true);
+    assert.equal(replay.alreadyApplied, true);
+    assert.equal(replay.file, 'page.html');
+    assert.equal(readFileSync(join(tmp, 'page.html'), 'utf-8'), afterFirst);
+    const conflict = runAccept(tmp, ['--id', 'RECEIPT1', '--variant', '2']);
+    assert.equal(conflict.handled, false);
+    assert.equal(conflict.error, 'accept_receipt_conflict');
+  });
+
  // Variant: same-line <style>…</style> block should also be treated as a
  // single skipped unit; the line has both open and close tags.
  it('finds the accepted variant after a single-line <style>…</style> block', () => {
@@ -207,6 +207,7 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
  it('queues carbonize cleanup onto the foreground control lane', async () => {
    const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-carbonize-'));
    const cleanups = [];
+    const order = [];
    const client = fakeClient();
    const supervisor = new CodexLiveWorkerSupervisor({
      cwd,
@@ -219,8 +220,10 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
      handleAccept: async (event) => ({
        ...event,
        _acceptResult: { handled: true, carbonize: true, file: 'src/App.jsx' },
+        _completionAck: { ok: false, deferred: true },
      }),
-      postCleanup: async (_base, _token, event) => { cleanups.push(event); },
+      postCleanup: async (_base, _token, event) => { cleanups.push(event); order.push('cleanup'); },
+      completeAccept: async () => { order.push('accept_ack'); },
    });
    supervisor.running = true;
    supervisor.thread = { id: 'live-worker-thread' };
@@ -233,11 +236,13 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
    };
    await supervisor.run();
    assert.deepEqual(cleanups, [{
+      id: 'abc12345',
      sessionId: 'abc12345',
      file: 'src/App.jsx',
      variantId: '1',
      acceptResult: { handled: true, carbonize: true, file: 'src/App.jsx' },
    }]);
+    assert.deepEqual(order, ['cleanup', 'accept_ack']);
  });

  it('renews but never queues the same long-running generation twice', async () => {
@@ -288,6 +293,63 @@ describe('Codex Live worker supervisor ownership and lifecycle', () => {
    assert.equal(client.calls.resumeDedicatedThread.length, 1);
  });

+  it('relinquishes Generate and advertises foreground fallback after permanent failure', async () => {
+    const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-fallback-'));
+    const replies = [];
+    const statePath = path.join(cwd, 'state.json');
+    const supervisor = new CodexLiveWorkerSupervisor({
+      cwd,
+      base: 'http://localhost:1',
+      token: 'token',
+      client: fakeClient(),
+      config: { model: null, effort: 'low', delivery: 'progressive', maxArtifactBytes: 2_000_000 },
+      statePath,
+      scriptsDir: path.join(cwd, 'skill/scripts'),
+      reply: async (_base, _token, value) => { replies.push(value); },
+    });
+    supervisor.running = true;
+    supervisor.pollAbortController = new AbortController();
+
+    await supervisor.handleGenerationFailure(
+      { type: 'generate', id: 'recoverable-generation' },
+      new Error('app-server remained unavailable after reconnect'),
+    );
+
+    assert.equal(supervisor.running, false);
+    assert.equal(supervisor.pollAbortController.signal.aborted, true);
+    assert.deepEqual(replies, [{
+      id: 'recoverable-generation',
+      type: 'retry',
+      sourceEventType: 'generate',
+    }]);
+    const state = JSON.parse(readFileSync(statePath, 'utf-8'));
+    assert.equal(state.status, 'failed');
+    assert.equal(state.eventId, 'recoverable-generation');
+    assert.match(state.error, /app-server remained unavailable/);
+  });
+
+  it('re-prepares once when foreground cleanup changes source during generation', async () => {
+    const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-source-race-'));
+    const supervisor = createSupervisor({
+      cwd,
+      statePath: path.join(cwd, 'state.json'),
+      client: fakeClient(),
+    });
+    let attempts = 0;
+    supervisor.runGenerationPhaseOnce = async () => {
+      attempts += 1;
+      if (attempts === 1) {
+        const error = new Error('publish_source_hash_mismatch');
+        error.code = 'publish_source_hash_mismatch';
+        throw error;
+      }
+    };
+
+    await supervisor.runGenerationPhase({ id: 'source-race' }, 'first', 1);
+
+    assert.equal(attempts, 2);
+  });
+
  it('resumes progressive delivery from durable variant checkpoints', async () => {
    const cwd = mkdtempSync(path.join(tmpdir(), 'codex-supervisor-checkpoint-resume-'));
    const phases = [];
@@ -2652,6 +2652,52 @@ colors: {}
    }
  });

+  it('releases a failed worker Generate lease without consuming or broadcasting it', async () => {
+    await drainPolls(server);
+    const id = 'fa11bac1';
+    const generated = await fetch(`http://localhost:${server.port}/events`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        token: server.token,
+        type: 'generate',
+        id,
+        action: 'bolder',
+        count: 3,
+        element: { outerHTML: '<article>fallback</article>', tagName: 'article' },
+      }),
+    });
+    assert.equal(generated.status, 200);
+    const leased = await fetch(`http://localhost:${server.port}/poll?token=${server.token}&types=generate&timeout=100&leaseMs=5000`).then((response) => response.json());
+    assert.equal(leased.id, id);
+
+    const retried = await fetch(`http://localhost:${server.port}/poll`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        token: server.token,
+        id,
+        type: 'retry',
+        sourceEventType: 'generate',
+      }),
+    });
+    assert.equal(retried.status, 200);
+    assert.equal((await retried.json()).released, true);
+
+    const fallback = await fetch(`http://localhost:${server.port}/poll?token=${server.token}&types=generate&timeout=100&leaseMs=100`).then((response) => response.json());
+    assert.equal(fallback.id, id);
+    assert.equal(fallback.type, 'generate');
+    const status = await fetch(`http://localhost:${server.port}/status?token=${server.token}`).then((response) => response.json());
+    assert.equal(status.pendingEvents.some((event) => event.id === id && event.type === 'generate'), true);
+
+    const done = await fetch(`http://localhost:${server.port}/poll`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ token: server.token, id, type: 'done', sourceEventType: 'generate' }),
+    });
+    assert.equal(done.status, 200);
+  });
+
  it('wakes a parked poll as soon as a missed-ack lease expires', async () => {
    await drainPolls(server);