Benchmark the production Codex Live worker

Let the browser E2E harness launch an independent production worker, exercise real sub-command selection, and carry realistic product/design context.\n\nAI-assisted: OpenAI Codex.
2026-09-17 00:26:41 +03:00 · 2026-07-12 20:50:28 -07:00
parent edde928736
commit aba1067cdf
6 changed files with 157 additions and 25 deletions
@@ -219,6 +219,10 @@ export async function stopDevServer(child) {
 * @param {import('playwright').Browser} opts.browser   shared browser instance
 * @param {object} opts.agent             VariantAgent (defaults to fake)
 * @param {object|function=} opts.wrapTarget live-wrap target or event mapper
+ * @param {(context: object) => Promise<object|void>} [opts.startWorker]
+ *        Optional production worker factory. Return {stop, done}; when used,
+ *        omit `agent` so the deterministic in-process loop is not started.
+ * @param {(context: object) => Promise<void>|void} [opts.prepareTmp]
 * @param {(msg: string) => void} [opts.log]
 */
 export async function bootFixtureSession({
@@ -227,6 +231,8 @@ export async function bootFixtureSession({
  browser,
  agent,
  wrapTarget,
+  startWorker,
+  prepareTmp,
  log = () => {},
  trace = () => {},
  progressive = false,
@@ -242,12 +248,15 @@ export async function bootFixtureSession({
  let dev;
  let agentAbort;
  let agentDone;
+  let externalWorker;
  let ctx;

  const teardown = async () => {
    try { if (ctx) await ctx.close(); } catch {}
    try { if (agentAbort) agentAbort.abort(); } catch {}
    try { if (agentDone) await agentDone.catch(() => {}); } catch {}
+    try { if (externalWorker?.stop) await externalWorker.stop(); } catch {}
+    try { if (externalWorker?.done) await externalWorker.done.catch(() => {}); } catch {}
    try { if (dev?.child) await stopDevServer(dev.child); } catch {}
    try { if (live) stopLiveServer(tmp); } catch {}
    try { rmSync(tmp, { recursive: true, force: true }); } catch {}
@@ -261,6 +270,7 @@ export async function bootFixtureSession({

  try {
    const startedAt = Date.now();
+    if (prepareTmp) await prepareTmp({ tmp, fixture, scriptsDir: SCRIPTS_DIR, trace, log });
    trace('setup.install.start', { fixture: name });
    log(`installing deps`);
    runInstall(tmp, runtime.install);
@@ -274,6 +284,12 @@ export async function bootFixtureSession({
    trace('setup.live_server.end', { fixture: name, port: live.port });
    log(`live-server ready in ${formatDuration(Date.now() - liveStartedAt)}`);

+    if (startWorker) {
+      trace('setup.worker.start', { fixture: name });
+      externalWorker = await startWorker({ tmp, fixture, scriptsDir: SCRIPTS_DIR, live, trace, log });
+      trace('setup.worker.end', { fixture: name });
+    }
+
    const injectStartedAt = Date.now();
    trace('setup.inject.start', { fixture: name });
    log(`live-inject --port ${live.port}`);
@@ -291,28 +307,30 @@ export async function bootFixtureSession({
    log(`dev server ready on ${devPort} in ${formatDuration(Date.now() - devStartedAt)}`);

    // Agent loop runs concurrently — abort on teardown.
-    agentAbort = new AbortController();
-    const loopOptions = {
-      tmp,
-      scriptsDir: SCRIPTS_DIR,
-      port: live.port,
-      token: live.token,
-      agent,
-      wrapTarget,
-      signal: agentAbort.signal,
-      trace,
-      progressive,
-      progressiveDelayMs,
-      progressiveInitialCount,
-      atomicDelayMs,
-      steerSourceFile: runtime.steer?.sourceFile,
-      steerTarget: runtime.steer?.target,
-    };
-    const loops = [runAgentLoop({ ...loopOptions, log: (m) => log('[worker] ' + m) })];
-    if (progressive) {
-      loops.push(runAgentLoop({ ...loopOptions, log: (m) => log('[supervisor] ' + m) }));
+    if (agent) {
+      agentAbort = new AbortController();
+      const loopOptions = {
+        tmp,
+        scriptsDir: SCRIPTS_DIR,
+        port: live.port,
+        token: live.token,
+        agent,
+        wrapTarget,
+        signal: agentAbort.signal,
+        trace,
+        progressive,
+        progressiveDelayMs,
+        progressiveInitialCount,
+        atomicDelayMs,
+        steerSourceFile: runtime.steer?.sourceFile,
+        steerTarget: runtime.steer?.target,
+      };
+      const loops = [runAgentLoop({ ...loopOptions, log: (m) => log('[worker] ' + m) })];
+      if (progressive) {
+        loops.push(runAgentLoop({ ...loopOptions, log: (m) => log('[supervisor] ' + m) }));
+      }
+      agentDone = Promise.all(loops);
    }
-    agentDone = Promise.all(loops);

    const scheme = runtime.scheme || 'http';
    ctx = await browser.newContext({
@@ -342,6 +360,7 @@ export async function bootFixtureSession({
      ctx,
      dev,
      live,
+      worker: externalWorker,
      consoleErrors,
      stopLiveServer: stopLiveForDeferredWork,
      teardown,
@@ -544,6 +544,36 @@ export async function setCount(page, count) {
  throw new Error(`could not cycle count to ${count}`);
 }

+/** Select a named Impeccable sub-command from the configure-row picker. */
+export async function selectAction(page, action) {
+  const pickerSelector = '#impeccable-live-picker';
+  const opened = await page.evaluate(({ barSel, pickerSel }) => {
+    const query = window.__impeccableLiveQuery || ((selector) => document.querySelector(selector));
+    const bar = query(barSel);
+    const picker = query(pickerSel);
+    const actionControl = [...(bar?.querySelectorAll('button') || [])]
+      .find((button) => (button.textContent || '').includes('\u25BE'));
+    if (!actionControl || !picker) return false;
+    actionControl.click();
+    return true;
+  }, { barSel: BAR_ID, pickerSel: pickerSelector });
+  if (!opened) throw new Error('could not open Live action picker');
+
+  await page.waitForFunction((selector) => {
+    const picker = window.__impeccableLiveQuery(selector);
+    return picker && picker.style.display !== 'none';
+  }, pickerSelector, { timeout: 5_000 });
+
+  const selected = await page.evaluate(({ pickerSel, value }) => {
+    const picker = window.__impeccableLiveQuery(pickerSel);
+    const chip = picker?.querySelector(`button[data-action="${CSS.escape(value)}"]`);
+    if (!chip) return false;
+    chip.click();
+    return true;
+  }, { pickerSel: pickerSelector, value: action });
+  if (!selected) throw new Error(`Live action ${JSON.stringify(action)} is unavailable`);
+}
+
 /**
 * Click Go. Browser POSTs the generate event; the agent picks it up. Headed
 * browser runs can occasionally accept the click without leaving configure