mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-14 23:26:39 +03:00
Publish Codex Live quality evidence
Update Live Lab and the Live reference with the default Sol worker, full-task quality gate, Spark control, cold readiness, and production architecture.\n\nAI-assisted: OpenAI Codex.
This commit is contained in:
@@ -1,9 +1,12 @@
|
||||
export const liveCodexWorkerResult = {
|
||||
generatedAt: '2026-07-13T01:05:15.178Z',
|
||||
generatedAt: '2026-07-13T03:03:13.899Z',
|
||||
codexVersion: '0.144.0-alpha.4',
|
||||
transport: 'stdio JSONL',
|
||||
model: 'GPT-5.3-Codex-Spark',
|
||||
effort: 'low',
|
||||
model: 'GPT-5.6-Sol',
|
||||
effort: 'medium',
|
||||
profile: 'quality',
|
||||
defaultInCodex: true,
|
||||
coldWorkerReadyMs: 5600,
|
||||
timings: {
|
||||
coldHandshakeMs: 36.83,
|
||||
coldThreadStartMs: 400.19,
|
||||
@@ -11,7 +14,24 @@ export const liveCodexWorkerResult = {
|
||||
wakeToTurnStartedMs: 1.47,
|
||||
trivialTurnCompletedMs: 2060.18,
|
||||
},
|
||||
result: 'READY',
|
||||
protocolProbe: 'GPT-5.3-Codex-Spark · low',
|
||||
quality: {
|
||||
tasks: ['bolder', 'polish'],
|
||||
runs: 4,
|
||||
passed: 4,
|
||||
medianGenerationMs: 34339,
|
||||
p95GenerationMs: 42780.1,
|
||||
averageScores: {
|
||||
commandFidelity: 8,
|
||||
brandAndSystemFidelity: 8.5,
|
||||
frontendQuality: 7.5,
|
||||
taskCompletion: 7.75,
|
||||
},
|
||||
sparkControlPassed: 1,
|
||||
sparkControlRuns: 2,
|
||||
sparkMedianGenerationMs: 16470.5,
|
||||
},
|
||||
result: '4/4 full-task quality gate',
|
||||
passed: true,
|
||||
architecture: 'A dedicated persistent Live app-server connection and worker thread. Do not attach a second app-server to the desktop task.',
|
||||
architecture: 'A dedicated persistent Live app-server connection and worker thread with native Impeccable skill, sub-command, project context, and annotation-image inputs. Do not attach a second app-server to the desktop task.',
|
||||
} as const;
|
||||
|
||||
@@ -3,8 +3,8 @@ export const harnessPaths = [
|
||||
name: 'Dedicated app-server worker',
|
||||
harness: 'Codex · isolated persistent thread',
|
||||
pickup: '1.47 ms turn wake',
|
||||
tradeoff: 'A separate Live-owned thread can generate while the control supervisor keeps polling. Cold setup is 437 ms; a persistent connection amortizes it.',
|
||||
status: 'Protocol probe passed',
|
||||
tradeoff: 'A separate Live-owned Sol thread generates while the control supervisor keeps polling. Full context protects design quality; the 5.6 s cold worker startup happens during Live initialization.',
|
||||
status: 'Codex default · gated',
|
||||
statusTone: 'ready',
|
||||
source: 'https://learn.chatgpt.com/docs/app-server',
|
||||
},
|
||||
@@ -136,11 +136,11 @@ export const liveExperiments = [
|
||||
},
|
||||
{
|
||||
rank: 13,
|
||||
title: 'Wake a dedicated Codex worker — protocol proven',
|
||||
evidence: 'A cold app-server handshake took 36.83 ms, thread creation 400.19 ms, and turn/start reached turn/started in 1.47 ms. A low-effort Spark probe completed in 2.06 s.',
|
||||
move: 'Keep one Live-owned app-server connection and isolated worker thread warm. Feed it durable events and structured output; never resume the desktop task from a second server.',
|
||||
expected: 'Removes terminal-output pickup and main-turn availability from generation startup while preserving transactional publication fencing.',
|
||||
confidence: 'Official protocol + live probe',
|
||||
title: 'Wake a dedicated Codex worker — shipped',
|
||||
evidence: 'The production Sol/medium path passed 4/4 full bolder + polish tasks after fidelity tuning (34.3 s median, 42.8 s p95). The Spark control passed only 1/2 despite a 16.5 s median. Turn wake itself is 1.47 ms.',
|
||||
move: 'Keep one Live-owned app-server connection and isolated quality-model thread warm. Attach the real skill, sub-command, resolved project context, source neighborhood, and annotation image; never resume the desktop task from a second server.',
|
||||
expected: 'Removes main-turn availability from generation and allows independent progressive delivery without trading away brand fidelity.',
|
||||
confidence: 'Full-task model gate + protocol smoke',
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
@@ -404,9 +404,9 @@ const providerColumns = [
|
||||
<p>{currentHarnessProbe.surface}: {currentHarnessProbe.result}</p>
|
||||
</div>
|
||||
<div class="live-performance-finding live-finding-card--quiet" role="note">
|
||||
<strong>Dedicated Codex worker probe</strong>
|
||||
<strong>Dedicated Codex worker</strong>
|
||||
<p>
|
||||
{liveCodexWorkerResult.model} at {liveCodexWorkerResult.effort} effort started a dedicated turn in {displayMs(liveCodexWorkerResult.timings.wakeToTurnStartedMs)} and completed a trivial structured response in {displayMs(liveCodexWorkerResult.timings.trivialTurnCompletedMs)}. Cold app-server plus thread setup cost {displayMs(liveCodexWorkerResult.timings.coldHandshakeMs + liveCodexWorkerResult.timings.coldThreadStartMs)} and can be amortized. The safe path owns a separate Live thread; it never resumes the desktop task from a competing app-server.
|
||||
Codex now defaults to {liveCodexWorkerResult.model} at {liveCodexWorkerResult.effort} effort in a separate Live-owned thread. It passed {liveCodexWorkerResult.quality.passed}/{liveCodexWorkerResult.quality.runs} full <code>/bolder</code> and <code>/polish</code> tasks at a {displayMs(liveCodexWorkerResult.quality.medianGenerationMs)} median; the Spark control passed {liveCodexWorkerResult.quality.sparkControlPassed}/{liveCodexWorkerResult.quality.sparkControlRuns}. Protocol wake is {displayMs(liveCodexWorkerResult.timings.wakeToTurnStartedMs)}; the measured cold worker reached ready in {displayMs(liveCodexWorkerResult.coldWorkerReadyMs)} during Live initialization.
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
@@ -470,9 +470,10 @@ const providerColumns = [
|
||||
<header><h2>Dedicated Codex worker</h2></header>
|
||||
<dl>
|
||||
<div><dt>Wake</dt><dd>{displayMs(liveCodexWorkerResult.timings.wakeToTurnStartedMs)}</dd></div>
|
||||
<div><dt>Cold setup</dt><dd>{displayMs(liveCodexWorkerResult.timings.coldHandshakeMs + liveCodexWorkerResult.timings.coldThreadStartMs)}</dd></div>
|
||||
<div><dt>Full-task median</dt><dd>{displayMs(liveCodexWorkerResult.quality.medianGenerationMs)}</dd></div>
|
||||
<div><dt>Quality gate</dt><dd>{liveCodexWorkerResult.quality.passed}/{liveCodexWorkerResult.quality.runs}</dd></div>
|
||||
</dl>
|
||||
<p>Live owns the worker thread. It never resumes the desktop task.</p>
|
||||
<p>Default in Codex; other harnesses keep the foreground path.</p>
|
||||
</section>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
@@ -99,9 +99,9 @@ node {{scripts_path}}/live-complete.mjs --id SESSION_ID
|
||||
|
||||
Server restart rule: start `live-server.mjs` again, then poll. Startup requeues unacknowledged pending events from the journal, so do not ask the user to click Go again unless `live-resume.mjs` says no active session exists.
|
||||
|
||||
### Experimental dedicated Codex worker
|
||||
### Dedicated Codex worker
|
||||
|
||||
Codex can opt into a Live-owned persistent app-server supervisor instead of using the desktop task as the poll supervisor:
|
||||
Codex uses a Live-owned persistent app-server supervisor instead of using the desktop task as the poll supervisor:
|
||||
|
||||
```bash
|
||||
node {{scripts_path}}/live.mjs
|
||||
|
||||
@@ -148,7 +148,7 @@ export class CodexLiveWorkerSupervisor {
|
||||
id: event.id,
|
||||
type: 'error',
|
||||
sourceEventType: event.type,
|
||||
message: `Experimental Codex worker does not handle ${event.type}; disable IMPECCABLE_LIVE_CODEX_WORKER for the portable foreground path.`,
|
||||
message: `Dedicated Codex worker does not handle ${event.type}; disable IMPECCABLE_LIVE_CODEX_WORKER for the portable foreground path.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user