mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-17 00:26:41 +03:00
490 lines
28 KiB
Plaintext
490 lines
28 KiB
Plaintext
---
|
||
import Base from '../../layouts/Base.astro';
|
||
import LiveUiGallery from '../../components/LiveUiGallery.astro';
|
||
import benchmarkData from '../../data/live-performance.json';
|
||
import { currentHarnessProbe, harnessPaths, liveExperiments } from '../../data/live-harnesses';
|
||
import { progressiveDeliveryResult } from '../../data/live-progressive-result';
|
||
import { liveInitResult } from '../../data/live-init-result';
|
||
import { liveControlResult } from '../../data/live-control-result';
|
||
import { liveFrameworkResults } from '../../data/live-framework-results';
|
||
import { liveProviderResult } from '../../data/live-provider-result';
|
||
import { liveCodexWorkerResult } from '../../data/live-codex-worker-result';
|
||
import { liveAnnotatedResult } from '../../data/live-annotated-result';
|
||
import '../../styles/sub-pages.css';
|
||
import '../../styles/live-performance.css';
|
||
import '../../styles/live-lab-workbench.css';
|
||
|
||
const reports = 'reports' in benchmarkData ? benchmarkData.reports : [benchmarkData];
|
||
const plainReports = reports.filter(report => report.benchmark.scenario === 'plain' && report.benchmark.agent === 'fake');
|
||
const annotatedReports = reports.filter(report => report.benchmark.scenario === 'annotated' && report.benchmark.agent === 'fake');
|
||
const baseline = plainReports[0] || reports[0];
|
||
const plain = plainReports.at(-1) || baseline;
|
||
const annotated = annotatedReports.at(-1);
|
||
const metric = (name: string) => plain.summary.metrics[name]?.median || 0;
|
||
const protocolFloor = metric('goToFirstVariantMs');
|
||
const baselineFloor = baseline.summary.metrics.goToFirstVariantMs?.median || protocolFloor;
|
||
const browserDispatch = metric('browserDispatchMs');
|
||
const automationClick = metric('automationClickMs');
|
||
const baselineAutomationClick = baseline.summary.metrics.automationClickMs?.median ?? automationClick;
|
||
const serverPickup = metric('serverPickupMs');
|
||
const scaffold = metric('scaffoldMs');
|
||
const writeAndRender = metric('writeMs') + metric('writeToFirstVariantMs');
|
||
const productFloor = Math.max(0, protocolFloor - automationClick);
|
||
const baselineProductFloor = Math.max(0, baselineFloor - baselineAutomationClick);
|
||
const improvement = baselineFloor ? Math.max(0, 1 - (protocolFloor / baselineFloor)) : 0;
|
||
const annotatedFloor = annotated?.summary.metrics.goToFirstVariantMs?.median || 0;
|
||
const maxScenarioFloor = Math.max(baselineFloor, protocolFloor, annotatedFloor, 1);
|
||
const optimizedTapeWidth = Math.max(12, (protocolFloor / baselineFloor) * 952);
|
||
const displayMs = (value: number) => {
|
||
if (value >= 1000) return `${(value / 1000).toFixed(value >= 10_000 ? 1 : 2)} s`;
|
||
if (value < 1) return '<1 ms';
|
||
return `${Math.round(value)} ms`;
|
||
};
|
||
|
||
const stageData = [
|
||
{ name: 'Playwright actionability', value: automationClick, tone: 'capture' },
|
||
{ name: 'Live dispatch + pickup', value: browserDispatch + serverPickup, tone: 'pickup' },
|
||
{ name: 'Source scaffold', value: scaffold, tone: 'scaffold' },
|
||
{ name: 'Write + browser settle', value: writeAndRender, tone: 'render' },
|
||
];
|
||
const progressiveGain = 1 - (
|
||
progressiveDeliveryResult.progressive.medianFirstMs
|
||
/ progressiveDeliveryResult.atomic.medianFirstMs
|
||
);
|
||
const progressiveP95Gain = 1 - (
|
||
progressiveDeliveryResult.progressive.p95FirstMs
|
||
/ progressiveDeliveryResult.atomic.p95FirstMs
|
||
);
|
||
const progressiveAllDelta = (
|
||
progressiveDeliveryResult.progressive.medianAllMs
|
||
- progressiveDeliveryResult.atomic.medianAllMs
|
||
);
|
||
const initGain = 1 - (liveInitResult.cold.medianMs / liveInitResult.baselineColdMedianMs);
|
||
const providerColumns = [
|
||
{ key: 'atomic', label: 'Atomic full' },
|
||
{ key: 'progressiveFull', label: 'Progressive full' },
|
||
{ key: 'progressiveCompact', label: 'Progressive compact' },
|
||
{ key: 'parallelCompact', label: 'Parallel compact' },
|
||
] as const;
|
||
---
|
||
|
||
<Base
|
||
title="Live Latency Lab | Impeccable"
|
||
description="Measured latency for Impeccable Live, from Go to the first usable variant."
|
||
bodyClass="sub-page live-performance-page live-lab-page"
|
||
mainClass="live-performance-main live-lab-main-shell"
|
||
noIndex
|
||
hideHeader
|
||
hideFooter
|
||
>
|
||
<section class="live-lab-workbench" aria-label="Live performance workbench" data-live-lab-workbench>
|
||
<aside class="live-lab-nav" aria-label="Live lab navigation">
|
||
<div class="live-lab-nav-head">
|
||
<a class="live-lab-brand ks-brand" href="/live" aria-label="Back to Live Mode">
|
||
<span class="ks-mark" aria-hidden="true">
|
||
<svg viewBox="0 0 24 24" fill="currentColor">
|
||
<path d="M5 2.5 L13.5 2.5 L5.5 21.5 L5 21.5 Q2.5 21.5 2.5 19 L2.5 5 Q2.5 2.5 5 2.5 Z"></path>
|
||
<path d="M16.5 2.5 L19 2.5 Q21.5 2.5 21.5 5 L21.5 19 Q21.5 21.5 19 21.5 L8.5 21.5 Z"></path>
|
||
</svg>
|
||
</span>
|
||
<span class="ks-wordmark">Impeccable</span>
|
||
</a>
|
||
<h1>Live Lab</h1>
|
||
<p>Measure Live and inspect its interface.</p>
|
||
</div>
|
||
<nav class="live-lab-nav-scroll" aria-label="Workbench views">
|
||
<div class="live-lab-nav-group">
|
||
<h2>Views</h2>
|
||
<div class="live-lab-nav-list">
|
||
<button type="button" data-lab-view="overview" aria-pressed="true">Performance</button>
|
||
<button type="button" data-lab-view="providers" aria-pressed="false">Providers</button>
|
||
<button type="button" data-lab-view="harness" aria-pressed="false">Harnesses</button>
|
||
<button type="button" data-lab-view="ui" aria-pressed="false">UI states</button>
|
||
</div>
|
||
</div>
|
||
</nav>
|
||
</aside>
|
||
|
||
<div class="live-lab-workspace">
|
||
<header class="live-lab-toolbar">
|
||
<div>
|
||
<h2 data-lab-view-title>Performance</h2>
|
||
<p data-lab-view-summary>Where Live spends time before the first usable variant.</p>
|
||
</div>
|
||
<div class="live-lab-toolbar-actions">
|
||
<button type="button" data-lab-reset data-lab-view-action="overview">Reset simulator</button>
|
||
</div>
|
||
</header>
|
||
|
||
<div class="live-lab-workspace-scroll">
|
||
<div class="live-lab-content-grid">
|
||
<article
|
||
class="live-performance live-lab-canvas"
|
||
data-live-performance
|
||
data-protocol-floor={baselineProductFloor}
|
||
data-overlap-floor={productFloor}
|
||
>
|
||
<header class="live-performance-hero ks-section" data-lab-panel="overview">
|
||
<p class="live-performance-kicker">Click → first usable variant</p>
|
||
<h1><span>{displayMs(protocolFloor)}</span> Live overhead</h1>
|
||
<p class="live-performance-lede">
|
||
Down {(improvement * 100).toFixed(1)}% from {displayMs(baselineFloor)} on the same fixture. Browser dispatch takes {displayMs(browserDispatch)} median.
|
||
</p>
|
||
<dl class="live-lab-run-stats" aria-label="Latest Live run stages">
|
||
<div><dt>Browser dispatch</dt><dd>{displayMs(browserDispatch)}</dd></div>
|
||
<div><dt>Server pickup</dt><dd>{displayMs(serverPickup)}</dd></div>
|
||
<div><dt>Source scaffold</dt><dd>{displayMs(scaffold)}</dd></div>
|
||
<div><dt>Write + settle</dt><dd>{displayMs(writeAndRender)}</dd></div>
|
||
</dl>
|
||
</header>
|
||
|
||
<section class="live-performance-section ks-section" aria-labelledby="critical-path-title" data-lab-panel="overview">
|
||
<div class="live-performance-section-head">
|
||
<div>
|
||
<p class="live-performance-label">Click → first usable variant</p>
|
||
<h2 id="critical-path-title">Where the time goes</h2>
|
||
</div>
|
||
<p>Median product overhead with model time removed.</p>
|
||
</div>
|
||
|
||
<div class="latency-tape">
|
||
<svg viewBox="0 0 1000 214" role="img" aria-labelledby="latency-title latency-desc">
|
||
<title id="latency-title">Impeccable Live model-free latency timeline</title>
|
||
<desc id="latency-desc">The same model-free benchmark improved from {displayMs(baselineFloor)} to {displayMs(protocolFloor)} median.</desc>
|
||
<text class="latency-major-label" x="24" y="42">Before · capture blocks dispatch</text>
|
||
<rect class="latency-segment is-capture" x="24" y="54" width="952" height="42"></rect>
|
||
<text class="latency-major-value" x="44" y="82">{displayMs(baselineFloor)}</text>
|
||
<text class="latency-major-label" x="24" y="128">After · dispatch precedes capture</text>
|
||
<rect class="latency-segment is-scaffold" x="24" y="140" width={optimizedTapeWidth} height="42"></rect>
|
||
<text class="latency-major-value" x="44" y="168">{displayMs(protocolFloor)}</text>
|
||
<path class="latency-bracket" d={`M${24 + optimizedTapeWidth} 184 V194 H976 V184`}></path>
|
||
<text class="latency-remainder-label" x="976" y="210" text-anchor="end">−{(improvement * 100).toFixed(1)}% · same fixture and deterministic agent</text>
|
||
</svg>
|
||
</div>
|
||
|
||
<ol class="latency-stage-list" aria-label="Critical path stages">
|
||
{stageData.map((stage, index) => (
|
||
<li data-tone={stage.tone}>
|
||
<span>{String(index + 1).padStart(2, '0')}</span>
|
||
<strong>{stage.name}</strong>
|
||
<em>{displayMs(stage.value)}</em>
|
||
</li>
|
||
))}
|
||
</ol>
|
||
|
||
<div class="live-performance-finding live-finding-card--quiet" role="note">
|
||
<strong>Capture no longer blocks Go</strong>
|
||
<p>Plain selections dispatch immediately. Annotated selections still upload the screenshot first because the model needs it.</p>
|
||
</div>
|
||
<p class="table-scroll-hint" aria-hidden="true">Scroll horizontally to compare →</p>
|
||
<div class="harness-table-wrap" tabindex="0" aria-label="Scrollable framework performance comparison">
|
||
<table class="harness-table">
|
||
<caption class="sr-only">Progressive Live performance by framework</caption>
|
||
<thead><tr><th>Framework</th><th>Accept → Pick</th><th>Next Go → pickup</th><th>Proof</th></tr></thead>
|
||
<tbody>
|
||
{liveFrameworkResults.map(result => (
|
||
<tr>
|
||
<td><strong>{result.framework}</strong><span>{result.runs} real {result.runs === 1 ? 'run' : 'runs'}</span></td>
|
||
<td><strong>{displayMs(result.acceptMedianMs)}</strong><span>p95 {displayMs(result.acceptP95Ms)}</span></td>
|
||
<td><strong>{displayMs(result.nextPickupMedianMs)}</strong><span>p95 {displayMs(result.nextPickupP95Ms)}</span></td>
|
||
<td><span class="ks-tag">{result.status}</span></td>
|
||
</tr>
|
||
))}
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="live-performance-section ks-section" aria-labelledby="progressive-title" data-lab-panel="overview">
|
||
<div class="live-performance-section-head">
|
||
<div>
|
||
<p class="live-performance-label">Progressive delivery · Codex path</p>
|
||
<h2 id="progressive-title">Variant 1 arrives first</h2>
|
||
</div>
|
||
<p>Review or accept the first result while later variants continue generating.</p>
|
||
</div>
|
||
<div
|
||
class="scenario-comparison"
|
||
role="img"
|
||
aria-label={`Atomic first variant ${displayMs(progressiveDeliveryResult.atomic.medianFirstMs)}; progressive first variant ${displayMs(progressiveDeliveryResult.progressive.medianFirstMs)}`}
|
||
>
|
||
<div class="scenario-row">
|
||
<div class="scenario-label"><strong>Atomic</strong><span>Wait for the full set</span></div>
|
||
<div class="scenario-track"><span class="is-plain" style="--scenario-width:100%"></span></div>
|
||
<output>{displayMs(progressiveDeliveryResult.atomic.medianFirstMs)}</output>
|
||
</div>
|
||
<div class="scenario-row">
|
||
<div class="scenario-label"><strong>Progressive</strong><span>Show variant 1, finish in parallel</span></div>
|
||
<div class="scenario-track"><span class="is-annotated" style={`--scenario-width:${(1 - progressiveGain) * 100}%`}></span></div>
|
||
<output>{displayMs(progressiveDeliveryResult.progressive.medianFirstMs)}</output>
|
||
</div>
|
||
</div>
|
||
<div class="live-performance-finding live-finding-card--quiet" role="note">
|
||
<strong>Measured improvement</strong>
|
||
<p>
|
||
Median first-reviewable latency fell {(progressiveGain * 100).toFixed(1)}%; p95 fell {(progressiveP95Gain * 100).toFixed(1)}%.
|
||
Full-set completion changed by {progressiveAllDelta.toFixed(1)} ms. This is a five-run model-backed comparison using {progressiveDeliveryResult.benchmark.provider} {progressiveDeliveryResult.benchmark.model} and a sanitized synthetic heading contract.
|
||
</p>
|
||
</div>
|
||
<div class="live-performance-finding live-finding-card--quiet" role="note">
|
||
<strong>Control lane</strong>
|
||
<p>
|
||
Across {liveControlResult.runs} real Vite/React runs, Accept dispatch → picker-ready is {displayMs(liveControlResult.acceptToPicking.medianMs)} median / {displayMs(liveControlResult.acceptToPicking.p95Ms)} p95.
|
||
A second Go reaches the poll supervisor in {displayMs(liveControlResult.nextGoToPickup.medianMs)} median / {displayMs(liveControlResult.nextGoToPickup.p95Ms)} p95 while the canceled worker unwinds.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="live-performance-section ks-section" aria-labelledby="provider-title" data-lab-panel="providers">
|
||
<div class="live-performance-section-head">
|
||
<div>
|
||
<p class="live-performance-label">Paid provider matrix · five runs per candidate</p>
|
||
<h2 id="provider-title">Choose delivery per provider</h2>
|
||
</div>
|
||
<p>Sixty strict-gated runs. Brand, component, token, copy, source, and cleanup fidelity must all pass.</p>
|
||
</div>
|
||
<p class="table-scroll-hint" aria-hidden="true">Scroll horizontally to compare →</p>
|
||
<div class="harness-table-wrap" tabindex="0" aria-label="Scrollable provider strategy comparison">
|
||
<table class="harness-table">
|
||
<caption class="sr-only">Paid Live generation strategy results by provider</caption>
|
||
<thead><tr><th>Provider</th>{providerColumns.map(column => <th>{column.label}</th>)}</tr></thead>
|
||
<tbody>
|
||
{liveProviderResult.providers.map(result => (
|
||
<tr>
|
||
<td><strong>{result.provider}</strong><span>{result.model} · {result.effort}</span><span>{result.recommendation}</span></td>
|
||
{providerColumns.map(column => {
|
||
const candidate = result[column.key];
|
||
return (
|
||
<td>
|
||
<strong>{candidate.firstMedianMs === null ? 'No valid run' : `${displayMs(candidate.firstMedianMs)} first`}</strong>
|
||
<span>{candidate.passedRuns}/5 pass{candidate.firstP95Ms === null ? '' : ` · p95 ${displayMs(candidate.firstP95Ms)}`}</span>
|
||
<span>{candidate.allMedianMs === null ? 'full set unavailable' : `${displayMs(candidate.allMedianMs)} all`}</span>
|
||
</td>
|
||
);
|
||
})}
|
||
</tr>
|
||
))}
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<div class="live-performance-finding live-finding-card--quiet" role="note">
|
||
<strong>Decision</strong>
|
||
<p>
|
||
Codex/OpenAI defaults to progressive compact: 5/5 passed, it had the fastest first review, and it cost less than the other OpenAI splits. Anthropic also keeps progressive compact because it was the only 5/5 candidate. Gemini uses parallel compact; every other Gemini strategy failed the reliability gate. Unmeasured harnesses retain atomic delivery.
|
||
</p>
|
||
</div>
|
||
<p class="live-performance-caption">
|
||
Provider matrix cost: ${liveProviderResult.totals.estimatedCostUsd.toFixed(2)} across {liveProviderResult.totals.apiCalls} calls, including {liveProviderResult.totals.retries} retries. {liveProviderResult.totals.passingRuns}/{liveProviderResult.totals.runs} runs passed the strict gate. Accept → clean Pick/build control passed in {displayMs(liveProviderResult.cleanupControl.acceptToCleanPickingMs)} with zero console errors or Live markers.
|
||
</p>
|
||
</section>
|
||
|
||
<section class="live-performance-section ks-section" aria-labelledby="init-title" data-lab-panel="overview">
|
||
<div class="live-performance-section-head">
|
||
<div>
|
||
<p class="live-performance-label">Configured cold initialization</p>
|
||
<h2 id="init-title">Cold setup takes {displayMs(liveInitResult.cold.medianMs)}</h2>
|
||
</div>
|
||
<p>Ten isolated starts. App dependency installation and dev-server startup are excluded.</p>
|
||
</div>
|
||
<div class="scenario-comparison" role="img" aria-label={`Cold initialization improved from ${displayMs(liveInitResult.baselineColdMedianMs)} to ${displayMs(liveInitResult.cold.medianMs)} median`}>
|
||
<div class="scenario-row">
|
||
<div class="scenario-label"><strong>Before</strong><span>200 ms readiness polling floor</span></div>
|
||
<div class="scenario-track"><span class="is-plain" style="--scenario-width:100%"></span></div>
|
||
<output>{displayMs(liveInitResult.baselineColdMedianMs)}</output>
|
||
</div>
|
||
<div class="scenario-row">
|
||
<div class="scenario-label"><strong>Cold now</strong><span>5 ms readiness polling</span></div>
|
||
<div class="scenario-track"><span class="is-annotated" style={`--scenario-width:${(1 - initGain) * 100}%`}></span></div>
|
||
<output>{displayMs(liveInitResult.cold.medianMs)}</output>
|
||
</div>
|
||
<div class="scenario-row">
|
||
<div class="scenario-label"><strong>Warm now</strong><span>Reuse the running helper</span></div>
|
||
<div class="scenario-track"><span class="is-control" style={`--scenario-width:${(liveInitResult.warm.medianMs / liveInitResult.baselineColdMedianMs) * 100}%`}></span></div>
|
||
<output>{displayMs(liveInitResult.warm.medianMs)}</output>
|
||
</div>
|
||
</div>
|
||
<div class="live-performance-finding live-finding-card--quiet" role="note">
|
||
<strong>Initialization result</strong>
|
||
<p>Cold median fell {(initGain * 100).toFixed(1)}% to {displayMs(liveInitResult.cold.medianMs)}; p95 is {displayMs(liveInitResult.cold.p95Ms)}. Warm median is {displayMs(liveInitResult.warm.medianMs)}.</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="live-performance-section ks-section" aria-labelledby="paths-title" data-lab-panel="providers">
|
||
<div class="live-performance-section-head">
|
||
<div>
|
||
<p class="live-performance-label">Before / after / control</p>
|
||
<h2 id="paths-title">Plain selections skip capture</h2>
|
||
</div>
|
||
<p>The annotated path remains a control: its screenshot must reach the agent before generation starts.</p>
|
||
</div>
|
||
<div class="scenario-comparison" role="img" aria-label={`Baseline ${displayMs(baselineFloor)}; optimized plain path ${displayMs(protocolFloor)}; annotated control ${displayMs(annotatedFloor)}`}>
|
||
<div class="scenario-row">
|
||
<div class="scenario-label"><strong>Before</strong><span>Plain pick, capture first</span></div>
|
||
<div class="scenario-track"><span class="is-plain" style={`--scenario-width:${(baselineFloor / maxScenarioFloor) * 100}%`}></span></div>
|
||
<output>{displayMs(baselineFloor)}</output>
|
||
</div>
|
||
<div class="scenario-row">
|
||
<div class="scenario-label"><strong>After</strong><span>Plain pick, dispatch first</span></div>
|
||
<div class="scenario-track"><span class="is-annotated" style={`--scenario-width:${(protocolFloor / maxScenarioFloor) * 100}%`}></span></div>
|
||
<output>{displayMs(protocolFloor)}</output>
|
||
</div>
|
||
<div class="scenario-row">
|
||
<div class="scenario-label"><strong>Annotated control</strong><span>Pin + stroke + upload</span></div>
|
||
<div class="scenario-track"><span class="is-control" style={`--scenario-width:${(annotatedFloor / maxScenarioFloor) * 100}%`}></span></div>
|
||
<output>{displayMs(annotatedFloor)}</output>
|
||
</div>
|
||
</div>
|
||
<p class="live-performance-caption">Median protocol measurements: {baseline.summary.count} before runs, {plain.summary.count} optimized runs, {annotated?.summary.count || 0} annotated controls. Browser and filesystem caches were warm.</p>
|
||
<div class="live-performance-finding live-finding-card--quiet" role="note">
|
||
<strong>Annotated model proof</strong>
|
||
<p>
|
||
{liveAnnotatedResult.provider} {liveAnnotatedResult.model} received a real screenshot, {liveAnnotatedResult.evidence.comments} comment, and {liveAnnotatedResult.evidence.strokes} stroke.
|
||
First review arrived in {displayMs(liveAnnotatedResult.firstReviewableMs)} with {displayMs(liveAnnotatedResult.impeccableOverheadMs)} of measured Impeccable overhead; all variants arrived in {displayMs(liveAnnotatedResult.allVariantsMs)} and teardown returned to clean source.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="live-performance-section live-simulator-section ks-section" aria-labelledby="simulator-title" data-lab-panel="overview">
|
||
<div class="live-performance-section-head">
|
||
<div>
|
||
<p class="live-performance-label">Latency simulator</p>
|
||
<h2 id="simulator-title">Compare model and product time</h2>
|
||
</div>
|
||
<p>Move model time to compare the old floor with the optimized product-side estimate.</p>
|
||
</div>
|
||
<div class="live-simulator">
|
||
<label for="model-latency">Model generates all variants <output data-model-value>15 s</output></label>
|
||
<input id="model-latency" data-model-latency type="range" min="0" max="60000" step="500" value="15000" />
|
||
<div class="simulator-results" aria-live="polite">
|
||
<div>
|
||
<span>Before optimization</span>
|
||
<strong data-current-total>15.9 s</strong>
|
||
<small>model + measured floor, normalized by matched actionability control</small>
|
||
</div>
|
||
<div>
|
||
<span>Current optimized path</span>
|
||
<strong data-overlap-total>15.1 s</strong>
|
||
<small>model + measured floor, normalized by matched actionability control</small>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="live-lab-ui-panel" data-lab-panel="ui" aria-label="Live UI state gallery">
|
||
<LiveUiGallery />
|
||
</section>
|
||
|
||
<section class="live-performance-section ks-section" aria-labelledby="harness-title" data-lab-panel="harness">
|
||
<div class="live-performance-section-head">
|
||
<div>
|
||
<p class="live-performance-label">Harness delivery</p>
|
||
<h2 id="harness-title">Compare worker wake paths</h2>
|
||
</div>
|
||
<p>Measure pickup before choosing architecture. Fast models cannot help while no model is running.</p>
|
||
</div>
|
||
<p class="table-scroll-hint" aria-hidden="true">Scroll horizontally to compare →</p>
|
||
<div class="harness-table-wrap" tabindex="0" aria-label="Scrollable harness delivery comparison">
|
||
<table class="harness-table">
|
||
<caption class="sr-only">Live event delivery paths by agent harness</caption>
|
||
<thead><tr><th>Path</th><th>Pickup</th><th>Tradeoff</th><th>Status</th></tr></thead>
|
||
<tbody>
|
||
{harnessPaths.map(path => (
|
||
<tr>
|
||
<td><strong>{path.name}</strong><span>{path.harness}</span></td>
|
||
<td>{path.pickup}</td>
|
||
<td>{path.tradeoff}{path.source && <> <a href={path.source}>Docs</a></>}</td>
|
||
<td><span class="ks-tag is-neutral" data-tone={path.statusTone}>{path.status}</span></td>
|
||
</tr>
|
||
))}
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<div class="live-performance-finding is-probe" role="note">
|
||
<strong>Harness probe</strong>
|
||
<p>{currentHarnessProbe.surface}: {currentHarnessProbe.result}</p>
|
||
</div>
|
||
<div class="live-performance-finding live-finding-card--quiet" role="note">
|
||
<strong>Dedicated Codex worker probe</strong>
|
||
<p>
|
||
{liveCodexWorkerResult.model} at {liveCodexWorkerResult.effort} effort started a dedicated turn in {displayMs(liveCodexWorkerResult.timings.wakeToTurnStartedMs)} and completed a trivial structured response in {displayMs(liveCodexWorkerResult.timings.trivialTurnCompletedMs)}. Cold app-server plus thread setup cost {displayMs(liveCodexWorkerResult.timings.coldHandshakeMs + liveCodexWorkerResult.timings.coldThreadStartMs)} and can be amortized. The safe path owns a separate Live thread; it never resumes the desktop task from a competing app-server.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="live-performance-section live-experiments-section ks-section" aria-labelledby="experiments-title" data-lab-panel="harness">
|
||
<div class="live-performance-section-head">
|
||
<div>
|
||
<p class="live-performance-label">Evidence-ranked decisions</p>
|
||
<h2 id="experiments-title">Decisions and open bets</h2>
|
||
</div>
|
||
<p>Shipped paths, rejected compromises, and unproven architectural bets are labeled separately.</p>
|
||
</div>
|
||
<ol class="experiment-list">
|
||
{liveExperiments.map(experiment => (
|
||
<li>
|
||
<span class="experiment-rank">{String(experiment.rank).padStart(2, '0')}</span>
|
||
<div class="experiment-body">
|
||
<header><h3>{experiment.title}</h3><span>{experiment.confidence}</span></header>
|
||
<p>{experiment.move}</p>
|
||
<dl>
|
||
<div><dt>Evidence</dt><dd>{experiment.evidence}</dd></div>
|
||
<div><dt>Expected</dt><dd>{experiment.expected}</dd></div>
|
||
</dl>
|
||
</div>
|
||
</li>
|
||
))}
|
||
</ol>
|
||
</section>
|
||
|
||
<section class="live-performance-method ks-section" aria-labelledby="method-title" data-lab-panel="harness">
|
||
<h2 id="method-title">How results are measured</h2>
|
||
<p>
|
||
<code>bun run bench:live</code> boots a real framework fixture and Chromium, drives Pick → Go → Cycle,
|
||
and records browser preparation, server pickup, scaffold, generation, write, and render boundaries.
|
||
<code>node scripts/benchmark-live-init.mjs --iterations 10</code> measures configured cold and warm helper initialization separately.
|
||
The deterministic agent makes Impeccable overhead visible; model-backed runs remain opt-in because they send fixture context to an external provider.
|
||
</p>
|
||
</section>
|
||
</article>
|
||
|
||
<aside class="live-lab-diagnostics" aria-label="Current view summary">
|
||
<section class="live-lab-diagnostic-panel" data-lab-context="overview">
|
||
<header><h2>Current run</h2></header>
|
||
<dl>
|
||
<div><dt>Go → first</dt><dd>{displayMs(protocolFloor)}</dd></div>
|
||
<div><dt>Dispatch</dt><dd>{displayMs(browserDispatch)}</dd></div>
|
||
<div><dt>Scaffold</dt><dd>{displayMs(scaffold)}</dd></div>
|
||
<div><dt>Accept → Pick</dt><dd>{displayMs(liveControlResult.acceptToPicking.medianMs)}</dd></div>
|
||
</dl>
|
||
</section>
|
||
<section class="live-lab-diagnostic-panel" data-lab-context="providers" hidden>
|
||
<header><h2>Recommended delivery</h2></header>
|
||
<ul class="live-lab-strategy-list">
|
||
<li><strong>Codex / OpenAI</strong><span>progressive compact</span></li>
|
||
<li><strong>Anthropic</strong><span>progressive compact</span></li>
|
||
<li><strong>Gemini</strong><span>parallel compact</span></li>
|
||
<li><strong>Unmeasured harness</strong><span>atomic fallback</span></li>
|
||
</ul>
|
||
</section>
|
||
<section class="live-lab-diagnostic-panel" data-lab-context="harness" hidden>
|
||
<header><h2>Dedicated Codex worker</h2></header>
|
||
<dl>
|
||
<div><dt>Wake</dt><dd>{displayMs(liveCodexWorkerResult.timings.wakeToTurnStartedMs)}</dd></div>
|
||
<div><dt>Cold setup</dt><dd>{displayMs(liveCodexWorkerResult.timings.coldHandshakeMs + liveCodexWorkerResult.timings.coldThreadStartMs)}</dd></div>
|
||
</dl>
|
||
<p>Live owns the worker thread. It never resumes the desktop task.</p>
|
||
</section>
|
||
</aside>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<script>
|
||
import { initLivePerformance } from '../../scripts/live-performance.js';
|
||
import { initLiveLabWorkbench } from '../../scripts/live-lab-workbench.js';
|
||
initLivePerformance();
|
||
initLiveLabWorkbench();
|
||
</script>
|
||
</Base>
|