skills/pbakaus_impeccable
1
0
Fork 0
mirror of https://github.com/pbakaus/impeccable.git synced 2026-09-17 08:36:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

Deliver Live variant two independently

Browse Source
This commit is contained in:
Paul Bakaus
2026-07-13 11:42:25 -07:00
parent 2f619f9bdb
commit 4c07124198
10 changed files with 207 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/benchmark-live.mjs
+31 -12
View File
@@ -51,7 +51,11 @@ const delivery = args.delivery === 'atomic'
: agentMode === 'codex' || args.delivery === 'progressive'
? 'progressive'
: 'atomic';
const interactionMode = args.acceptFirst ? 'accept-first-then-next-go' : 'complete-then-discard';
const acceptVariant = positiveInt(args.acceptVariant, args.acceptFirst ? 1 : 0);
if (acceptVariant > 2) throw new Error('--accept-variant currently supports variant 1 or 2');
const interactionMode = acceptVariant
? `accept-variant-${acceptVariant}-then-next-go`
: 'complete-then-discard';
const simulatedTailMs = positiveInt(args.simulatedTailMs, 0);
const outputPath = args.output ? resolve(ROOT, String(args.output)) : null;
const artifactRoot = args.artifacts ? resolve(ROOT, String(args.artifacts)) : null;
@@ -61,7 +65,7 @@ const fixture = JSON.parse(await readFile(join(FIXTURES_DIR, fixtureName, 'fixtu
if (!fixture.runtime) throw new Error(`fixture ${fixtureName} has no runtime configuration`);
if (fixture.runtime.mode === 'insert') throw new Error('live benchmark currently measures replace-mode fixtures only');
if (judgeRendered && !artifactRoot) throw new Error('--judge-rendered requires --artifacts=<directory>');
if (judgeRendered && args.acceptFirst) throw new Error('--judge-rendered requires complete variants; omit --accept-first');
if (judgeRendered && acceptVariant) throw new Error('--judge-rendered requires complete variants; omit --accept-first/--accept-variant');
if (judgeRendered && fixture.renderedQuality?.remoteSafe !== true) {
throw new Error(`fixture ${fixtureName} is not explicitly remote-safe for rendered judging`);
}
@@ -158,20 +162,30 @@ try {
const firstVariant = waitForFirstVariant(session.page).then(() => {
recorder.mark('browser.first_variant', { iteration, scenario });
});
const secondVariant = acceptVariant === 1
? null
: waitForVariantCount(session.page, 2).then(() => {
recorder.mark('browser.second_variant', { iteration, scenario });
});
await clickGo(session.page);
recorder.mark('ui.generating_visible', { iteration, scenario });
await firstVariant;
if (renderedArtifacts && args.acceptFirst) {
if (acceptVariant === 2) {
await secondVariant;
await ensureBenchmarkVariant(session.page, 2);
}
if (renderedArtifacts && acceptVariant) {
renderedArtifacts.variants.push(await captureRenderedElement(session.page, {
filePath: join(runArtifactDir, 'variant-1.png'),
variantId: 1,
filePath: join(runArtifactDir, `variant-${acceptVariant}.png`),
variantId: acceptVariant,
selector: renderedContext.captureSelector,
}));
}
const browserTiming = await readBrowserTimingProbe(session.page);
if (!args.acceptFirst) {
if (!acceptVariant) {
await waitForCycling(session.page, 3, { timeout: agentMode === 'fake' ? 30_000 : 240_000 });
await secondVariant;
recorder.mark('browser.all_variants', { iteration, scenario });
if (renderedArtifacts) {
for (const variantId of [1, 2, 3]) {
@@ -212,9 +226,9 @@ try {
runArtifactDir,
});
}
if (args.acceptFirst) {
if (acceptVariant) {
const acceptStartedAt = performance.now();
await clickAccept(session.page, { expectedVariant: 1 });
await clickAccept(session.page, { expectedVariant: acceptVariant });
await waitForReset(session.page);
run.acceptToResetMs = roundMs(performance.now() - acceptStartedAt);
@@ -548,12 +562,17 @@ function createSplitProgressiveAgent(agent) {
}
async function waitForFirstVariant(page) {
const handle = await page.waitForFunction(() => {
await waitForVariantCount(page, 1);
}
async function waitForVariantCount(page, expectedCount) {
const handle = await page.waitForFunction((count) => {
const activeGeneration = document.querySelector('[data-impeccable-variants]');
if (!activeGeneration) return false;
const variants = [...activeGeneration.querySelectorAll('[data-impeccable-variant]')];
return variants.some((element) => element.getAttribute('data-impeccable-variant') !== 'original');
}, undefined, { timeout: 150_000 });
const debugCount = Number(window.__IMPECCABLE_LIVE_CHROME_CORE__?.debugState?.()?.arrivedVariants || 0);
const domCount = activeGeneration.querySelectorAll('[data-impeccable-variant]:not([data-impeccable-variant="original"])').length;
return Math.max(debugCount, domCount) >= count;
}, expectedCount, { timeout: 240_000 });
await handle.dispose();
}
scripts/lib/live-benchmark.mjs
+12
View File
@@ -15,12 +15,17 @@ const METRIC_KEYS = [
'writeToFirstVariantMs',
'replyMs',
'goToFirstVariantMs',
'goToSecondVariantMs',
'goToAllVariantsMs',
'firstToSecondGapMs',
'secondToAllGapMs',
'deliveryGapMs',
'impeccableOverheadMs',
'workerPickupToSourceReadyMs',
'workerFirstGenerationToReviewableMs',
'workerFirstValidationToReviewableMs',
'workerSecondGenerationToReviewableMs',
'workerSecondValidationToReviewableMs',
'workerRemainingGenerationToReadyMs',
'workerRemainingValidationToReadyMs',
'acceptToResetMs',
@@ -107,6 +112,7 @@ export function buildInteractionRun(events, { iteration, scenario, goStartedAt,
const eventPost = events.find((event) => event.name === 'browser.generate_post' && forId(event));
const mark = (name) => events.find((event) => event.name === name && event.iteration === iteration);
const first = mark('browser.first_variant');
const second = mark('browser.second_variant');
const all = mark('browser.all_variants');
const writeEnd = events.find((event) => event.name === 'agent.write.end' && forId(event));
const firstWriteEnd = events.find((event) => event.name === 'agent.first_variant.write.end' && forId(event));
@@ -121,6 +127,7 @@ export function buildInteractionRun(events, { iteration, scenario, goStartedAt,
? eventPost.at - browserDispatchMs
: goStartedAt;
const measuredGoToFirstVariantMs = first ? roundMs(first.at - interactionStartedAt) : null;
const measuredGoToSecondVariantMs = second ? roundMs(second.at - interactionStartedAt) : null;
const measuredGoToAllVariantsMs = all ? roundMs(all.at - interactionStartedAt) : null;
return {
@@ -154,7 +161,10 @@ export function buildInteractionRun(events, { iteration, scenario, goStartedAt,
: null,
replyMs: durationBetween(events, 'agent.reply.start', 'agent.reply.end', forId),
goToFirstVariantMs: measuredGoToFirstVariantMs,
goToSecondVariantMs: measuredGoToSecondVariantMs,
goToAllVariantsMs: measuredGoToAllVariantsMs,
firstToSecondGapMs: first && second ? roundMs(Math.max(0, second.at - first.at)) : null,
secondToAllGapMs: second && all ? roundMs(Math.max(0, all.at - second.at)) : null,
deliveryGapMs: first && all ? roundMs(Math.max(0, all.at - first.at)) : null,
impeccableOverheadMs: measuredGoToFirstVariantMs == null || generationToFirstMs == null
? null
@@ -178,6 +188,8 @@ export function deriveJournalGenerationMetrics(snapshot = {}) {
workerPickupToSourceReadyMs: delta('picked_up', 'source_ready'),
workerFirstGenerationToReviewableMs: delta('first_variant_generating', 'first_reviewable'),
workerFirstValidationToReviewableMs: delta('first_variant_validating', 'first_reviewable'),
workerSecondGenerationToReviewableMs: delta('second_variant_generating', 'second_reviewable'),
workerSecondValidationToReviewableMs: delta('second_variant_validating', 'second_reviewable'),
workerRemainingGenerationToReadyMs: delta('remaining_variants_generating', 'all_variants_ready'),
workerRemainingValidationToReadyMs: delta('remaining_variants_validating', 'all_variants_ready'),
journalTimingErrors: timingErrors,