diff --git a/scripts/benchmark-live.mjs b/scripts/benchmark-live.mjs index 41c05534b..fb9cbef86 100644 --- a/scripts/benchmark-live.mjs +++ b/scripts/benchmark-live.mjs @@ -108,6 +108,10 @@ try { if (payload?.type === 'generate' && payload.id) { recorder.mark('browser.generate_post', { id: payload.id, + selectedTagName: String(payload.element?.tagName || '').toLowerCase(), + selectedClasses: Array.isArray(payload.element?.classes) + ? payload.element.classes.map(String) + : String(payload.element?.className || '').split(/\s+/).filter(Boolean), hasScreenshotPath: typeof payload.screenshotPath === 'string' && payload.screenshotPath.length > 0, commentCount: Array.isArray(payload.comments) ? payload.comments.length : 0, strokeCount: Array.isArray(payload.strokes) ? payload.strokes.length : 0, @@ -133,7 +137,10 @@ try { ? join(artifactRoot, scenario, `run-${String(iteration).padStart(2, '0')}`) : null; if (runArtifactDir) await mkdir(runArtifactDir, { recursive: true }); - await pickElement(session.page, pickSelector, { resetPickMode: iteration > 1 }); + await pickElement(session.page, pickSelector, { + position: fixture.runtime.pickPosition || null, + resetPickMode: iteration > 1, + }); if (args.action) await selectAction(session.page, String(args.action)); if (scenario === 'annotated') { await drawAnnotationPinAndStroke(session.page, { comment: 'Benchmark annotation' }); @@ -186,6 +193,7 @@ try { browserTiming, }); assertScenarioEvidence(run, scenario); + assertSelectionEvidence(run, fixture.runtime.expectedPick); if (renderedArtifacts) run.renderedArtifacts = renderedArtifacts; if (judgeRendered) { run.renderedJudge = await judgeRenderedVariants({ @@ -210,7 +218,10 @@ try { await waitForReset(session.page); run.acceptToResetMs = roundMs(performance.now() - acceptStartedAt); - await pickElement(session.page, pickSelector, { resetPickMode: true }); + await pickElement(session.page, pickSelector, { + position: fixture.runtime.pickPosition || null, + resetPickMode: true, + }); if (args.action) await selectAction(session.page, String(args.action)); await resetBrowserTimingProbe(session.page, `${iteration}-followup`); const nextFirstVariant = waitForFirstVariant(session.page); @@ -228,7 +239,9 @@ try { await clickDiscard(session.page); await waitForReset(session.page); } - Object.assign(run, deriveJournalGenerationMetrics(await readGenerationSnapshot(session.tmp, run.eventId))); + const generationSnapshot = await readGenerationSnapshot(session.tmp, run.eventId); + Object.assign(run, deriveJournalGenerationMetrics(generationSnapshot)); + if (generationSnapshot.variantPlan) run.variantPlan = generationSnapshot.variantPlan; runs.push(run); if (!args.quiet) process.stderr.write(formatRun(runs.at(-1)) + '\n'); @@ -618,6 +631,19 @@ function assertScenarioEvidence(run, currentScenario) { } } +function assertSelectionEvidence(run, expected) { + if (!expected) return; + const actual = run.selectionEvidence || {}; + const expectedTag = String(expected.tagName || '').toLowerCase(); + const expectedClasses = Array.isArray(expected.classes) ? expected.classes.map(String) : []; + if ((expectedTag && actual.tagName !== expectedTag) + || expectedClasses.some((className) => !actual.classes?.includes(className))) { + throw new Error( + `iteration ${run.iteration}: picked ${actual.tagName || 'unknown'}.${(actual.classes || []).join('.')} instead of ${expectedTag || '*'}.${expectedClasses.join('.')}`, + ); + } +} + function wrapTargetFromPickedElement(event) { const element = event.element || {}; return { diff --git a/scripts/lib/live-benchmark.mjs b/scripts/lib/live-benchmark.mjs index 1617c701f..818464e17 100644 --- a/scripts/lib/live-benchmark.mjs +++ b/scripts/lib/live-benchmark.mjs @@ -127,6 +127,10 @@ export function buildInteractionRun(events, { iteration, scenario, goStartedAt, iteration, scenario, eventId: id, + selectionEvidence: { + tagName: eventPost?.selectedTagName || null, + classes: Array.isArray(eventPost?.selectedClasses) ? eventPost.selectedClasses : [], + }, annotationEvidence: { screenshotPath: eventPost?.hasScreenshotPath === true, comments: Number(eventPost?.commentCount || 0), diff --git a/tests/framework-fixtures/vite8-react-brand-fidelity/fixture.json b/tests/framework-fixtures/vite8-react-brand-fidelity/fixture.json index b011530e1..64ee190ea 100644 --- a/tests/framework-fixtures/vite8-react-brand-fidelity/fixture.json +++ b/tests/framework-fixtures/vite8-react-brand-fidelity/fixture.json @@ -49,6 +49,8 @@ "readyPattern": "Local:\\s+https?://[^:]+:(\\d+)", "readyTimeoutMs": 120000, "pickSelector": "article.offer-card", + "pickPosition": { "x": 8, "y": 8 }, + "expectedPick": { "tagName": "article", "classes": ["offer-card"] }, "acceptedSourcePattern": "]*(class|className)=\"[^\"]*\\boffer-card\\b[^\"]*\"", "steer": { "message": "steer-e2e mark offer", diff --git a/tests/live-benchmark.test.mjs b/tests/live-benchmark.test.mjs index 0c0acb33d..7360ed62c 100644 --- a/tests/live-benchmark.test.mjs +++ b/tests/live-benchmark.test.mjs @@ -108,7 +108,7 @@ describe('live benchmark metrics', () => { it('separates model generation from Impeccable overhead', () => { const events = [ { name: 'ui.go.start', at: 100, iteration: 1 }, - { name: 'browser.generate_post', at: 108, id: 'abc', hasScreenshotPath: false, commentCount: 0, strokeCount: 0 }, + { name: 'browser.generate_post', at: 108, id: 'abc', selectedTagName: 'article', selectedClasses: ['offer-card'], hasScreenshotPath: false, commentCount: 0, strokeCount: 0 }, { name: 'agent.event.received', at: 110, id: 'abc', type: 'generate' }, { name: 'agent.scaffold.start', at: 112, id: 'abc' }, { name: 'agent.scaffold.end', at: 132, id: 'abc' }, @@ -134,6 +134,7 @@ describe('live benchmark metrics', () => { assert.equal(run.browserDispatchMs, 2.5); assert.equal(run.automationClickMs, 5.5); assert.deepEqual(run.annotationEvidence, { screenshotPath: false, comments: 0, strokes: 0 }); + assert.deepEqual(run.selectionEvidence, { tagName: 'article', classes: ['offer-card'] }); assert.equal(run.serverPickupMs, 2); assert.equal(run.generationMs, 1000); assert.equal(run.impeccableOverheadMs, 94.5);