Prove Live benchmark target selection

This commit is contained in:
Paul Bakaus
2026-07-13 11:22:43 -07:00
parent eedccaedb0
commit 2f619f9bdb
4 changed files with 37 additions and 4 deletions
+29 -3
View File
@@ -108,6 +108,10 @@ try {
if (payload?.type === 'generate' && payload.id) {
recorder.mark('browser.generate_post', {
id: payload.id,
selectedTagName: String(payload.element?.tagName || '').toLowerCase(),
selectedClasses: Array.isArray(payload.element?.classes)
? payload.element.classes.map(String)
: String(payload.element?.className || '').split(/\s+/).filter(Boolean),
hasScreenshotPath: typeof payload.screenshotPath === 'string' && payload.screenshotPath.length > 0,
commentCount: Array.isArray(payload.comments) ? payload.comments.length : 0,
strokeCount: Array.isArray(payload.strokes) ? payload.strokes.length : 0,
@@ -133,7 +137,10 @@ try {
? join(artifactRoot, scenario, `run-${String(iteration).padStart(2, '0')}`)
: null;
if (runArtifactDir) await mkdir(runArtifactDir, { recursive: true });
await pickElement(session.page, pickSelector, { resetPickMode: iteration > 1 });
await pickElement(session.page, pickSelector, {
position: fixture.runtime.pickPosition || null,
resetPickMode: iteration > 1,
});
if (args.action) await selectAction(session.page, String(args.action));
if (scenario === 'annotated') {
await drawAnnotationPinAndStroke(session.page, { comment: 'Benchmark annotation' });
@@ -186,6 +193,7 @@ try {
browserTiming,
});
assertScenarioEvidence(run, scenario);
assertSelectionEvidence(run, fixture.runtime.expectedPick);
if (renderedArtifacts) run.renderedArtifacts = renderedArtifacts;
if (judgeRendered) {
run.renderedJudge = await judgeRenderedVariants({
@@ -210,7 +218,10 @@ try {
await waitForReset(session.page);
run.acceptToResetMs = roundMs(performance.now() - acceptStartedAt);
await pickElement(session.page, pickSelector, { resetPickMode: true });
await pickElement(session.page, pickSelector, {
position: fixture.runtime.pickPosition || null,
resetPickMode: true,
});
if (args.action) await selectAction(session.page, String(args.action));
await resetBrowserTimingProbe(session.page, `${iteration}-followup`);
const nextFirstVariant = waitForFirstVariant(session.page);
@@ -228,7 +239,9 @@ try {
await clickDiscard(session.page);
await waitForReset(session.page);
}
Object.assign(run, deriveJournalGenerationMetrics(await readGenerationSnapshot(session.tmp, run.eventId)));
const generationSnapshot = await readGenerationSnapshot(session.tmp, run.eventId);
Object.assign(run, deriveJournalGenerationMetrics(generationSnapshot));
if (generationSnapshot.variantPlan) run.variantPlan = generationSnapshot.variantPlan;
runs.push(run);
if (!args.quiet) process.stderr.write(formatRun(runs.at(-1)) + '\n');
@@ -618,6 +631,19 @@ function assertScenarioEvidence(run, currentScenario) {
}
}
function assertSelectionEvidence(run, expected) {
if (!expected) return;
const actual = run.selectionEvidence || {};
const expectedTag = String(expected.tagName || '').toLowerCase();
const expectedClasses = Array.isArray(expected.classes) ? expected.classes.map(String) : [];
if ((expectedTag && actual.tagName !== expectedTag)
|| expectedClasses.some((className) => !actual.classes?.includes(className))) {
throw new Error(
`iteration ${run.iteration}: picked ${actual.tagName || 'unknown'}.${(actual.classes || []).join('.')} instead of ${expectedTag || '*'}.${expectedClasses.join('.')}`,
);
}
}
function wrapTargetFromPickedElement(event) {
const element = event.element || {};
return {
+4
View File
@@ -127,6 +127,10 @@ export function buildInteractionRun(events, { iteration, scenario, goStartedAt,
iteration,
scenario,
eventId: id,
selectionEvidence: {
tagName: eventPost?.selectedTagName || null,
classes: Array.isArray(eventPost?.selectedClasses) ? eventPost.selectedClasses : [],
},
annotationEvidence: {
screenshotPath: eventPost?.hasScreenshotPath === true,
comments: Number(eventPost?.commentCount || 0),
@@ -49,6 +49,8 @@
"readyPattern": "Local:\\s+https?://[^:]+:(\\d+)",
"readyTimeoutMs": 120000,
"pickSelector": "article.offer-card",
"pickPosition": { "x": 8, "y": 8 },
"expectedPick": { "tagName": "article", "classes": ["offer-card"] },
"acceptedSourcePattern": "<article[^>]*(class|className)=\"[^\"]*\\boffer-card\\b[^\"]*\"",
"steer": {
"message": "steer-e2e mark offer",
+2 -1
View File
@@ -108,7 +108,7 @@ describe('live benchmark metrics', () => {
it('separates model generation from Impeccable overhead', () => {
const events = [
{ name: 'ui.go.start', at: 100, iteration: 1 },
{ name: 'browser.generate_post', at: 108, id: 'abc', hasScreenshotPath: false, commentCount: 0, strokeCount: 0 },
{ name: 'browser.generate_post', at: 108, id: 'abc', selectedTagName: 'article', selectedClasses: ['offer-card'], hasScreenshotPath: false, commentCount: 0, strokeCount: 0 },
{ name: 'agent.event.received', at: 110, id: 'abc', type: 'generate' },
{ name: 'agent.scaffold.start', at: 112, id: 'abc' },
{ name: 'agent.scaffold.end', at: 132, id: 'abc' },
@@ -134,6 +134,7 @@ describe('live benchmark metrics', () => {
assert.equal(run.browserDispatchMs, 2.5);
assert.equal(run.automationClickMs, 5.5);
assert.deepEqual(run.annotationEvidence, { screenshotPath: false, comments: 0, strokes: 0 });
assert.deepEqual(run.selectionEvidence, { tagName: 'article', classes: ['offer-card'] });
assert.equal(run.serverPickupMs, 2);
assert.equal(run.generationMs, 1000);
assert.equal(run.impeccableOverheadMs, 94.5);