mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-18 09:06:53 +03:00
Prove Live benchmark target selection
This commit is contained in:
@@ -108,6 +108,10 @@ try {
|
||||
if (payload?.type === 'generate' && payload.id) {
|
||||
recorder.mark('browser.generate_post', {
|
||||
id: payload.id,
|
||||
selectedTagName: String(payload.element?.tagName || '').toLowerCase(),
|
||||
selectedClasses: Array.isArray(payload.element?.classes)
|
||||
? payload.element.classes.map(String)
|
||||
: String(payload.element?.className || '').split(/\s+/).filter(Boolean),
|
||||
hasScreenshotPath: typeof payload.screenshotPath === 'string' && payload.screenshotPath.length > 0,
|
||||
commentCount: Array.isArray(payload.comments) ? payload.comments.length : 0,
|
||||
strokeCount: Array.isArray(payload.strokes) ? payload.strokes.length : 0,
|
||||
@@ -133,7 +137,10 @@ try {
|
||||
? join(artifactRoot, scenario, `run-${String(iteration).padStart(2, '0')}`)
|
||||
: null;
|
||||
if (runArtifactDir) await mkdir(runArtifactDir, { recursive: true });
|
||||
await pickElement(session.page, pickSelector, { resetPickMode: iteration > 1 });
|
||||
await pickElement(session.page, pickSelector, {
|
||||
position: fixture.runtime.pickPosition || null,
|
||||
resetPickMode: iteration > 1,
|
||||
});
|
||||
if (args.action) await selectAction(session.page, String(args.action));
|
||||
if (scenario === 'annotated') {
|
||||
await drawAnnotationPinAndStroke(session.page, { comment: 'Benchmark annotation' });
|
||||
@@ -186,6 +193,7 @@ try {
|
||||
browserTiming,
|
||||
});
|
||||
assertScenarioEvidence(run, scenario);
|
||||
assertSelectionEvidence(run, fixture.runtime.expectedPick);
|
||||
if (renderedArtifacts) run.renderedArtifacts = renderedArtifacts;
|
||||
if (judgeRendered) {
|
||||
run.renderedJudge = await judgeRenderedVariants({
|
||||
@@ -210,7 +218,10 @@ try {
|
||||
await waitForReset(session.page);
|
||||
run.acceptToResetMs = roundMs(performance.now() - acceptStartedAt);
|
||||
|
||||
await pickElement(session.page, pickSelector, { resetPickMode: true });
|
||||
await pickElement(session.page, pickSelector, {
|
||||
position: fixture.runtime.pickPosition || null,
|
||||
resetPickMode: true,
|
||||
});
|
||||
if (args.action) await selectAction(session.page, String(args.action));
|
||||
await resetBrowserTimingProbe(session.page, `${iteration}-followup`);
|
||||
const nextFirstVariant = waitForFirstVariant(session.page);
|
||||
@@ -228,7 +239,9 @@ try {
|
||||
await clickDiscard(session.page);
|
||||
await waitForReset(session.page);
|
||||
}
|
||||
Object.assign(run, deriveJournalGenerationMetrics(await readGenerationSnapshot(session.tmp, run.eventId)));
|
||||
const generationSnapshot = await readGenerationSnapshot(session.tmp, run.eventId);
|
||||
Object.assign(run, deriveJournalGenerationMetrics(generationSnapshot));
|
||||
if (generationSnapshot.variantPlan) run.variantPlan = generationSnapshot.variantPlan;
|
||||
runs.push(run);
|
||||
|
||||
if (!args.quiet) process.stderr.write(formatRun(runs.at(-1)) + '\n');
|
||||
@@ -618,6 +631,19 @@ function assertScenarioEvidence(run, currentScenario) {
|
||||
}
|
||||
}
|
||||
|
||||
function assertSelectionEvidence(run, expected) {
|
||||
if (!expected) return;
|
||||
const actual = run.selectionEvidence || {};
|
||||
const expectedTag = String(expected.tagName || '').toLowerCase();
|
||||
const expectedClasses = Array.isArray(expected.classes) ? expected.classes.map(String) : [];
|
||||
if ((expectedTag && actual.tagName !== expectedTag)
|
||||
|| expectedClasses.some((className) => !actual.classes?.includes(className))) {
|
||||
throw new Error(
|
||||
`iteration ${run.iteration}: picked ${actual.tagName || 'unknown'}.${(actual.classes || []).join('.')} instead of ${expectedTag || '*'}.${expectedClasses.join('.')}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function wrapTargetFromPickedElement(event) {
|
||||
const element = event.element || {};
|
||||
return {
|
||||
|
||||
@@ -127,6 +127,10 @@ export function buildInteractionRun(events, { iteration, scenario, goStartedAt,
|
||||
iteration,
|
||||
scenario,
|
||||
eventId: id,
|
||||
selectionEvidence: {
|
||||
tagName: eventPost?.selectedTagName || null,
|
||||
classes: Array.isArray(eventPost?.selectedClasses) ? eventPost.selectedClasses : [],
|
||||
},
|
||||
annotationEvidence: {
|
||||
screenshotPath: eventPost?.hasScreenshotPath === true,
|
||||
comments: Number(eventPost?.commentCount || 0),
|
||||
|
||||
@@ -49,6 +49,8 @@
|
||||
"readyPattern": "Local:\\s+https?://[^:]+:(\\d+)",
|
||||
"readyTimeoutMs": 120000,
|
||||
"pickSelector": "article.offer-card",
|
||||
"pickPosition": { "x": 8, "y": 8 },
|
||||
"expectedPick": { "tagName": "article", "classes": ["offer-card"] },
|
||||
"acceptedSourcePattern": "<article[^>]*(class|className)=\"[^\"]*\\boffer-card\\b[^\"]*\"",
|
||||
"steer": {
|
||||
"message": "steer-e2e mark offer",
|
||||
|
||||
@@ -108,7 +108,7 @@ describe('live benchmark metrics', () => {
|
||||
it('separates model generation from Impeccable overhead', () => {
|
||||
const events = [
|
||||
{ name: 'ui.go.start', at: 100, iteration: 1 },
|
||||
{ name: 'browser.generate_post', at: 108, id: 'abc', hasScreenshotPath: false, commentCount: 0, strokeCount: 0 },
|
||||
{ name: 'browser.generate_post', at: 108, id: 'abc', selectedTagName: 'article', selectedClasses: ['offer-card'], hasScreenshotPath: false, commentCount: 0, strokeCount: 0 },
|
||||
{ name: 'agent.event.received', at: 110, id: 'abc', type: 'generate' },
|
||||
{ name: 'agent.scaffold.start', at: 112, id: 'abc' },
|
||||
{ name: 'agent.scaffold.end', at: 132, id: 'abc' },
|
||||
@@ -134,6 +134,7 @@ describe('live benchmark metrics', () => {
|
||||
assert.equal(run.browserDispatchMs, 2.5);
|
||||
assert.equal(run.automationClickMs, 5.5);
|
||||
assert.deepEqual(run.annotationEvidence, { screenshotPath: false, comments: 0, strokes: 0 });
|
||||
assert.deepEqual(run.selectionEvidence, { tagName: 'article', classes: ['offer-card'] });
|
||||
assert.equal(run.serverPickupMs, 2);
|
||||
assert.equal(run.generationMs, 1000);
|
||||
assert.equal(run.impeccableOverheadMs, 94.5);
|
||||
|
||||
Reference in New Issue
Block a user