mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-19 17:46:36 +03:00
Stop the progressive benchmark agent inventing a second variant on count:1
`Math.max(1, event.count - 1)` floored the tail request at one variant, so a one-variant request fetched a second direction and assembled two. Ask for `count - 1` and return the first variant untouched when there is no tail. Latent rather than live: the only caller hardcodes `count: 3`. The reason it is worth fixing is the caller inconsistency it exposed. tests/live-e2e/agent.mjs gates its split-progressive path on `event.count > 1`; benchmark-live-providers.mjs had no such guard, so it would have run the tail for a one-variant request, and the parallel strategy would have assembled its three fixed lanes regardless of what was asked for. Guard the caller the same way. Prepared with AI assistance under maintainer direction. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -175,7 +175,13 @@ async function runGenerationOne({ liveSpec: loadedLiveSpec, providerConfig, stra
|
||||
if (typeof agent.generateFirstVariant === 'function') {
|
||||
firstOutput = await agent.generateFirstVariant(event, {});
|
||||
firstReviewableMs = roundMs(performance.now() - startedAt);
|
||||
output = await agent.generateRemainingVariants(event, { firstOutput });
|
||||
// Only ask for a tail when one was requested. tests/live-e2e/agent.mjs
|
||||
// already gates its split-progressive path on `event.count > 1`; without the
|
||||
// same guard here a one-variant request still ran the tail, and the
|
||||
// parallel strategy would assemble its three fixed lanes regardless.
|
||||
output = Number(event.count) > 1
|
||||
? await agent.generateRemainingVariants(event, { firstOutput })
|
||||
: firstOutput;
|
||||
} else {
|
||||
output = await agent.generateVariants(event, {});
|
||||
firstReviewableMs = roundMs(performance.now() - startedAt);
|
||||
|
||||
@@ -309,8 +309,16 @@ export function createProviderLiveAgent({ provider, model, strategy, liveSpec, o
|
||||
async generateRemainingVariants(event, context) {
|
||||
const first = pendingFirst.get(event.id) || context.firstOutput;
|
||||
if (!first?.variants?.[0]) throw new Error(`first variant state missing for ${event.id}`);
|
||||
const tailCount = Number(event.count) - 1;
|
||||
// A one-variant request has no tail. Math.max(1, ...) floored the count at
|
||||
// one, so this fetched a second direction and assembled two variants for a
|
||||
// set the caller asked to be one.
|
||||
if (tailCount < 1) {
|
||||
pendingFirst.delete(event.id);
|
||||
return first;
|
||||
}
|
||||
const remaining = await request({
|
||||
event: { ...event, count: Math.max(1, event.count - 1) },
|
||||
event: { ...event, count: tailCount },
|
||||
phase: 'remaining-directions',
|
||||
firstVariant: first.variants[0],
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user