From 36457e191ffbfaefd8f31c70318d048e3881cd6b Mon Sep 17 00:00:00 2001 From: Paul Bakaus Date: Sat, 15 Aug 2026 22:03:18 -0400 Subject: [PATCH] Bump skill-behavior test lineup: gemini-3.7-flash (#598) * Bump skill-behavior google lineup to gemini-3.7-flash gemini-3.7-flash replaces gemini-3.6-flash in DEFAULT_MODELS. The README notes that the recorded gemini baseline cells were measured on 3.6-flash (or 3.5-flash where marked) and count as unmeasured on 3.7 per the suite's own cross-version rule, to be re-run on the next Setup or routing change. AI-assisted change. Co-Authored-By: Claude Fable 5 * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Claude Fable 5 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/skill-behavior/README.md | 6 ++++++ tests/skill-behavior/providers.mjs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/skill-behavior/README.md b/tests/skill-behavior/README.md index 01054e417..c9bb5038a 100644 --- a/tests/skill-behavior/README.md +++ b/tests/skill-behavior/README.md @@ -90,6 +90,12 @@ Against the current default lineup, two cells are the known floor: `redesign replaces DESIGN` is flaky on every model, and `critique closes` is flaky on gemini-3.6-flash. A regression is a failure beyond those two. +The Google slot in `DEFAULT_MODELS` moved to `gemini-3.7-flash` on 2026-08-15. +Every Gemini cell in the tables below was measured on 3.6-flash (or 3.5-flash +where marked), and per the cross-version rule further down, those results are +unmeasured on 3.7, not inherited. Re-run the sweep on the next Setup or routing +change and update the tables to the new column. + **Read any failure against the clock before calling it behavior.** The suite ran at a 300s per-test timeout until 2026-08-13, and for the workflow-contract scenarios that cap was below the runtime of a correct run. `initialized natural diff --git a/tests/skill-behavior/providers.mjs b/tests/skill-behavior/providers.mjs index d6bf0dc8b..9d31e4acf 100644 --- a/tests/skill-behavior/providers.mjs +++ b/tests/skill-behavior/providers.mjs @@ -127,7 +127,7 @@ export function getProviderOptions(modelId) { * its own floor: * IMPECCABLE_SKILL_BEHAVIOR_MODELS=gpt-5.6-luna,deepseek-v4-flash */ -export const DEFAULT_MODELS = ['claude-sonnet-5', 'gpt-5.6-terra', 'gemini-3.6-flash']; +export const DEFAULT_MODELS = ['claude-sonnet-5', 'gpt-5.6-terra', 'gemini-3.7-flash']; export function resolveModelList() { const override = process.env.IMPECCABLE_SKILL_BEHAVIOR_MODELS;