From 07663f5fbd5173e989e2660ecf29e574be7a1a29 Mon Sep 17 00:00:00 2001 From: Paul Bakaus Date: Thu, 13 Aug 2026 22:04:53 -0400 Subject: [PATCH] Stop the update directive from spelling out a command it forbids Two defects the skill-behavior baseline had recorded as failing on main. `UPDATE_AVAILABLE` told the agent to ask once, then said "If they agree, run `npx impeccable update`", then said to continue without waiting. Nothing gated the run on an answer, and the same sentence removed the wait that could have produced one, so the command read as the next step and sonnet took it. The offer stays; the command leaves the turn. Running it mid-session rewrites the files the session is reading and only takes effect next session, so there is nothing to gain by running it now, and the directive says that rather than relying on the model to infer it. Failed 3 of 3 before, passes 3 of 3 after. Scenario 15 was a broken fixture, not a routing defect. The iOS workspace held PRODUCT.md and nothing else, so `audit the app in this workspace` named an app that was not there: sonnet spent its step budget hunting for it, including a `find /` across the filesystem, and read no reference file at all. The assertion reported "loaded audit.md instead of the variant" when the truth was "loaded neither". One SwiftUI screen makes the request answerable, and the scenario then passes on unmodified main, which is the evidence that the skill text was never at fault. This is the convention MINIMAL_LANDING_HTML already established for the web scenarios; the native fixture never received it. Written with AI assistance (Claude Code). Co-Authored-By: Claude Opus 5 (1M context) --- skill/scripts/context.mjs | 14 ++++++-- tests/context.test.mjs | 12 +++++++ tests/skill-behavior/fixtures.mjs | 43 +++++++++++++++++++++++++ tests/skill-behavior/scenarios.test.mjs | 5 +-- 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/skill/scripts/context.mjs b/skill/scripts/context.mjs index 2c9ca46ea..096c400cf 100644 --- a/skill/scripts/context.mjs +++ b/skill/scripts/context.mjs @@ -1013,14 +1013,22 @@ async function fetchLatestSkillVersion() { } } +// Two instructions used to sit in one directive: ask, and "if they agree, run +// it". Nothing gated the second on an answer, and the same sentence said to +// continue without waiting, so a run that could never establish agreement was +// still spelled out as the next command. The offer stays; the command leaves +// this turn entirely, because installing over the skill mid-session changes +// files the session is reading and only takes effect in the next one anyway. function buildUpdateDirective(localVersion, latestVersion) { return ( `UPDATE_AVAILABLE: A newer Impeccable skill is available ` + `(installed v${localVersion}, latest v${latestVersion}). ` + - `Before continuing, ask the user once: "A newer Impeccable (v${latestVersion}) is available. ` + + `Mention it once, in this form: "A newer Impeccable (v${latestVersion}) is available. ` + `Update now? It runs \`npx impeccable update\`." ` + - `If they agree, run \`npx impeccable update\` (the update applies to the next session, not this one). ` + - `Either way, continue the current task without waiting, and do not raise this again.` + `Do not run \`npx impeccable update\` in this turn, whatever the user answers: it rewrites the skill files ` + + `this session is reading, and the update only takes effect in the next session, so there is nothing to gain now. ` + + `Run it in a later turn, only after the user has asked for it in their own words. ` + + `Continue the current task now without waiting, and do not raise this again.` ); } diff --git a/tests/context.test.mjs b/tests/context.test.mjs index 1378d123f..ea2b55dbf 100644 --- a/tests/context.test.mjs +++ b/tests/context.test.mjs @@ -1451,6 +1451,18 @@ describe('context.mjs update check', () => { assert.match(res.stdout, /^# PRODUCT\.md/); }); + // The directive used to say "ask once" and "if they agree, run it" while also + // saying to continue without waiting. Nothing gated the run on an answer that + // could not arrive, so the command read as the next step. It now forbids + // running in this turn outright, whatever the answer. + it('forbids running the update in the same turn, on any answer', () => { + const { stdout } = run({ lastCheck: Date.now(), latestVersion: '2.0.0' }); + assert.match(stdout, /Do not run `npx impeccable update` in this turn, whatever the user answers/); + assert.match(stdout, /only after the user has asked for it in their own words/); + // The conditional that made the command look reachable must be gone. + assert.equal(/If they agree, run/.test(stdout), false); + }); + it('stays silent when the cached latest version is not newer', () => { const res = run({ lastCheck: Date.now(), latestVersion: '0.0.1' }); assert.equal(res.status, 0); diff --git a/tests/skill-behavior/fixtures.mjs b/tests/skill-behavior/fixtures.mjs index 0479d76bf..429d70950 100644 --- a/tests/skill-behavior/fixtures.mjs +++ b/tests/skill-behavior/fixtures.mjs @@ -162,6 +162,49 @@ Dynamic Type, VoiceOver, reduced motion, high contrast in direct sun, and targets usable one-handed with wet hands. `; +/** + * The native counterpart to MINIMAL_LANDING_HTML, and it exists for the same + * reason. A native scenario carrying only PRODUCT.md gives an audit nothing to + * audit: the agent goes looking for the app it was told exists, and a routing + * assertion ends up measuring how a model copes with an empty workspace + * instead. One screen is enough to make the request answerable. + */ +export const MINIMAL_IOS_SOURCE = `import SwiftUI + +struct TideDetailView: View { + let station: String + @State private var showsLog = false + + var body: some View { + NavigationStack { + List { + Section("Next window") { + HStack { + Text("High") + Spacer() + Text("4:12 PM").foregroundStyle(.secondary) + } + HStack { + Text("Low") + Spacer() + Text("10:38 PM").foregroundStyle(.secondary) + } + } + Section { + Button("Log a catch") { showsLog = true } + } + } + .navigationTitle(station) + .toolbar { + ToolbarItem(placement: .topBarTrailing) { + Button("Refresh") { } + } + } + } + } +} +`; + /** * Tiny static landing page fixture for scenarios that invoke sub-commands * (polish, audit) without standing up a full framework project. Gives the diff --git a/tests/skill-behavior/scenarios.test.mjs b/tests/skill-behavior/scenarios.test.mjs index a22fe8c78..e1304c2b8 100644 --- a/tests/skill-behavior/scenarios.test.mjs +++ b/tests/skill-behavior/scenarios.test.mjs @@ -29,6 +29,7 @@ import { PRODUCT_MD_SAMPLE, PRODUCT_MD_SAMPLE_NO_REGISTER, PRODUCT_MD_SAMPLE_IOS, + MINIMAL_IOS_SOURCE, DESIGN_MD_SAMPLE, MINIMAL_LANDING_HTML, SVELTE_PROJECT_FILES, @@ -531,7 +532,7 @@ for (const modelId of resolveModelList()) { // reference/ios.md itself, so native guidance enters the conversation // without relying on a second model-directed file read. const workspace = prepareWorkspace({ - files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_IOS }, + files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_IOS, 'TideDetailView.swift': MINIMAL_IOS_SOURCE }, }); try { const { trace, text } = await runTurn({ @@ -566,7 +567,7 @@ for (const modelId of resolveModelList()) { // switching via its web-only guard is acceptable; never reaching the // variant is the failure). const workspace = prepareWorkspace({ - files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_IOS }, + files: { 'PRODUCT.md': PRODUCT_MD_SAMPLE_IOS, 'TideDetailView.swift': MINIMAL_IOS_SOURCE }, }); try { const { trace, text } = await runTurn({