Fix skill workflow regression coverage (#783)

Clarify launcher fallback and completed documentation handoffs; separate bounded protocol checkpoints from opt-in browser-backed completion diagnostics. Correct fixture containment, target syntax, and artifact assertions. AI assistance: Codex, under maintainer direction.
This commit is contained in:
Paul Bakaus
2026-09-08 08:41:54 -07:00
committed by GitHub
parent 2bc2879276
commit 6496f49a1e
23 changed files with 1305 additions and 154 deletions
+27 -20
View File
@@ -8,6 +8,7 @@ export const OPT_IN_SUITES = [
'live-e2e-accept-cleanup',
'new-work-e2e',
'skill-behavior',
'skill-workflow',
'live-svelte-adapter-deepseek',
];
@@ -267,38 +268,44 @@ export const SUITES = {
],
},
'skill-behavior': {
description: 'LLM-backed skill setup behavior scenarios.',
description: 'LLM-backed protocol checkpoints, not full builds.',
optIn: true,
triggers: [
...COMMON_INFRA_PATTERNS,
/^skill\/SKILL\.src\.md$/,
/^skill\/reference\/(init|document|brand|product|shape|craft|audit|polish|live|routing)\.md$/,
/^skill\/reference\//,
/^ENGINE_VERSION$/,
/^tests\/skill-behavior\//,
],
commands: [{
runner: 'node',
timeoutMs: 240000,
wallClockMs: 1_800_000,
files: ['tests/skill-behavior/scenarios.test.mjs'],
}],
},
'skill-workflow': {
description: 'Explicitly opt-in completed workflows with a preflighted browser.',
optIn: true,
needsPlaywright: true,
triggers: [
...COMMON_INFRA_PATTERNS,
/^skill\//,
/^ENGINE_VERSION$/,
/^tests\/skill-workflow\//,
/^tests\/skill-behavior\//,
],
commands: [
{ runner: 'node', files: ['tests/skill-workflow-browser.test.mjs'] },
{
runner: 'node', timeoutMs: 240000, wallClockMs: 600000,
files: ['tests/skill-workflow/finish-handoff.test.mjs'],
},
{
runner: 'node',
// 300000 was too low to measure what these scenarios assert. The
// workflow-contract turns run 20+ steps against a frontier model, and
// the *correct* path is the slow one: a run that stops to put the
// concept to the user before building was measured at 579s, while the
// runs that skipped that checkpoint and failed the assertion finished
// in 130-200s. At a 300s cap the thorough path is killed and the hasty
// path is graded, so the cap was selecting for the behavior the suite
// exists to forbid.
timeoutMs: 900000,
// Overall wall-clock safety cap for the whole sweep: if a provider
// call wedges past every inner guard (the harness's 840s per-turn
// AbortSignal and the 900s per-test timeout), the runner SIGKILLs the
// process group so the sweep still ends with a per-provider tally
// instead of hanging overnight. Sized well above a healthy two-provider
// sweep; override with IMPECCABLE_TEST_WALL_CLOCK_MS to scope it down.
wallClockMs: 3_600_000,
files: [
'tests/skill-behavior/scenarios.test.mjs',
'tests/skill-behavior/workflow-contract.test.mjs',
],
files: ['tests/skill-workflow/full-build.test.mjs'],
},
],
},