Separate protocol checkpoints from provisioned workflow tests

Keep full completion and current visual evidence distinct from setup routing. Add opt-in preflighted browser tools and explicit failure outcomes. AI assistance: Codex, under maintainer direction.
This commit is contained in:
Paul Bakaus
2026-09-07 16:38:58 -07:00
parent 75ad869c06
commit e1fe48b05a
14 changed files with 472 additions and 48 deletions
+23 -20
View File
@@ -8,6 +8,7 @@ export const OPT_IN_SUITES = [
'live-e2e-accept-cleanup',
'new-work-e2e',
'skill-behavior',
'skill-workflow',
'live-svelte-adapter-deepseek',
];
@@ -267,38 +268,40 @@ export const SUITES = {
],
},
'skill-behavior': {
description: 'LLM-backed skill setup behavior scenarios.',
description: 'LLM-backed protocol checkpoints, not full builds.',
optIn: true,
triggers: [
...COMMON_INFRA_PATTERNS,
/^skill\/SKILL\.src\.md$/,
/^skill\/reference\/(init|document|brand|product|shape|craft|audit|polish|live|routing)\.md$/,
/^skill\/reference\//,
/^ENGINE_VERSION$/,
/^tests\/skill-behavior\//,
],
commands: [{
runner: 'node',
timeoutMs: 240000,
wallClockMs: 1_800_000,
files: ['tests/skill-behavior/scenarios.test.mjs'],
}],
},
'skill-workflow': {
description: 'Explicitly opt-in completed workflows with a preflighted browser.',
optIn: true,
needsPlaywright: true,
triggers: [
...COMMON_INFRA_PATTERNS,
/^skill\//,
/^ENGINE_VERSION$/,
/^tests\/skill-workflow\//,
/^tests\/skill-behavior\//,
],
commands: [
{ runner: 'node', files: ['tests/skill-workflow-browser.test.mjs'] },
{
runner: 'node',
// 300000 was too low to measure what these scenarios assert. The
// workflow-contract turns run 20+ steps against a frontier model, and
// the *correct* path is the slow one: a run that stops to put the
// concept to the user before building was measured at 579s, while the
// runs that skipped that checkpoint and failed the assertion finished
// in 130-200s. At a 300s cap the thorough path is killed and the hasty
// path is graded, so the cap was selecting for the behavior the suite
// exists to forbid.
timeoutMs: 900000,
// Overall wall-clock safety cap for the whole sweep: if a provider
// call wedges past every inner guard (the harness's 840s per-turn
// AbortSignal and the 900s per-test timeout), the runner SIGKILLs the
// process group so the sweep still ends with a per-provider tally
// instead of hanging overnight. Sized well above a healthy two-provider
// sweep; override with IMPECCABLE_TEST_WALL_CLOCK_MS to scope it down.
wallClockMs: 3_600_000,
files: [
'tests/skill-behavior/scenarios.test.mjs',
'tests/skill-behavior/workflow-contract.test.mjs',
],
files: ['tests/skill-workflow/full-build.test.mjs'],
},
],
},