mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-11 21:57:14 +03:00
Fix skill workflow regression coverage (#783)
Clarify launcher fallback and completed documentation handoffs; separate bounded protocol checkpoints from opt-in browser-backed completion diagnostics. Correct fixture containment, target syntax, and artifact assertions. AI assistance: Codex, under maintainer direction.
This commit is contained in:
+27
-20
@@ -8,6 +8,7 @@ export const OPT_IN_SUITES = [
|
||||
'live-e2e-accept-cleanup',
|
||||
'new-work-e2e',
|
||||
'skill-behavior',
|
||||
'skill-workflow',
|
||||
'live-svelte-adapter-deepseek',
|
||||
];
|
||||
|
||||
@@ -267,38 +268,44 @@ export const SUITES = {
|
||||
],
|
||||
},
|
||||
'skill-behavior': {
|
||||
description: 'LLM-backed skill setup behavior scenarios.',
|
||||
description: 'LLM-backed protocol checkpoints, not full builds.',
|
||||
optIn: true,
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^skill\/SKILL\.src\.md$/,
|
||||
/^skill\/reference\/(init|document|brand|product|shape|craft|audit|polish|live|routing)\.md$/,
|
||||
/^skill\/reference\//,
|
||||
/^ENGINE_VERSION$/,
|
||||
/^tests\/skill-behavior\//,
|
||||
],
|
||||
commands: [{
|
||||
runner: 'node',
|
||||
timeoutMs: 240000,
|
||||
wallClockMs: 1_800_000,
|
||||
files: ['tests/skill-behavior/scenarios.test.mjs'],
|
||||
}],
|
||||
},
|
||||
'skill-workflow': {
|
||||
description: 'Explicitly opt-in completed workflows with a preflighted browser.',
|
||||
optIn: true,
|
||||
needsPlaywright: true,
|
||||
triggers: [
|
||||
...COMMON_INFRA_PATTERNS,
|
||||
/^skill\//,
|
||||
/^ENGINE_VERSION$/,
|
||||
/^tests\/skill-workflow\//,
|
||||
/^tests\/skill-behavior\//,
|
||||
],
|
||||
commands: [
|
||||
{ runner: 'node', files: ['tests/skill-workflow-browser.test.mjs'] },
|
||||
{
|
||||
runner: 'node', timeoutMs: 240000, wallClockMs: 600000,
|
||||
files: ['tests/skill-workflow/finish-handoff.test.mjs'],
|
||||
},
|
||||
{
|
||||
runner: 'node',
|
||||
// 300000 was too low to measure what these scenarios assert. The
|
||||
// workflow-contract turns run 20+ steps against a frontier model, and
|
||||
// the *correct* path is the slow one: a run that stops to put the
|
||||
// concept to the user before building was measured at 579s, while the
|
||||
// runs that skipped that checkpoint and failed the assertion finished
|
||||
// in 130-200s. At a 300s cap the thorough path is killed and the hasty
|
||||
// path is graded, so the cap was selecting for the behavior the suite
|
||||
// exists to forbid.
|
||||
timeoutMs: 900000,
|
||||
// Overall wall-clock safety cap for the whole sweep: if a provider
|
||||
// call wedges past every inner guard (the harness's 840s per-turn
|
||||
// AbortSignal and the 900s per-test timeout), the runner SIGKILLs the
|
||||
// process group so the sweep still ends with a per-provider tally
|
||||
// instead of hanging overnight. Sized well above a healthy two-provider
|
||||
// sweep; override with IMPECCABLE_TEST_WALL_CLOCK_MS to scope it down.
|
||||
wallClockMs: 3_600_000,
|
||||
files: [
|
||||
'tests/skill-behavior/scenarios.test.mjs',
|
||||
'tests/skill-behavior/workflow-contract.test.mjs',
|
||||
],
|
||||
files: ['tests/skill-workflow/full-build.test.mjs'],
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user