mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-12 06:06:37 +03:00
Comparing this branch's live against main's turned up two whole features that never made sense here. -2,466 lines. 1. The isolated source-artifact preview was never switched on. `scaffoldSourceArtifactSession` is only reachable via live-wrap's `--isolated`, and nothing passes it: not the server's preflight, not live.md, nothing. Proved it end-to-end — the default wrap writes markers straight into real source and creates no previews/ session. So the mode was wired through three modules, carried its own accept/discard branches, browser branches, server metadata resolution, preview-mode classifier entry, and test suites, and none of it could run. Worse, live.md documented it as the active path and told the agent "The true source is only the publisher's hash fence and must remain byte-identical until Accept." That is false: the wrapper lands in source at scaffold time and each revision rewrites it. An agent following that sentence believes source is protected when it isn't, and the leftover artifacts are what made accept resolve the wrong file in the first real run. live.md now describes what actually happens, including that markers are visible in source until Accept or Discard. Removed: source-artifact.mjs, --isolated, the preflight's isolated option, the accept/discard branches, four dead browser branches, the server's previews/ resolution, the classifier entry, and their tests. Kept the previews/ gitignore pattern: an ignore line for a directory that cannot exist is free, and a test pins it. 2. Quality judging belongs to the private evals repo, which says so. runner/live/README.md there is explicit: the public repo owns protocol correctness, framework coverage, timing, source commit, recovery, and a rubric-free evidence bundle; the private repo owns the task corpus, baselines, comparative judges, and release-quality decisions — "Do not add quality rubrics, competitor comparisons, or broad fixture corpora to the public Live benchmark." This branch added exactly those: an LLM judge scoring 1-10 on "off-brand, generic-AI" (live-rendered-quality.mjs, judge-live-rendered.mjs), a cross-provider comparison with a BRAND_CONTRACT rubric (live-provider-benchmark .mjs, benchmark-live-providers.mjs), and a brand-fidelity fixture corpus. All removed, with bench:live:providers and their suite entries. Also removed tests/framework-fixtures/README.md's "External quality-eval fixtures" section: it documented a bench:live workflow using --fixture-dir, --agent=codex, --action and --evidence-bundle, none of which benchmark-live.mjs implements, plus an evidenceCapture block nothing reads. Kept: timing benchmarks (the public repo's half of that boundary), progressive publication, the source lock, poll lanes, and Nuxt/Vue component previews. Coverage note: deleting the isolated suites took the only tests for `source_locked` classification with them, so the plain wrapper path — now the only non-component preview — gets equivalent accept and discard coverage. Both new tests fail if mode:'error' is removed. Prepared with AI assistance under maintainer direction. Co-Authored-By: Claude <noreply@anthropic.com>
84 lines
3.7 KiB
JavaScript
84 lines
3.7 KiB
JavaScript
import { describe, it } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
|
|
import {
|
|
completionAckForAcceptResult,
|
|
completionTypeForAcceptResult,
|
|
} from '../skill/scripts/live/completion.mjs';
|
|
|
|
describe('live completion type classification', () => {
|
|
it('treats generated-file fallback accept as normal agent handoff, not error', () => {
|
|
assert.equal(
|
|
completionTypeForAcceptResult('accept', { handled: false, mode: 'fallback' }),
|
|
'agent_done',
|
|
'event=live_poll.fallback_completion actor=agent operation=accept_generated_file risk=fallback_handoff_recorded_as_agent_error expected=agent_done actual=error',
|
|
);
|
|
});
|
|
|
|
it('treats unhandled non-error accept as normal manual agent handoff', () => {
|
|
assert.equal(
|
|
completionTypeForAcceptResult('accept', { handled: false, error: 'Session markers not found' }),
|
|
'agent_done',
|
|
'event=live_poll.manual_accept_completion actor=agent operation=accept_manual_cleanup risk=manual_handoff_recorded_as_agent_error expected=agent_done actual=error',
|
|
);
|
|
});
|
|
|
|
it('keeps carbonize-required accepts recoverable until cleanup is completed', () => {
|
|
assert.equal(
|
|
completionTypeForAcceptResult('accept', { handled: true, carbonize: true }),
|
|
'agent_done',
|
|
'event=live_poll.carbonize_completion actor=agent operation=accept_with_carbonize risk=carbonize_session_marked_completed_before_cleanup expected=agent_done actual=complete',
|
|
);
|
|
});
|
|
|
|
it('marks carbonize acknowledgements as non-final and requiring explicit completion', () => {
|
|
assert.deepEqual(
|
|
completionAckForAcceptResult('carbonize-1', 'agent_done', { handled: true, carbonize: true }),
|
|
{
|
|
ok: true,
|
|
type: 'agent_done',
|
|
final: false,
|
|
requiresComplete: true,
|
|
nextCommand: 'live-complete.mjs --id carbonize-1',
|
|
message: 'Carbonize cleanup must be verified, then the session must be completed explicitly before polling again.',
|
|
},
|
|
'event=live_poll.carbonize_ack actor=agent operation=accept_with_carbonize risk=active_session_never_completed expected=explicit_complete_required actual=missing_requires_complete',
|
|
);
|
|
});
|
|
|
|
it('keeps normal handled accepts terminal', () => {
|
|
assert.deepEqual(
|
|
completionAckForAcceptResult('done-1', 'complete', { handled: true, carbonize: false }),
|
|
{ ok: true, type: 'complete' },
|
|
);
|
|
});
|
|
|
|
// Component previews keep their variants in module files, not in the user's
|
|
// source, so a failed accept leaves nothing to hand-edit: that is a failure, not
|
|
// live.md's "read file, find markers, edit" handoff. Only svelte-component was
|
|
// special cased, so the identical failure on a Vue preview read as success.
|
|
for (const previewMode of ['svelte-component', 'vue-component']) {
|
|
it(`treats a failed ${previewMode} accept as an error, not a manual handoff`, () => {
|
|
assert.equal(
|
|
completionTypeForAcceptResult('accept', { handled: false, error: 'source_locked', previewMode }),
|
|
'error',
|
|
);
|
|
});
|
|
}
|
|
|
|
it('still treats a failed plain-wrapper accept as a manual handoff', () => {
|
|
// The one shape with editable markers in source. This must not regress into
|
|
// an error, or every hand-editable session starts failing the poll loop.
|
|
assert.equal(
|
|
completionTypeForAcceptResult('accept', { handled: false, error: 'Markers not found' }),
|
|
'agent_done',
|
|
);
|
|
});
|
|
|
|
it('classifies handled accept/discard and real failures explicitly', () => {
|
|
assert.equal(completionTypeForAcceptResult('accept', { handled: true }), 'complete');
|
|
assert.equal(completionTypeForAcceptResult('discard', { handled: true }), 'discarded');
|
|
assert.equal(completionTypeForAcceptResult('accept', { handled: false, mode: 'error', error: 'boom' }), 'error');
|
|
});
|
|
});
|