mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-17 00:26:41 +03:00
Fix source-safety, detector, and lock defects in Live polling work
Addresses the review findings on #371, plus several the bots did not catch. All fixes have regression coverage that fails on the prior code. Source corruption: - Vue accept dropped valueless root attrs (disabled, v-cloak) and, worse, rewrote @click="x" as a literal click="x" DOM attribute, because the attr parser was name-anchored and skipped the sigil. Tokenize the whole Vue attr grammar and normalize shorthands so accept round-trips directives. - --variant was interpolated unescaped into a RegExp, so --variant '.*' matched the original block first and reported a successful accept while silently restoring the original. Validate against the digits pattern the browser and the /events schema already enforce. - --id reached path.join unvalidated, so --id ../../../../etc/evil wrote and read receipts outside the project. Hoist the existing safeSessionId check into impeccable-paths and apply it at every id-to-path sink. Accept/lock correctness: - Plain HTML/JSX accept and discard did not catch SOURCE_LOCKED, so contention exited non-zero with empty stdout and the agent got no JSON to retry on. - Lock staleness was mtime-only and never read the pid it records: a holder whose critical section outran 60s had its live lock swept, admitting a second writer to the same file, while a crashed holder blocked accepts for a full 60s. Decide staleness by owner liveness, and release only our own lock. Detector: - isNeutralColor only parses computed color forms, so routing authored CSS through it reported inset 4px 0 0 #000 / black / #e5e7eb as chromatic side-tab stripes. Add an authored-color neutrality test covering hex and named neutrals; the fixture had no literal-color cases at all. - Rule line numbers were off by one for every rule after the first, and commented-out CSS was scanned as live rules. Server: - An error reply carries no sourceEventType, and inferSourceEventType returned undefined, which acknowledgePendingEvent treats as a wildcard: a stale generate worker's failure consumed the user's queued Accept, which then reached no agent and left the browser in SAVING forever. - The generate preflight spawned live-wrap.mjs synchronously inside the request handler, freezing the single-threaded server for the whole scaffold (~7.6s measured on this repo, 15s ceiling) and stalling Accept/Discard/SSE. Make it async, claiming the lease before the first await so no event double-delivers. - Every browser checkpoint was echoed back as variant_progress, so a Tune slider drag remounted the preview under the user's cursor and latched the *_reviewable phases from the wrong trigger. Gate on the reason. Cleanup: - Collapse four divergent benchmark argv parsers into scripts/lib/cli-args.mjs. Three silently misread flags: --iterations 20 benchmarked 5, --agent llm ran the fake agent, --median-target=0.4 used the default threshold. - Drop a snapshot cache this branch made write-only (it grew per session for the server's lifetime and was never read), a dead exported reconcile helper, and the unused deferReply branch. Prepared with AI assistance under maintainer direction. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { bootFixtureSession, FIXTURES_DIR } from '../tests/live-e2e/session.mjs';
|
||||
import { createFakeAgent } from '../tests/live-e2e/agent.mjs';
|
||||
import { parseArgs, positiveIntFlag } from './lib/cli-args.mjs';
|
||||
import {
|
||||
clickAccept,
|
||||
clickGo,
|
||||
@@ -34,7 +35,7 @@ const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const FIXTURE_NAME = 'vite8-react-brand-fidelity';
|
||||
const SOURCE_FILE = 'src/App.jsx';
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const iterations = positiveInt(args.iterations, 1);
|
||||
const iterations = positiveIntFlag(args.iterations, 1);
|
||||
const providers = csv(args.providers || 'anthropic,openai,google');
|
||||
const strategies = csv(args.strategies || Object.keys(STRATEGIES).join(','));
|
||||
const outputPath = args.output ? resolve(ROOT, String(args.output)) : null;
|
||||
@@ -495,41 +496,10 @@ async function persist(report, file) {
|
||||
process.stderr.write(`[live-provider-bench] wrote ${file}\n`);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = {};
|
||||
for (let position = 0; position < argv.length; position += 1) {
|
||||
const arg = argv[position];
|
||||
if (!arg.startsWith('--')) continue;
|
||||
const body = arg.slice(2);
|
||||
const index = body.indexOf('=');
|
||||
if (index !== -1) {
|
||||
out[camel(body.slice(0, index))] = body.slice(index + 1);
|
||||
continue;
|
||||
}
|
||||
const next = argv[position + 1];
|
||||
if (next !== undefined && !next.startsWith('--')) {
|
||||
out[camel(body)] = next;
|
||||
position += 1;
|
||||
} else {
|
||||
out[camel(body)] = true;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function camel(value) {
|
||||
return value.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase());
|
||||
}
|
||||
|
||||
function csv(value) {
|
||||
return String(value).split(',').map((item) => item.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function positiveInt(value, fallback) {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function roundMs(value) {
|
||||
return Number(Number(value).toFixed(2));
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
waitForCycling,
|
||||
waitForHandshake,
|
||||
} from '../tests/live-e2e/ui.mjs';
|
||||
import { boolFlag, parseArgs, positiveIntFlag } from './lib/cli-args.mjs';
|
||||
import {
|
||||
buildInteractionRun,
|
||||
assembleSplitProgressiveOutput,
|
||||
@@ -26,18 +27,19 @@ import {
|
||||
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const fixtureName = String(args.fixture || 'vite8-react-plain');
|
||||
const iterations = positiveInt(args.iterations, 5);
|
||||
const iterations = positiveIntFlag(args.iterations, 5);
|
||||
const agentMode = args.agent === 'llm' ? 'llm' : 'fake';
|
||||
const scenario = args.scenario === 'annotated' ? 'annotated' : 'plain';
|
||||
const delivery = args.delivery === 'progressive' ? 'progressive' : 'atomic';
|
||||
const simulatedTailMs = positiveInt(args.simulatedTailMs, 0);
|
||||
const simulatedTailMs = positiveIntFlag(args.simulatedTailMs, 0);
|
||||
const quiet = boolFlag(args.quiet);
|
||||
const outputPath = args.output ? resolve(ROOT, String(args.output)) : null;
|
||||
const fixture = JSON.parse(await readFile(join(FIXTURES_DIR, fixtureName, 'fixture.json'), 'utf-8'));
|
||||
if (!fixture.runtime) throw new Error(`fixture ${fixtureName} has no runtime configuration`);
|
||||
if (fixture.runtime.mode === 'insert') throw new Error('live benchmark currently measures replace-mode fixtures only');
|
||||
|
||||
const { chromium } = await import('playwright');
|
||||
const browser = await chromium.launch({ headless: args.headed !== true });
|
||||
const browser = await chromium.launch({ headless: !boolFlag(args.headed) });
|
||||
const recorder = createTraceRecorder();
|
||||
let session;
|
||||
|
||||
@@ -56,7 +58,7 @@ try {
|
||||
progressive: delivery === 'progressive',
|
||||
progressiveDelayMs: delivery === 'progressive' ? simulatedTailMs : 0,
|
||||
atomicDelayMs: delivery === 'atomic' ? simulatedTailMs : 0,
|
||||
log: args.quiet ? () => {} : (message) => process.stderr.write(`[live-bench] ${message}\n`),
|
||||
log: quiet ? () => {} : (message) => process.stderr.write(`[live-bench] ${message}\n`),
|
||||
});
|
||||
|
||||
recorder.mark('setup.handshake.start');
|
||||
@@ -107,7 +109,7 @@ try {
|
||||
assertScenarioEvidence(run, scenario);
|
||||
runs.push(run);
|
||||
|
||||
if (!args.quiet) process.stderr.write(formatRun(runs.at(-1)) + '\n');
|
||||
if (!quiet) process.stderr.write(formatRun(runs.at(-1)) + '\n');
|
||||
await clickDiscard(session.page);
|
||||
await waitForReset(session.page);
|
||||
}
|
||||
@@ -277,23 +279,6 @@ function wrapTargetFromPickedElement(event) {
|
||||
};
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = {};
|
||||
for (const arg of argv) {
|
||||
if (!arg.startsWith('--')) continue;
|
||||
const body = arg.slice(2);
|
||||
const index = body.indexOf('=');
|
||||
if (index === -1) out[body] = true;
|
||||
else out[body.slice(0, index)] = body.slice(index + 1);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function positiveInt(value, fallback) {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function formatRun(run) {
|
||||
return `[live-bench] run ${run.iteration}: first=${run.goToFirstVariantMs}ms all=${run.goToAllVariantsMs}ms generation=${run.generationMs}ms overhead=${run.impeccableOverheadMs}ms`;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
import { parseArgs } from './lib/cli-args.mjs';
|
||||
import { compareModelBackedReports } from './lib/live-benchmark.mjs';
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
@@ -30,18 +31,8 @@ async function readReport(file, delivery) {
|
||||
return report;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = {};
|
||||
for (const arg of argv) {
|
||||
if (!arg.startsWith('--')) continue;
|
||||
const index = arg.indexOf('=');
|
||||
if (index > 2) out[arg.slice(2, index)] = arg.slice(index + 1);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function ratioArg(value, fallback) {
|
||||
if (value == null) return fallback;
|
||||
if (value == null || value === true) return fallback;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed) || parsed < 0 || parsed >= 1) throw new Error(`invalid threshold ratio: ${value}`);
|
||||
return parsed;
|
||||
|
||||
@@ -7,6 +7,7 @@ import { fileURLToPath } from 'node:url';
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
|
||||
import { FIXTURES_DIR } from '../tests/live-e2e/session.mjs';
|
||||
import { parseArgs } from './lib/cli-args.mjs';
|
||||
import { loadBenchmarkEnv } from './lib/live-provider-benchmark.mjs';
|
||||
import {
|
||||
buildRenderedReviewContext,
|
||||
@@ -83,28 +84,3 @@ function required(values, key) {
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = {};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (!arg.startsWith('--')) continue;
|
||||
const equals = arg.indexOf('=');
|
||||
if (equals !== -1) {
|
||||
out[toCamel(arg.slice(2, equals))] = arg.slice(equals + 1);
|
||||
continue;
|
||||
}
|
||||
const key = toCamel(arg.slice(2));
|
||||
const next = argv[index + 1];
|
||||
if (next && !next.startsWith('--')) {
|
||||
out[key] = next;
|
||||
index += 1;
|
||||
} else {
|
||||
out[key] = true;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function toCamel(value) {
|
||||
return value.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* One argv parser for the Live benchmark / judging scripts.
|
||||
*
|
||||
* These scripts had four subtly different hand-rolled parsers, and the gaps
|
||||
* failed silently rather than loudly: a parser without the `argv[i + 1]`
|
||||
* lookahead turned `--iterations 20` into `iterations: true` and benchmarked
|
||||
* the default 5 runs; a parser without kebab→camel mapping turned
|
||||
* `--median-target=0.4` into a key nothing read, so the comparison ran against
|
||||
* the default threshold. Both produce a clean-looking report of the wrong thing.
|
||||
*
|
||||
* Supported forms, per flag:
|
||||
* --flag → true
|
||||
* --flag=value → 'value'
|
||||
* --flag value → 'value' (unless `value` itself starts with `--`)
|
||||
*
|
||||
* Keys are camel-cased, so `--simulated-tail-ms` and `--simulatedTailMs` both
|
||||
* land on `simulatedTailMs`.
|
||||
*/
|
||||
export function parseArgs(argv) {
|
||||
const out = {};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (!arg.startsWith('--')) continue;
|
||||
const body = arg.slice(2);
|
||||
if (!body) continue;
|
||||
const equals = body.indexOf('=');
|
||||
if (equals !== -1) {
|
||||
out[toCamel(body.slice(0, equals))] = body.slice(equals + 1);
|
||||
continue;
|
||||
}
|
||||
const next = argv[index + 1];
|
||||
if (next !== undefined && !next.startsWith('--')) {
|
||||
out[toCamel(body)] = next;
|
||||
index += 1;
|
||||
} else {
|
||||
out[toCamel(body)] = true;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function toCamel(value) {
|
||||
return String(value).replace(/-([a-z0-9])/gi, (_, char) => char.toUpperCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a boolean flag. `--headed` and `--headed=true` must mean the same thing;
|
||||
* comparing the raw value against `true` silently ignores the second form.
|
||||
*/
|
||||
export function boolFlag(value, fallback = false) {
|
||||
if (value === undefined) return fallback;
|
||||
if (typeof value === 'boolean') return value;
|
||||
const normalized = String(value).trim().toLowerCase();
|
||||
if (['', 'true', '1', 'yes', 'on'].includes(normalized)) return true;
|
||||
if (['false', '0', 'no', 'off'].includes(normalized)) return false;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a positive integer flag, falling back when absent. Throws on a value
|
||||
* that was clearly meant as a number but isn't one, so `--iterations abc`
|
||||
* fails instead of quietly benchmarking the default.
|
||||
*/
|
||||
export function positiveIntFlag(value, fallback) {
|
||||
if (value === undefined || value === true) return fallback;
|
||||
const parsed = Number.parseInt(String(value), 10);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0 || String(parsed) !== String(value).trim()) {
|
||||
throw new Error(`expected a positive integer, got: ${value}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
@@ -54,6 +54,7 @@ export const SUITES = {
|
||||
runner: 'node',
|
||||
files: [
|
||||
'tests/ci-test-plan.test.mjs',
|
||||
'tests/cli-args.test.mjs',
|
||||
'tests/context.test.mjs',
|
||||
'tests/context-signals.test.mjs',
|
||||
'tests/critique-storage.test.mjs',
|
||||
@@ -149,6 +150,7 @@ export const SUITES = {
|
||||
'tests/live-rendered-quality.test.mjs',
|
||||
'tests/live-server.test.mjs',
|
||||
'tests/live-session-store.test.mjs',
|
||||
'tests/live-source-lock.test.mjs',
|
||||
'tests/live-target-context.test.mjs',
|
||||
'tests/live-vue-component.test.mjs',
|
||||
'tests/live-wrap.test.mjs',
|
||||
|
||||
Reference in New Issue
Block a user