mirror of
https://github.com/pbakaus/impeccable.git
synced 2026-09-12 14:16:28 +03:00
Live: polling rework, source locks, preflight scaffolding, Vue previews
Carved out of #371, minus progressive publication. Everything here works against real project source the way main's Live already does: the agent writes variants into the file the browser loaded, HMR fires, Accept promotes and carbonizes. Nothing is staged anywhere. Poll lanes. Events now carry an explicit priority: accept/discard/exit ahead of manual_edit_apply/steer/carbonize_cleanup ahead of generate. A long generate can no longer sit in front of the Accept the user just clicked. leaseEvent claims its lease before awaiting, so a slow prepare cannot hand the same event to two pollers. Source locks. A per-file mutex around every accept and discard path, keyed on a digest of the absolute path. Staleness is decided by owner-pid liveness rather than mtime, so a wedged lock clears when its owner dies instead of after an arbitrary timeout, and a slow-but-live accept is never stolen from. Only the owning process can release a lock. Preflight scaffolding. The server runs live-wrap (or live-insert) before the poll returns and hands the result back as event.scaffold. That walk is measured at ~7.6s on a large repo; moving it off the agent's critical path removes a deterministic tool round trip without touching the generated design. Falls back cleanly to the agent running the helper itself. Vue previews. previewMode: "vue-component" for Nuxt/Vue targets, matching the existing Svelte component path: variants compile as real SFCs from a dev-only directory so the route is never rewritten during generation, and Vite mounts them without invalidating page state. Accept is the only route write. Includes a Vue attr tokenizer that normalizes shorthand bindings (@x, :x, #x) to their canonical forms. Accept hardening. Every thrown failure now returns mode: 'error' rather than an ambiguous unhandled result, so a real failure is never classified as a deliberate manual handoff and silently dropped. The marker search skips node_modules/.git/dist/build/.impeccable. Shared CLI arg parsing extracted to scripts/lib/cli-args.mjs. Assisted-by: Claude Code
This commit is contained in:
Paul Bakaus
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { parseArgs, positiveIntFlag } from './lib/cli-args.mjs';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const iterations = positiveIntFlag(args.iterations, 5);
|
||||
const fixture = args.fixture ? String(args.fixture) : 'vite8-react-plain';
|
||||
const metricsFile = path.join(os.tmpdir(), 'impeccable-live-control-' + process.pid + '.jsonl');
|
||||
|
||||
try {
|
||||
for (let index = 0; index < iterations; index += 1) {
|
||||
execFileSync('bun', ['run', 'test:live-e2e'], {
|
||||
cwd: root,
|
||||
stdio: 'ignore',
|
||||
timeout: 120_000,
|
||||
env: {
|
||||
...process.env,
|
||||
IMPECCABLE_E2E_ONLY: fixture,
|
||||
IMPECCABLE_E2E_SCENARIOS: 'progressive',
|
||||
IMPECCABLE_E2E_METRICS_FILE: metricsFile,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const rows = readMetrics(metricsFile);
|
||||
console.log(JSON.stringify({
|
||||
fixture,
|
||||
iterations: rows.length,
|
||||
measuredAt: new Date().toISOString(),
|
||||
acceptToPicking: summarize(rows.map((row) => row.acceptToPickingMs)),
|
||||
nextGoToPickup: summarize(rows.map((row) => row.nextGoToPickupMs)),
|
||||
samples: rows,
|
||||
}, null, 2));
|
||||
} finally {
|
||||
try { fs.unlinkSync(metricsFile); } catch {}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the metrics the e2e run appended. Fail loudly rather than reporting a
|
||||
* summary of nothing: an absent file means the run never produced a sample, and
|
||||
* an ENOENT stack or a `{"medianMs": null}` report both read as "measured" when
|
||||
* nothing was measured at all.
|
||||
*/
|
||||
function readMetrics(file) {
|
||||
let raw;
|
||||
try {
|
||||
raw = fs.readFileSync(file, 'utf-8');
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') throw error;
|
||||
throw new Error(`no metrics were recorded at ${file}. Did the e2e run emit IMPECCABLE_E2E_METRICS_FILE rows?`);
|
||||
}
|
||||
const rows = raw.trim().split('\n').filter(Boolean).map((line, index) => {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch (error) {
|
||||
throw new Error(`metrics line ${index + 1} is not valid JSON: ${error.message}`);
|
||||
}
|
||||
});
|
||||
if (rows.length === 0) throw new Error(`metrics file ${file} is empty; nothing to summarize`);
|
||||
return rows;
|
||||
}
|
||||
|
||||
function summarize(values) {
|
||||
const sorted = values.filter((value) => Number.isFinite(value)).sort((a, b) => a - b);
|
||||
// Distinguish "every sample was missing this metric" from a real measurement.
|
||||
// percentile() on an empty array reads sorted[-1] and yields NaN, which
|
||||
// JSON.stringify turns into null and silently passes for a result.
|
||||
if (sorted.length === 0) return { samples: 0, medianMs: null, p95Ms: null, minMs: null, maxMs: null };
|
||||
return {
|
||||
samples: sorted.length,
|
||||
medianMs: percentile(sorted, 0.5),
|
||||
p95Ms: percentile(sorted, 0.95),
|
||||
minMs: sorted[0],
|
||||
maxMs: sorted.at(-1),
|
||||
};
|
||||
}
|
||||
|
||||
function percentile(sorted, p) {
|
||||
const index = (sorted.length - 1) * p;
|
||||
const lower = Math.floor(index);
|
||||
const upper = Math.ceil(index);
|
||||
return Math.round((sorted[lower] * (1 - (index - lower)) + sorted[upper] * (index - lower)) * 100) / 100;
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { parseArgs, positiveIntFlag } from './lib/cli-args.mjs';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const liveScript = path.join(root, 'skill/scripts/live.mjs');
|
||||
const serverScript = path.join(root, 'skill/scripts/live-server.mjs');
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const iterations = positiveIntFlag(args.iterations, 10);
|
||||
const fixture = args.fixture ? String(args.fixture) : 'vite8-react-plain';
|
||||
const fixtureDir = path.join(root, 'tests/framework-fixtures', fixture, 'files');
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'impeccable-live-init-'));
|
||||
|
||||
try {
|
||||
fs.cpSync(fixtureDir, tmp, { recursive: true });
|
||||
fs.writeFileSync(path.join(tmp, 'PRODUCT.md'), '# Product\n\nA realistic Live initialization benchmark fixture.\n');
|
||||
fs.writeFileSync(path.join(tmp, 'DESIGN.md'), '# Design\n\nUse the fixture\'s existing type, color, and component system.\n');
|
||||
fs.mkdirSync(path.join(tmp, '.impeccable/live'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmp, '.impeccable/live/config.json'), JSON.stringify({
|
||||
files: ['index.html'],
|
||||
insertBefore: '</body>',
|
||||
commentSyntax: 'html',
|
||||
cspChecked: true,
|
||||
}, null, 2) + '\n');
|
||||
|
||||
const cold = [];
|
||||
for (let i = 0; i < iterations; i += 1) {
|
||||
stop();
|
||||
cold.push(runLive());
|
||||
}
|
||||
|
||||
stop();
|
||||
runLive();
|
||||
const warm = [];
|
||||
for (let i = 0; i < iterations; i += 1) warm.push(runLive());
|
||||
|
||||
console.log(JSON.stringify({
|
||||
fixture,
|
||||
iterations,
|
||||
measuredAt: new Date().toISOString(),
|
||||
cold: summarize(cold),
|
||||
warm: summarize(warm),
|
||||
samples: { cold, warm },
|
||||
}, null, 2));
|
||||
} finally {
|
||||
stop();
|
||||
fs.rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function runLive() {
|
||||
const start = performance.now();
|
||||
const stdout = execFileSync(process.execPath, [liveScript], {
|
||||
cwd: tmp,
|
||||
encoding: 'utf-8',
|
||||
timeout: 15_000,
|
||||
});
|
||||
const elapsed = performance.now() - start;
|
||||
const result = JSON.parse(stdout);
|
||||
if (!result.ok) throw new Error('live init failed: ' + stdout);
|
||||
return round(elapsed);
|
||||
}
|
||||
|
||||
function stop() {
|
||||
try {
|
||||
execFileSync(process.execPath, [serverScript, 'stop'], {
|
||||
cwd: tmp,
|
||||
stdio: 'ignore',
|
||||
timeout: 5_000,
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
|
||||
function summarize(samples) {
|
||||
const sorted = [...samples].sort((a, b) => a - b);
|
||||
return {
|
||||
medianMs: percentile(sorted, 0.5),
|
||||
p95Ms: percentile(sorted, 0.95),
|
||||
minMs: sorted[0],
|
||||
maxMs: sorted.at(-1),
|
||||
};
|
||||
}
|
||||
|
||||
function percentile(sorted, value) {
|
||||
if (sorted.length === 1) return sorted[0];
|
||||
const index = (sorted.length - 1) * value;
|
||||
const lower = Math.floor(index);
|
||||
const upper = Math.ceil(index);
|
||||
const weight = index - lower;
|
||||
return round(sorted[lower] * (1 - weight) + sorted[upper] * weight);
|
||||
}
|
||||
|
||||
function round(value) {
|
||||
return Math.round(value * 100) / 100;
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
import { parseArgs } from './lib/cli-args.mjs';
|
||||
import { compareModelBackedReports } from './lib/live-benchmark.mjs';
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (!args.atomic || !args.progressive) {
|
||||
throw new Error('usage: node scripts/compare-live-benchmarks.mjs --atomic=<report.json> --progressive=<report.json>');
|
||||
}
|
||||
|
||||
const [atomic, progressive] = await Promise.all([
|
||||
readReport(args.atomic, 'atomic'),
|
||||
readReport(args.progressive, 'progressive'),
|
||||
]);
|
||||
const comparison = compareModelBackedReports(atomic, progressive, {
|
||||
medianTarget: ratioArg(args.medianTarget, 0.35),
|
||||
p95Target: ratioArg(args.p95Target, 0.25),
|
||||
});
|
||||
|
||||
process.stdout.write(JSON.stringify(comparison, null, 2) + '\n');
|
||||
if (!comparison.passed) process.exitCode = 1;
|
||||
|
||||
async function readReport(file, delivery) {
|
||||
const value = JSON.parse(await readFile(resolve(String(file)), 'utf-8'));
|
||||
const reports = Array.isArray(value?.reports) ? value.reports : [value];
|
||||
const report = reports.find((item) => item?.benchmark?.delivery === delivery);
|
||||
if (!report) throw new Error(`${file} does not contain a ${delivery} benchmark report`);
|
||||
return report;
|
||||
}
|
||||
|
||||
function ratioArg(value, fallback) {
|
||||
if (value == null || value === true) return fallback;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed) || parsed < 0 || parsed >= 1) throw new Error(`invalid threshold ratio: ${value}`);
|
||||
return parsed;
|
||||
}
|
||||
@@ -22,10 +22,7 @@ export const PROVIDERS = {
|
||||
},
|
||||
'claude-code': {
|
||||
provider: 'claude-code',
|
||||
// live-progressive: Live delivers variant 1 as soon as it validates instead of
|
||||
// one atomic edit. Claude Code polls in a background task, so the extra
|
||||
// publish calls do not stall its control lane.
|
||||
providerTags: ['claude-code', 'claude', 'live-progressive'],
|
||||
providerTags: ['claude-code', 'claude'],
|
||||
configDir: '.claude',
|
||||
displayName: 'Claude Code',
|
||||
frontmatterFields: ['user-invocable', 'argument-hint', 'license', 'compatibility', 'metadata', 'allowed-tools'],
|
||||
@@ -43,7 +40,7 @@ export const PROVIDERS = {
|
||||
},
|
||||
codex: {
|
||||
provider: 'codex',
|
||||
providerTags: ['codex', 'live-progressive'],
|
||||
providerTags: ['codex'],
|
||||
configDir: '.codex',
|
||||
displayName: 'Codex',
|
||||
frontmatterFields: [],
|
||||
@@ -57,7 +54,7 @@ export const PROVIDERS = {
|
||||
},
|
||||
agents: {
|
||||
provider: 'agents',
|
||||
providerTags: ['agents', 'codex', 'live-progressive'],
|
||||
providerTags: ['agents', 'codex'],
|
||||
configDir: '.agents',
|
||||
displayName: 'Codex Repo Skills',
|
||||
placeholderProvider: 'codex',
|
||||
|
||||
@@ -645,11 +645,6 @@ export const PROVIDER_BLOCK_TAGS = new Set([
|
||||
'rovo-dev',
|
||||
'trae',
|
||||
'trae-cn',
|
||||
// Capability tags. Not harness names: they mark instructions that belong to a
|
||||
// shared capability several harnesses opt into. Listing the harnesses instead
|
||||
// would mean duplicating the block body per provider tag, since a block takes
|
||||
// one tag. Opt a provider in by adding the tag to its providerTags.
|
||||
'live-progressive',
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -137,7 +137,6 @@ export const SUITES = {
|
||||
'tests/live-e2e/agent-insert.test.mjs',
|
||||
'tests/live-event-validation.test.mjs',
|
||||
'tests/live-generation-preflight.test.mjs',
|
||||
'tests/live-generation-publisher.test.mjs',
|
||||
'tests/live-inject.test.mjs',
|
||||
'tests/live-insert.test.mjs',
|
||||
'tests/live-insert-ui.test.mjs',
|
||||
|
||||
Reference in New Issue
Block a user