From e8e3665142ed96e8afe6a1b3f7302971bc7386b6 Mon Sep 17 00:00:00 2001 From: Abdul Wahab <32850166+abdulwahabone@users.noreply.github.com> Date: Fri, 29 May 2026 11:02:12 +0900 Subject: [PATCH] Live mode: staged AI copy edits (#158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(live): manual text-edit panel + Astro inject + stale-lockfile reap Adds a manual text-edit popover under the live-mode bar so users can retype copy directly without going through generate. The footer's "Apply edits" button fires a manual_edits event; the server writes the changes back to source via the new live-edit.mjs deterministic file mutator. Mirrors the wrap+accept flow but skips variant generation. New scripts: - skill/scripts/live-edit.mjs: writes manual_edits back to source - skill/scripts/live-text-rows.js: browser walker that surfaces every pure-text descendant of the picked element as an editable row Touched scripts: - skill/scripts/live-browser.js: text panel UI, CONFIGURING state hook - skill/scripts/live-poll.mjs: manual_edits routing - skill/scripts/live-server.mjs: manual_edits endpoint + handler - skill/scripts/live-wrap.mjs: small adjustments to support the flow Docs + tests: - skill/reference/live.md: manual-edit section - tests/live-edit.test.mjs, tests/live-text-rows.test.mjs Also bundles two live-mode reliability fixes that surfaced during manual testing of the feature: 1. live-inject now emits is:inline when the inject target is a .astro file. Astro otherwise processes the \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.agents/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.agents/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.agents/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.agents/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.agents/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.agents/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.agents/skills/impeccable/scripts/live-poll.mjs b/.agents/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.agents/skills/impeccable/scripts/live-poll.mjs +++ b/.agents/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.agents/skills/impeccable/scripts/live-resume.mjs b/.agents/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.agents/skills/impeccable/scripts/live-resume.mjs +++ b/.agents/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.agents/skills/impeccable/scripts/live-server.mjs b/.agents/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.agents/skills/impeccable/scripts/live-server.mjs +++ b/.agents/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.claude/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.claude/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.claude/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.claude/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.claude/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.claude/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.claude/skills/impeccable/scripts/live-poll.mjs b/.claude/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.claude/skills/impeccable/scripts/live-poll.mjs +++ b/.claude/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.claude/skills/impeccable/scripts/live-resume.mjs b/.claude/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.claude/skills/impeccable/scripts/live-resume.mjs +++ b/.claude/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.claude/skills/impeccable/scripts/live-server.mjs b/.claude/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.claude/skills/impeccable/scripts/live-server.mjs +++ b/.claude/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.cursor/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.cursor/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.cursor/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.cursor/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.cursor/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.cursor/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.cursor/skills/impeccable/scripts/live-poll.mjs b/.cursor/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.cursor/skills/impeccable/scripts/live-poll.mjs +++ b/.cursor/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.cursor/skills/impeccable/scripts/live-resume.mjs b/.cursor/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.cursor/skills/impeccable/scripts/live-resume.mjs +++ b/.cursor/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.cursor/skills/impeccable/scripts/live-server.mjs b/.cursor/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.cursor/skills/impeccable/scripts/live-server.mjs +++ b/.cursor/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.gemini/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.gemini/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.gemini/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.gemini/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.gemini/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.gemini/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.gemini/skills/impeccable/scripts/live-poll.mjs b/.gemini/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.gemini/skills/impeccable/scripts/live-poll.mjs +++ b/.gemini/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.gemini/skills/impeccable/scripts/live-resume.mjs b/.gemini/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.gemini/skills/impeccable/scripts/live-resume.mjs +++ b/.gemini/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.gemini/skills/impeccable/scripts/live-server.mjs b/.gemini/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.gemini/skills/impeccable/scripts/live-server.mjs +++ b/.gemini/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.github/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.github/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.github/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.github/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.github/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.github/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.github/skills/impeccable/scripts/live-poll.mjs b/.github/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.github/skills/impeccable/scripts/live-poll.mjs +++ b/.github/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.github/skills/impeccable/scripts/live-resume.mjs b/.github/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.github/skills/impeccable/scripts/live-resume.mjs +++ b/.github/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.github/skills/impeccable/scripts/live-server.mjs b/.github/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.github/skills/impeccable/scripts/live-server.mjs +++ b/.github/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.kiro/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.kiro/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.kiro/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.kiro/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.kiro/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.kiro/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.kiro/skills/impeccable/scripts/live-poll.mjs b/.kiro/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.kiro/skills/impeccable/scripts/live-poll.mjs +++ b/.kiro/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.kiro/skills/impeccable/scripts/live-resume.mjs b/.kiro/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.kiro/skills/impeccable/scripts/live-resume.mjs +++ b/.kiro/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.kiro/skills/impeccable/scripts/live-server.mjs b/.kiro/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.kiro/skills/impeccable/scripts/live-server.mjs +++ b/.kiro/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.opencode/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.opencode/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.opencode/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.opencode/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.opencode/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.opencode/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.opencode/skills/impeccable/scripts/live-poll.mjs b/.opencode/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.opencode/skills/impeccable/scripts/live-poll.mjs +++ b/.opencode/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.opencode/skills/impeccable/scripts/live-resume.mjs b/.opencode/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.opencode/skills/impeccable/scripts/live-resume.mjs +++ b/.opencode/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.opencode/skills/impeccable/scripts/live-server.mjs b/.opencode/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.opencode/skills/impeccable/scripts/live-server.mjs +++ b/.opencode/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.pi/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.pi/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.pi/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.pi/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.pi/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.pi/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.pi/skills/impeccable/scripts/live-poll.mjs b/.pi/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.pi/skills/impeccable/scripts/live-poll.mjs +++ b/.pi/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.pi/skills/impeccable/scripts/live-resume.mjs b/.pi/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.pi/skills/impeccable/scripts/live-resume.mjs +++ b/.pi/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.pi/skills/impeccable/scripts/live-server.mjs b/.pi/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.pi/skills/impeccable/scripts/live-server.mjs +++ b/.pi/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.qoder/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.qoder/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.qoder/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.qoder/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.qoder/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.qoder/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.qoder/skills/impeccable/scripts/live-poll.mjs b/.qoder/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.qoder/skills/impeccable/scripts/live-poll.mjs +++ b/.qoder/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.qoder/skills/impeccable/scripts/live-resume.mjs b/.qoder/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.qoder/skills/impeccable/scripts/live-resume.mjs +++ b/.qoder/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.qoder/skills/impeccable/scripts/live-server.mjs b/.qoder/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.qoder/skills/impeccable/scripts/live-server.mjs +++ b/.qoder/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.rovodev/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.rovodev/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.rovodev/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.rovodev/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.rovodev/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.rovodev/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.rovodev/skills/impeccable/scripts/live-poll.mjs b/.rovodev/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.rovodev/skills/impeccable/scripts/live-poll.mjs +++ b/.rovodev/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.rovodev/skills/impeccable/scripts/live-resume.mjs b/.rovodev/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.rovodev/skills/impeccable/scripts/live-resume.mjs +++ b/.rovodev/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.rovodev/skills/impeccable/scripts/live-server.mjs b/.rovodev/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.rovodev/skills/impeccable/scripts/live-server.mjs +++ b/.rovodev/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.trae-cn/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.trae-cn/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.trae-cn/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.trae-cn/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.trae-cn/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.trae-cn/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.trae-cn/skills/impeccable/scripts/live-poll.mjs b/.trae-cn/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.trae-cn/skills/impeccable/scripts/live-poll.mjs +++ b/.trae-cn/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.trae-cn/skills/impeccable/scripts/live-resume.mjs b/.trae-cn/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.trae-cn/skills/impeccable/scripts/live-resume.mjs +++ b/.trae-cn/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.trae-cn/skills/impeccable/scripts/live-server.mjs b/.trae-cn/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.trae-cn/skills/impeccable/scripts/live-server.mjs +++ b/.trae-cn/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/.trae/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/.trae/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/.trae/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/.trae/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/.trae/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/.trae/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/.trae/skills/impeccable/scripts/live-poll.mjs b/.trae/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/.trae/skills/impeccable/scripts/live-poll.mjs +++ b/.trae/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/.trae/skills/impeccable/scripts/live-resume.mjs b/.trae/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/.trae/skills/impeccable/scripts/live-resume.mjs +++ b/.trae/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/.trae/skills/impeccable/scripts/live-server.mjs b/.trae/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/.trae/skills/impeccable/scripts/live-server.mjs +++ b/.trae/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/plugin/skills/impeccable/scripts/live-manual-edit-evidence.mjs b/plugin/skills/impeccable/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/plugin/skills/impeccable/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/plugin/skills/impeccable/scripts/live-manual-edits-buffer.mjs b/plugin/skills/impeccable/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/plugin/skills/impeccable/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/plugin/skills/impeccable/scripts/live-poll.mjs b/plugin/skills/impeccable/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/plugin/skills/impeccable/scripts/live-poll.mjs +++ b/plugin/skills/impeccable/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/plugin/skills/impeccable/scripts/live-resume.mjs b/plugin/skills/impeccable/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/plugin/skills/impeccable/scripts/live-resume.mjs +++ b/plugin/skills/impeccable/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/plugin/skills/impeccable/scripts/live-server.mjs b/plugin/skills/impeccable/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/plugin/skills/impeccable/scripts/live-server.mjs +++ b/plugin/skills/impeccable/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: \n' + + '\n' + open + ' ' + MARKER_CLOSE_TEXT + ' ' + close + '\n' ); } -function insertTag(content, config, port) { - const block = buildTagBlock(config.commentSyntax, port); +function insertTag(content, config, port, filePath) { + const block = buildTagBlock(config.commentSyntax, port, filePath); // insertBefore: match the LAST occurrence. Anchors like `` naturally // belong at the end, and the same literal can appear earlier in code blocks // within rendered documentation pages. @@ -299,12 +303,21 @@ function insertTag(content, config, port) { */ function removeTag(content, _syntax) { const patterns = [ - /([ \t]*)[\s\S]*?[ \t]*\n/, - /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}[ \t]*\n/, + /([ \t]*)[\s\S]*?([ \t]*(?:\n|$)?)/, + /([ \t]*)\{\/\*\s*impeccable-live-start\s*\*\/\}[\s\S]*?\{\/\*\s*impeccable-live-end\s*\*\/\}([ \t]*(?:\n|$)?)/, ]; for (const pat of patterns) { - const next = content.replace(pat, '$1'); - if (next !== content) return next; + let changed = false; + let next = content; + do { + content = next; + next = content.replace(pat, (_match, leadingIndent, trailing = '') => { + if (trailing.includes('\n')) return leadingIndent; + return leadingIndent || trailing || ''; + }); + if (next !== content) changed = true; + } while (next !== content); + if (changed) return next; } return content; } diff --git a/skill/scripts/live-manual-edit-evidence.mjs b/skill/scripts/live-manual-edit-evidence.mjs new file mode 100644 index 000000000..860278b73 --- /dev/null +++ b/skill/scripts/live-manual-edit-evidence.mjs @@ -0,0 +1,363 @@ +#!/usr/bin/env node +/** + * Collect evidence for pending live copy edits. + * + * This module intentionally does not edit source files and does not choose a + * winner. It gathers staged browser edits, rendered context, framework source + * hints, and likely source candidates so the AI copy-edit batch runner can make + * source changes with full repo context. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isGeneratedFile } from './is-generated.mjs'; +import { readBuffer, getBufferPath } from './live-manual-edits-buffer.mjs'; + +const EVIDENCE_VERSION = 1; +const TEXT_EXTENSIONS = new Set(['.html', '.jsx', '.tsx', '.vue', '.svelte', '.astro', '.js', '.mjs', '.ts']); +const SEARCH_DIRS = ['src', 'app', 'pages', 'components', 'public', 'views', 'templates', 'site', 'lib', 'data']; +const STRONG_LITERAL_MATCH_LIMIT = 8; +const WEAK_LITERAL_MATCH_LIMIT = 4; +const OBJECT_KEY_MATCH_LIMIT = 8; +const LOCATOR_MATCH_LIMIT = 4; +const CONTEXT_MATCH_LIMIT = 8; +const CONTEXT_MATCH_PER_HINT = 2; +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + '.impeccable', + '.astro', + '.next', + '.nuxt', + '.svelte-kit', + 'dist', + 'build', + 'out', + 'coverage', +]); + +export function buildManualEditEvidence({ cwd = process.cwd(), pageUrl = null } = {}) { + const buffer = readBuffer(cwd); + const entries = pageUrl + ? buffer.entries.filter((entry) => entry.pageUrl === pageUrl) + : buffer.entries; + const opCount = countOps(entries); + + if (opCount === 0) { + return { + pageUrl, + count: 0, + entries: [], + ops: [], + candidates: [], + }; + } + + const searchFiles = collectSearchFiles(cwd); + const ops = flattenOps(entries); + const candidates = ops.map((op) => buildCandidatesForOp(op, cwd, searchFiles)); + return { + version: EVIDENCE_VERSION, + pageUrl: pageUrl || null, + count: opCount, + entries, + ops, + context: { + cwd, + bufferPath: path.relative(cwd, getBufferPath(cwd)), + totalEntries: entries.length, + totalOps: opCount, + }, + candidates, + }; +} + +function countOps(entries) { + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function flattenOps(entries) { + const out = []; + for (const entry of entries) { + const contextHintsByRef = buildContextHintsByRef(entry); + for (const op of entry.ops || []) { + out.push({ + entryId: entry.id, + pageUrl: entry.pageUrl, + ref: op.ref, + contextRef: op.contextRef || null, + tag: op.tag, + elementId: op.elementId || null, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true, + sourceHint: op.sourceHint || null, + leaf: op.leaf || null, + nearbyEditableTexts: Array.isArray(op.nearbyEditableTexts) ? op.nearbyEditableTexts : [], + container: op.container || null, + contextHints: contextHintsByRef.get(op.ref) || [], + }); + } + } + return out; +} + +function buildContextHintsByRef(entry) { + const map = new Map(); + for (const op of entry.ops || []) { + const hints = new Set(); + const add = (value) => { + const text = normalizeText(decodeBasicHtml(String(value || ''))); + if (text.length < 3 || text.length > 160) return; + if (text === normalizeText(op.originalText) || text === normalizeText(op.newText)) return; + hints.add(text); + }; + + for (const item of op.nearbyEditableTexts || []) { + add(typeof item === 'string' ? item : item?.text); + } + const outer = typeof entry.element?.outerHTML === 'string' ? entry.element.outerHTML : ''; + for (const match of outer.matchAll(/data-impeccable-original-text="([^"]*)"/g)) add(match[1]); + if (typeof entry.element?.textContent === 'string') { + for (const chunk of entry.element.textContent.split(/\s{2,}|\n|\t/)) add(chunk); + } + map.set(op.ref, [...hints].slice(0, 16)); + } + return map; +} + +function buildCandidatesForOp(op, cwd, searchFiles) { + const originalText = String(op.originalText || ''); + const contextNeedles = op.contextHints || []; + return { + entryId: op.entryId, + ref: op.ref, + originalText, + sourceHint: analyzeSourceHint(op, cwd), + textMatches: originalText ? findLiteralMatches(searchFiles, originalText, { max: literalMatchLimit(originalText) }) : [], + objectKeyMatches: originalText ? findObjectKeyMatches(searchFiles, originalText, { max: OBJECT_KEY_MATCH_LIMIT }) : [], + locatorMatches: findLocatorMatches(searchFiles, op, { max: LOCATOR_MATCH_LIMIT }), + contextTextMatches: findContextMatches(searchFiles, contextNeedles, { maxPerHint: CONTEXT_MATCH_PER_HINT, max: CONTEXT_MATCH_LIMIT }), + }; +} + +function literalMatchLimit(text) { + return isWeakSourceNeedle(text) ? WEAK_LITERAL_MATCH_LIMIT : STRONG_LITERAL_MATCH_LIMIT; +} + +function isWeakSourceNeedle(text) { + const normalized = normalizeText(text); + return normalized.length < 4 || /^[\d.,+\-%\s]+$/.test(normalized); +} + +function analyzeSourceHint(op, cwd) { + const hint = normalizeSourceHint(op.sourceHint); + if (!hint.file) return null; + const file = path.resolve(cwd, hint.file); + const relativeFile = path.relative(cwd, file); + if (!isPathInsideOrEqual(cwd, file)) { + return { ...hint, status: 'outside_cwd', relativeFile: hint.file }; + } + if (!fs.existsSync(file)) { + return { ...hint, status: 'file_missing', relativeFile }; + } + if (isGeneratedFile(file, { cwd })) { + return { ...hint, status: 'generated', relativeFile }; + } + + const content = fs.readFileSync(file, 'utf-8'); + const lines = content.split('\n'); + const line = hint.line || 1; + const start = Math.max(0, line - 4); + const end = Math.min(lines.length, line + 3); + const windowText = lines.slice(start, end).join('\n'); + const containsOriginalText = typeof op.originalText === 'string' && windowText.includes(op.originalText); + return { + ...hint, + status: containsOriginalText ? 'ok' : 'text_not_found_near_hint', + relativeFile, + excerpt: lines.slice(start, end).map((text, index) => ({ + line: start + index + 1, + text: text.slice(0, 240), + })), + }; +} + +function normalizeSourceHint(hint) { + if (!hint || typeof hint !== 'object') return {}; + let line = Number.isFinite(Number(hint.line)) ? Number(hint.line) : null; + let column = Number.isFinite(Number(hint.column)) ? Number(hint.column) : null; + if ((!line || !column) && typeof hint.loc === 'string') { + const match = hint.loc.match(/^(\d+)(?::(\d+))?/); + if (match) { + line = Number(match[1]); + if (match[2]) column = Number(match[2]); + } + } + return { + file: typeof hint.file === 'string' ? hint.file : '', + loc: typeof hint.loc === 'string' ? hint.loc : '', + line, + column, + }; +} + +function collectSearchFiles(cwd) { + const out = []; + const seenDirs = new Set(); + const seenFiles = new Set(); + for (const dir of SEARCH_DIRS) { + scanDir(path.join(cwd, dir), cwd, seenDirs, seenFiles, out, 0); + } + scanRootFiles(cwd, seenFiles, out); + return out; +} + +function scanDir(dir, cwd, seenDirs, seenFiles, out, depth) { + if (depth > 7 || !fs.existsSync(dir)) return; + let realDir; + try { realDir = fs.realpathSync(dir); } catch { return; } + if (seenDirs.has(realDir)) return; + seenDirs.add(realDir); + + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + scanDir(fullPath, cwd, seenDirs, seenFiles, out, depth + 1); + continue; + } + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(fullPath, cwd, seenFiles, out); + } +} + +function scanRootFiles(cwd, seenFiles, out) { + let entries; + try { entries = fs.readdirSync(cwd, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isFile() || !TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) continue; + maybeAddSearchFile(path.join(cwd, entry.name), cwd, seenFiles, out); + } +} + +function maybeAddSearchFile(file, cwd, seenFiles, out) { + let realFile; + try { realFile = fs.realpathSync(file); } catch { return; } + if (seenFiles.has(realFile)) return; + seenFiles.add(realFile); + if (isGeneratedFile(file, { cwd })) return; + let content; + try { content = fs.readFileSync(file, 'utf-8'); } catch { return; } + out.push({ file, relativeFile: path.relative(cwd, file), content, lines: content.split('\n') }); +} + +function findLiteralMatches(searchFiles, needle, { max }) { + return findMatches(searchFiles, needle, { kind: 'text', max }); +} + +function findObjectKeyMatches(searchFiles, text, { max }) { + const re = new RegExp('(["\\\'`])' + escapeRegExp(text) + '\\1(?=\\s*:)', 'g'); + const out = []; + for (const file of searchFiles) { + for (const match of file.content.matchAll(re)) { + out.push(matchForIndex(file, match.index, 'object_key', text)); + if (out.length >= max) return out; + } + } + return out; +} + +function findLocatorMatches(searchFiles, op, { max }) { + const needles = []; + if (op.elementId) needles.push({ kind: 'id', needle: op.elementId }); + for (const cls of op.classes || []) { + if (cls) needles.push({ kind: 'class', needle: cls }); + } + if (op.tag) needles.push({ kind: 'tag', needle: '<' + op.tag }); + + const out = []; + const seen = new Set(); + for (const { kind, needle } of needles) { + for (const match of findMatches(searchFiles, needle, { kind, max })) { + const key = match.file + ':' + match.line + ':' + kind + ':' + needle; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle }); + if (out.length >= max) return out; + } + } + return out; +} + +function findContextMatches(searchFiles, hints, { maxPerHint, max }) { + const out = []; + const seen = new Set(); + for (const hint of hints || []) { + for (const match of findMatches(searchFiles, hint, { kind: 'context', max: maxPerHint })) { + const key = match.file + ':' + match.line + ':' + hint; + if (seen.has(key)) continue; + seen.add(key); + out.push({ ...match, needle: hint }); + if (out.length >= max) return out; + } + } + return out; +} + +function findMatches(searchFiles, needle, { kind, max }) { + const text = String(needle || ''); + if (!text) return []; + const out = []; + for (const file of searchFiles) { + let index = 0; + while (out.length < max) { + index = file.content.indexOf(text, index); + if (index === -1) break; + out.push(matchForIndex(file, index, kind, text)); + index += Math.max(1, text.length); + } + if (out.length >= max) break; + } + return out; +} + +function matchForIndex(file, index, kind, needle) { + const line = file.content.slice(0, index).split('\n').length; + const lineText = file.lines[line - 1] || ''; + return { + kind, + file: file.relativeFile, + line, + needle, + excerpt: lineText.trim().slice(0, 240), + }; +} + +function isPathInsideOrEqual(cwd, file) { + const rel = path.relative(path.resolve(cwd), path.resolve(file)); + return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function normalizeText(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function decodeBasicHtml(value) { + return value + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/skill/scripts/live-manual-edits-buffer.mjs b/skill/scripts/live-manual-edits-buffer.mjs new file mode 100644 index 000000000..9e3dcf455 --- /dev/null +++ b/skill/scripts/live-manual-edits-buffer.mjs @@ -0,0 +1,152 @@ +/** + * Shared helpers for the pending-manual-edits buffer on disk. + * + * Location: .impeccable/live/pending-manual-edits.json (project-local). + * Schema: { version: 1, entries: [{ id, pageUrl, element, ops, stagedAt }] } + * + * Each entry corresponds to one Save action from the browser. Ops merge by + * (pageUrl, ref): if the user re-edits the same element before committing, the + * existing entry's `newText` is replaced and `originalText` is kept (it holds + * the real source state). + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { getLiveDir } from './impeccable-paths.mjs'; + +const BUFFER_VERSION = 1; +const BUFFER_FILENAME = 'pending-manual-edits.json'; + +export function getBufferPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), BUFFER_FILENAME); +} + +export function readBuffer(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: false }); +} + +export function readBufferStrict(cwd = process.cwd()) { + return readBufferInternal(cwd, { strict: true }); +} + +function readBufferInternal(cwd, { strict }) { + const filePath = getBufferPath(cwd); + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.entries)) { + if (strict) throw new Error('manual_edit_buffer_invalid_schema'); + return { version: BUFFER_VERSION, entries: [] }; + } + return { version: BUFFER_VERSION, entries: parsed.entries }; + } catch (err) { + if (strict && err?.code !== 'ENOENT') { + throw new Error('manual_edit_buffer_unreadable: ' + (err.message || String(err))); + } + return { version: BUFFER_VERSION, entries: [] }; + } +} + +export function writeBuffer(cwd, buffer) { + const filePath = getBufferPath(cwd); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ version: BUFFER_VERSION, entries: buffer.entries }, null, 2)); +} + +/** + * Merge a new entry into the buffer. For each op in the new entry, if there's + * already a buffered op for the same (pageUrl, ref), update that op's newText + * and keep its original originalText (the true source state). Otherwise add + * the op (creating an entry if needed). + * + * Multiple ops in one Save are allowed; each is keyed by (pageUrl, ref). + */ +export function stageEntry(cwd, newEntry) { + const buf = readBufferStrict(cwd); + const pageUrl = newEntry.pageUrl; + for (const newOp of newEntry.ops) { + let mergedIntoExisting = false; + for (const existing of buf.entries) { + if (existing.pageUrl !== pageUrl) continue; + const existingOpIdx = existing.ops.findIndex((op) => op.ref === newOp.ref); + if (existingOpIdx >= 0) { + // Keep the original source text but refresh the latest DOM/source evidence. + existing.ops[existingOpIdx] = { + ...newOp, + originalText: existing.ops[existingOpIdx].originalText, + newText: newOp.newText, + deleted: newOp.deleted || false, + }; + if (newEntry.element) existing.element = newEntry.element; + existing.stagedAt = new Date().toISOString(); + mergedIntoExisting = true; + break; + } + } + if (mergedIntoExisting) continue; + // No existing op for this (pageUrl, ref). Find or create an entry to hold it. + let entry = buf.entries.find((e) => e.pageUrl === pageUrl && e.id === newEntry.id); + if (!entry) { + entry = { + id: newEntry.id, + pageUrl, + element: newEntry.element, + ops: [], + stagedAt: new Date().toISOString(), + }; + buf.entries.push(entry); + } + entry.ops.push(newOp); + entry.stagedAt = new Date().toISOString(); + } + writeBuffer(cwd, buf); + return buf; +} + +/** + * Remove entries matching a predicate. Returns count of removed *ops* (not + * entries) so callers report a unit consistent with truncateBuffer and the + * pill's per-page op count. Empty entries (no ops left) are also pruned. + */ +export function removeEntries(cwd, predicate) { + const buf = readBuffer(cwd); + let removedOps = 0; + const kept = []; + for (const entry of buf.entries) { + if (predicate(entry)) { + removedOps += entry.ops?.length || 0; + } else if (entry.ops && entry.ops.length > 0) { + kept.push(entry); + } + } + buf.entries = kept; + writeBuffer(cwd, buf); + return removedOps; +} + +/** + * Count by page for the counter UI. Returns { totalCount, perPage: {[pageUrl]: count} }. + */ +export function countByPage(cwd = process.cwd()) { + const buf = readBuffer(cwd); + const perPage = {}; + let totalCount = 0; + for (const entry of buf.entries) { + const n = entry.ops.length; + perPage[entry.pageUrl] = (perPage[entry.pageUrl] || 0) + n; + totalCount += n; + } + return { totalCount, perPage }; +} + +/** + * Truncate the buffer to empty (used by discard-all). Returns the count of + * removed ops. + */ +export function truncateBuffer(cwd) { + const buf = readBuffer(cwd); + let removed = 0; + for (const entry of buf.entries) removed += entry.ops.length; + writeBuffer(cwd, { version: BUFFER_VERSION, entries: [] }); + return removed; +} diff --git a/skill/scripts/live-poll.mjs b/skill/scripts/live-poll.mjs index cbf17d54f..fad836612 100644 --- a/skill/scripts/live-poll.mjs +++ b/skill/scripts/live-poll.mjs @@ -21,7 +21,7 @@ import { readLiveServerInfo } from './impeccable-paths.mjs'; // depending on the standalone undici package. export const PER_REQUEST_TIMEOUT_MS = 270_000; -const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer']); +const EVENT_TYPES_NEEDING_AGENT_REPLY = new Set(['generate', 'steer', 'manual_edit_apply']); function readServerInfo() { const record = readLiveServerInfo(process.cwd()); @@ -36,6 +36,69 @@ export function buildPollReplyPayload(token, { id, type, message, file, data }) return { token, id, type, message, file, data }; } +export function manualApplyPollBanner(event = {}) { + const id = event.id || 'EVENT_ID'; + return [ + `Manual Apply action required: edit source, then reply with \`live-poll.mjs --reply ${id} done --data ''\`.`, + 'The JSON data must include status, appliedEntryIds, failed, files, and notes; summary counters are only a recovery fallback.', + 'Do not run live-commit-manual-edits.mjs for this leased event.', + 'Do not poll again before replying.', + ].join('\n') + '\n'; +} + +/** + * Parse `--reply [--file path] [--data ''] [message]` argv + * into a reply object. Returns null when `--reply` is absent. Throws (code + * INVALID_REPLY_ARGS) when the reply shape is missing its event id/status and + * INVALID_DATA_JSON when `--data` is present but not valid JSON. + */ +export function parseReplyArgs(args) { + const replyIdx = args.indexOf('--reply'); + if (replyIdx === -1) return null; + const id = args[replyIdx + 1]; + const status = args[replyIdx + 2]; + validateReplyArgs({ id, status }); + const fileIdx = args.indexOf('--file'); + const file = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; + const dataIdx = args.indexOf('--data'); + let data; + if (dataIdx !== -1 && dataIdx + 1 < args.length) { + try { + data = JSON.parse(args[dataIdx + 1]); + } catch (err) { + const wrapped = new Error('--data must be valid JSON: ' + err.message); + wrapped.code = 'INVALID_DATA_JSON'; + throw wrapped; + } + } + const message = args.find((a, i) => + i > replyIdx + 2 + && !a.startsWith('--') + && i !== fileIdx + 1 + && i !== dataIdx + 1 + ) || undefined; + return { id, type: status, message, file, data }; +} + +function validateReplyArgs({ id, status }) { + const usage = "Usage: npx impeccable poll --reply [--file path] [--data ''] [message]"; + if (!id || id.startsWith('--')) { + const err = new Error(`${usage}\nMissing event id after --reply.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (['done', 'error', 'complete', 'discard', 'discarded'].includes(id)) { + const err = new Error(`${usage}\nThe value after --reply must be the event id, not the status ${JSON.stringify(id)}. Use --reply EVENT_ID ${id}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } + if (!status || status.startsWith('--')) { + const err = new Error(`${usage}\nMissing reply status after event id ${JSON.stringify(id)}.`); + err.code = 'INVALID_REPLY_ARGS'; + throw err; + } +} + export function requiresAgentReply(event) { return EVENT_TYPES_NEEDING_AGENT_REPLY.has(event?.type); } @@ -48,7 +111,8 @@ export async function postReply(base, token, reply) { }); if (!res.ok) { const body = await res.json().catch(() => ({})); - throw new Error(body.error || res.statusText); + const parts = [body.error || res.statusText, body.reason, body.hint].filter(Boolean); + throw new Error(parts.join(': ')); } } @@ -119,12 +183,7 @@ export async function augmentEventWithAcceptHandling(event, base, token) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const acceptScript = path.join(__dirname, 'live-accept.mjs'); - const scriptArgs = event.type === 'discard' - ? ['--id', event.id, '--discard'] - : ['--id', event.id, '--variant', event.variantId]; - if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { - scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); - } + const scriptArgs = buildAcceptScriptArgs(event); try { const out = execFileSync( @@ -156,7 +215,21 @@ export async function augmentEventWithAcceptHandling(event, base, token) { return event; } +export function buildAcceptScriptArgs(event) { + const scriptArgs = event.type === 'discard' + ? ['--id', String(event.id), '--discard'] + : ['--id', String(event.id), '--variant', String(event.variantId)]; + if (event.pageUrl) scriptArgs.push('--page-url', String(event.pageUrl)); + if (event.type === 'accept' && event.paramValues && Object.keys(event.paramValues).length > 0) { + scriptArgs.push('--param-values', JSON.stringify(event.paramValues)); + } + return scriptArgs; +} + export function writeCarbonizeBanner(event) { + if (event.type === 'manual_edit_apply') { + process.stderr.write('\n' + manualApplyPollBanner(event) + '\n'); + } if (event._acceptResult?.carbonize === true) { process.stderr.write('\n⚠ Carbonize cleanup REQUIRED before next poll. After cleanup, run live-complete.mjs --id ' + event.id + '. See reference/live.md "Required after accept".\n\n'); } @@ -238,10 +311,14 @@ Modes: poll --reply done Reply "done" to event (replace or insert generate) poll --reply steer_done Reply after handling a steer event (unlocks Steer bar) poll --reply error "msg" Reply with an error message + poll --reply done --data '' + Reply with a structured JSON result (manual_edit_apply) Options: --timeout=MS One-shot poll timeout in ms (default: 600000). Ignored in --stream mode --ack-timeout=MS Stream mode: max wait for --reply after generate/steer (default: 600000) + --file PATH Attach a source file path to the reply (generate flow) + --data JSON Attach a JSON result object to the reply (manual_edit_apply flow). Must be valid JSON --help Show this help message Harness note: @@ -253,22 +330,18 @@ Harness note: const info = readServerInfo(); const base = `http://localhost:${info.port}`; - // Reply mode: npx impeccable poll --reply [--file path] [message] - const replyIdx = args.indexOf('--reply'); - if (replyIdx !== -1) { - const id = args[replyIdx + 1]; - const status = args[replyIdx + 2] || 'done'; - const fileIdx = args.indexOf('--file'); - const filePath = fileIdx !== -1 && fileIdx + 1 < args.length ? args[fileIdx + 1] : undefined; - const message = args.find((a, i) => i > replyIdx + 2 && !a.startsWith('--') && i !== fileIdx + 1) || undefined; - - if (!id) { - console.error('Usage: npx impeccable poll --reply [--file path] [message]'); + // Reply mode: npx impeccable poll --reply [--file path] [--data ''] [message] + if (args.includes('--reply')) { + let reply; + try { + reply = parseReplyArgs(args); + } catch (err) { + console.error(err.message); process.exit(1); } try { - await postReply(base, info.token, { id, type: status, message, file: filePath }); + await postReply(base, info.token, reply); } catch (err) { if (err.cause?.code === 'ECONNREFUSED') { console.error('Live server not running. Start one with: npx impeccable live'); diff --git a/skill/scripts/live-resume.mjs b/skill/scripts/live-resume.mjs index a3465c9b5..e54831f12 100644 --- a/skill/scripts/live-resume.mjs +++ b/skill/scripts/live-resume.mjs @@ -5,6 +5,50 @@ import { createLiveSessionStore } from './live-session-store.mjs'; +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +export function manualApplyResumeHint(event = {}) { + const summary = event.manualApplySummary || summarizeManualApplyEvent(event); + const parts = []; + if (summary.pageUrl) parts.push(`page ${summary.pageUrl}`); + if (summary.chunk) parts.push(`chunk ${summary.chunk.index}/${summary.chunk.total}`); + if (Number.isFinite(summary.opCount)) parts.push(`${summary.opCount} op(s)`); + if (Number.isFinite(summary.entryCount)) parts.push(`${summary.entryCount} entr${summary.entryCount === 1 ? 'y' : 'ies'}`); + if (summary.files?.length) parts.push(`likely files: ${summary.files.join(', ')}`); + const scope = parts.length ? ` (${parts.join(', ')})` : ''; + return `Manual Apply pending${scope}. If you have not already leased it, run live-poll.mjs. Apply the source edits from the manual_edit_apply batch, then reply with ${manualApplyReplyCommand(event.id)}. Polling only leases this work item; it does not commit source edits. Do not run live-commit-manual-edits.mjs for this leased event. Do not poll again before replying.`; +} + +function summarizeManualApplyEvent(event = {}) { + const entries = Array.isArray(event.batch?.entries) ? event.batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(event.batch), + }; +} + +function collectManualApplyFiles(batch) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + return [...new Set(files.filter((file) => typeof file === 'string' && file.length > 0))].sort(); +} + function parseArgs(argv) { const out = { id: null }; for (let i = 0; i < argv.length; i++) { @@ -32,7 +76,9 @@ export async function resumeCli() { const pending = snapshot.pendingEvent || null; const nextAction = pending - ? `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` + ? pending.type === 'manual_edit_apply' + ? manualApplyResumeHint(pending) + : `Run live-poll.mjs, handle ${pending.type} ${pending.id}, then acknowledge with live-poll.mjs --reply ${pending.id} done.` : snapshot.phase === 'carbonize_required' ? `Finish carbonize cleanup${snapshot.sourceFile ? ` in ${snapshot.sourceFile}` : ''}, then run live-complete.mjs --id ${snapshot.id}.` : snapshot.phase === 'accept_requested' diff --git a/skill/scripts/live-server.mjs b/skill/scripts/live-server.mjs index 574b574ef..16c8285b9 100644 --- a/skill/scripts/live-server.mjs +++ b/skill/scripts/live-server.mjs @@ -26,12 +26,22 @@ import { createLiveSessionStore } from './live-session-store.mjs'; import { validateEvent } from './live-event-validation.mjs'; import { getDesignSidecarPath, + getLiveDir, getLiveAnnotationsDir, readLiveServerInfo, removeLiveServerInfo, resolveDesignSidecarPath, writeLiveServerInfo, } from './impeccable-paths.mjs'; +import { + countByPage as countPendingByPage, + readBuffer as readManualEditsBuffer, + removeEntries as removeManualEditEntries, + stageEntry as stageManualEditEntry, + truncateBuffer as truncateManualEditsBuffer, +} from './live-manual-edits-buffer.mjs'; +import { buildManualEditEvidence } from './live-manual-edit-evidence.mjs'; +import { commitManualEdits } from './live-commit-manual-edits.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // PRODUCT.md / DESIGN.md live wherever context.mjs resolves. The generated @@ -66,20 +76,802 @@ const state = { sseClients: new Set(), // SSE response objects (server→browser push) pendingEvents: [], // browser events waiting for agent ack ({ event, leaseUntil }) pendingPolls: [], // agent poll callbacks waiting for browser events + nextEventSeq: 1, lastAgentPollingBroadcast: null, exitTimer: null, sessionDir: null, // per-session tmp dir for annotation screenshots sessionStore: null, leaseTimer: null, + manualEditActivity: null, + nextManualEditSeq: 1, + // Deferreds for in-flight chat-routed Apply events. Keyed by event id; each + // entry is resolved when the chat agent POSTs an ack carrying the batch + // result, or rejected when the hard timeout fires. + pendingApplyDeferreds: new Map(), + // Updated whenever a /poll long-poll request arrives or is resolved with an + // event. Used to detect "a chat agent is likely attached" without requiring + // a poll to be parked at the exact moment we dispatch. + lastPollAt: 0, + timedOutApplyIds: new Map(), }; +const CHAT_POLL_FRESHNESS_MS = 60_000; +const APPLY_EVENT_HARD_TIMEOUT_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_HARD_TIMEOUT_MS || 150_000); +const APPLY_EVENT_SOFT_DEADLINE_MS = Number(process.env.IMPECCABLE_LIVE_APPLY_EVENT_SOFT_DEADLINE_MS || 120_000); +const DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE = 3; +const MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE = 1; +const MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE = 20; +const MANUAL_APPLY_COMPACT_TEXT_LIMIT = 240; +const MANUAL_APPLY_COMPACT_NEARBY_LIMIT = 4; +const DEBUG_MANUAL_EDIT_EVENTS = /^(1|true|yes)$/i.test(process.env.IMPECCABLE_LIVE_DEBUG_EVENTS || ''); + +function tombstoneTimedOutApplyId(eventId, details = {}) { + if (!eventId) return; + state.timedOutApplyIds.set(eventId, details); + if (state.timedOutApplyIds.size <= 200) return; + const oldest = state.timedOutApplyIds.keys().next().value; + state.timedOutApplyIds.delete(oldest); +} + +function chatAgentLikelyActive() { + if (state.pendingPolls.length > 0) return true; + if (!state.lastPollAt) return false; + return Date.now() - state.lastPollAt < CHAT_POLL_FRESHNESS_MS; +} + +function manualEditApplyChunkSize(env = process.env) { + const raw = Number(env.IMPECCABLE_LIVE_MANUAL_EDIT_CHUNK_SIZE); + if (!Number.isFinite(raw)) return DEFAULT_MANUAL_EDIT_APPLY_CHUNK_SIZE; + const size = Math.trunc(raw); + return Math.max(MIN_MANUAL_EDIT_APPLY_CHUNK_SIZE, Math.min(MAX_MANUAL_EDIT_APPLY_CHUNK_SIZE, size)); +} + +function countManualApplyOps(entriesOrBatch) { + const entries = Array.isArray(entriesOrBatch) + ? entriesOrBatch + : Array.isArray(entriesOrBatch?.entries) ? entriesOrBatch.entries : []; + let count = 0; + for (const entry of entries) count += Array.isArray(entry.ops) ? entry.ops.length : 0; + return count; +} + +function pushApplyEventAndWait(batch, pageUrl, chunk = null, repair = null) { + const eventId = randomUUID().replace(/-/g, '').slice(0, 8); + const evidencePath = writeManualApplyEvidence(eventId, batch); + const event = { + type: 'manual_edit_apply', + id: eventId, + pageUrl, + batch: compactManualApplyBatch(batch), + evidencePath, + agentAction: buildManualApplyAgentAction(eventId), + schemaVersion: 1, + deadlineMs: APPLY_EVENT_SOFT_DEADLINE_MS, + }; + if (chunk) event.chunk = chunk; + if (repair) event.repair = repair; + const rollbackSnapshot = snapshotApplyEventFiles(batch); + recordManualEditActivity('manual_edit_apply_dispatched', { + id: eventId, + pageUrl, + chunk, + repair, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + fileCount: collectManualApplyFiles(batch).length, + }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pendingApplyDeferreds.delete(eventId); + tombstoneTimedOutApplyId(eventId, { batch, rollbackSnapshot }); + acknowledgePendingEvent(eventId); + removeManualApplyEvidence(evidencePath); + recordManualEditActivity('manual_edit_apply_timeout', { + id: eventId, + pageUrl, + chunk, + entryCount: Array.isArray(batch.entries) ? batch.entries.length : 0, + opCount: countManualApplyOps(batch), + }); + reject(new Error('chat_agent_timeout')); + }, APPLY_EVENT_HARD_TIMEOUT_MS); + state.pendingApplyDeferreds.set(eventId, { resolve, reject, timer, event, batch, pageUrl, rollbackSnapshot }); + enqueueEvent(event); + }); +} + +function writeManualApplyEvidence(eventId, batch) { + const dir = manualApplyEvidenceDir(process.cwd()); + fs.mkdirSync(dir, { recursive: true }); + const evidencePath = path.join(dir, `${eventId}.json`); + fs.writeFileSync(evidencePath, JSON.stringify(batch, null, 2) + '\n', 'utf-8'); + return evidencePath; +} + +function manualApplyEvidenceDir(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-evidence'); +} + +function normalizeManualApplyEvidencePath(evidencePath, cwd = process.cwd()) { + if (!evidencePath || typeof evidencePath !== 'string') return null; + const fullPath = path.isAbsolute(evidencePath) ? evidencePath : path.resolve(cwd, evidencePath); + const evidenceDir = manualApplyEvidenceDir(cwd); + const relative = path.relative(evidenceDir, fullPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + if (path.extname(relative) !== '.json') return null; + return fullPath; +} + +function removeManualApplyEvidence(evidencePath, cwd = process.cwd()) { + const fullPath = normalizeManualApplyEvidencePath(evidencePath, cwd); + if (!fullPath) return false; + try { + fs.unlinkSync(fullPath); + return true; + } catch { + return false; + } +} + +function referencedManualApplyEvidencePaths(cwd = process.cwd()) { + const referenced = new Set(); + const add = (event) => { + const fullPath = normalizeManualApplyEvidencePath(event?.evidencePath, cwd); + if (fullPath) referenced.add(fullPath); + }; + for (const entry of state.pendingEvents) add(entry.event); + for (const deferred of state.pendingApplyDeferreds.values()) add(deferred.event); + return referenced; +} + +function pruneStaleManualApplyEvidence(cwd = process.cwd()) { + const dir = manualApplyEvidenceDir(cwd); + if (!fs.existsSync(dir)) return []; + const referenced = referencedManualApplyEvidencePaths(cwd); + const removed = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + const fullPath = path.join(dir, name); + if (referenced.has(fullPath)) continue; + try { + fs.unlinkSync(fullPath); + removed.push(fullPath); + } catch { + // Stale evidence cleanup is best-effort; Apply verification never relies + // on deleting these files. + } + } + return removed; +} + +function compactManualApplyBatch(batch = {}) { + const entries = (batch.entries || []).map(compactManualApplyEntry); + const candidates = compactManualApplyCandidates(batch.candidates || []); + return { + version: batch.version, + pageUrl: batch.pageUrl || null, + count: batch.count, + entries, + ops: entries.flatMap((entry) => entry.ops.map((op) => ({ ...op, entryId: entry.id }))), + candidates: candidates.length > 0 ? candidates : undefined, + context: batch.context ? { + bufferPath: batch.context.bufferPath, + totalEntries: batch.context.totalEntries, + totalOps: batch.context.totalOps, + chunkIndex: batch.context.chunkIndex, + chunkTotal: batch.context.chunkTotal, + totalApplyOps: batch.context.totalApplyOps, + } : undefined, + }; +} + +function compactManualApplyCandidates(candidates) { + return (Array.isArray(candidates) ? candidates : []) + .slice(0, 24) + .map((candidate) => ({ + entryId: candidate.entryId, + ref: candidate.ref, + sourceHint: compactManualApplySourceMatch(candidate.sourceHint), + textMatches: compactManualApplySourceMatches(candidate.textMatches, 8), + objectKeyMatches: compactManualApplySourceMatches(candidate.objectKeyMatches, 8), + contextTextMatches: compactManualApplySourceMatches(candidate.contextTextMatches, 8), + locatorMatches: compactManualApplySourceMatches(candidate.locatorMatches, 6), + })); +} + +function compactManualApplySourceMatches(matches, limit) { + return (Array.isArray(matches) ? matches : []) + .slice(0, limit) + .map(compactManualApplySourceMatch) + .filter(Boolean); +} + +function compactManualApplySourceMatch(match) { + if (!match || typeof match !== 'object') return null; + const file = match.relativeFile || match.file; + if (!file && !match.line) return null; + return { + file: summarizeManualLogFile(file), + line: match.line || null, + column: match.column || null, + reason: match.reason || match.kind || undefined, + status: match.status || undefined, + }; +} + +function compactManualApplyEntry(entry = {}) { + return { + id: entry.id, + pageUrl: entry.pageUrl, + stagedAt: entry.stagedAt || null, + element: compactManualApplyContext(entry.element), + ops: (entry.ops || []).map(compactManualApplyOp), + }; +} + +function compactManualApplyOp(op = {}) { + return { + entryId: op.entryId, + ref: op.ref, + contextRef: op.contextRef, + tag: op.tag, + elementId: op.elementId, + classes: Array.isArray(op.classes) ? op.classes : [], + originalText: op.originalText, + newText: op.newText, + deleted: op.deleted === true || undefined, + sourceHint: op.sourceHint || null, + leaf: compactManualApplyContext(op.leaf), + nearbyEditableTexts: compactNearbyManualEditTexts(op.nearbyEditableTexts), + container: compactManualApplyContext(op.container), + contextHints: Array.isArray(op.contextHints) ? op.contextHints.slice(0, 8) : undefined, + }; +} + +function compactManualApplyContext(value) { + if (!value || typeof value !== 'object') return null; + return { + ref: value.ref, + tagName: value.tagName || value.tag || null, + id: value.id || null, + classes: Array.isArray(value.classes) ? value.classes : [], + textContent: truncateManualApplyText(value.textContent, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }; +} + +function compactNearbyManualEditTexts(items) { + return (Array.isArray(items) ? items : []) + .slice(0, MANUAL_APPLY_COMPACT_NEARBY_LIMIT) + .map((item) => typeof item === 'string' ? { text: truncateManualApplyText(item, MANUAL_APPLY_COMPACT_TEXT_LIMIT) } : { + ref: item?.ref, + tag: item?.tag, + classes: Array.isArray(item?.classes) ? item.classes : [], + text: truncateManualApplyText(item?.text, MANUAL_APPLY_COMPACT_TEXT_LIMIT), + }); +} + +function truncateManualApplyText(value, max) { + if (typeof value !== 'string') return value || null; + return value.length > max ? value.slice(0, max) : value; +} + +async function pushApplyBatchInChunksAndWait(batch, pageUrl, context = {}) { + const repair = context?.repair || batch?.repair || null; + if (repair) return pushApplyEventAndWait(batch, pageUrl, null, repair); + const chunks = splitManualApplyBatch(batch, manualEditApplyChunkSize()); + if (chunks.length <= 1) return pushApplyEventAndWait(batch, pageUrl); + + const expectedOpsByEntry = new Map(); + for (const entry of batch?.entries || []) { + expectedOpsByEntry.set(entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0); + } + + const appliedOpsByEntry = new Map(); + const failedByEntry = new Map(); + const files = new Set(); + const notes = []; + let aborted = false; + + for (const chunk of chunks) { + if (aborted) { + markChunkEntriesFailed(failedByEntry, chunk, 'manual_edit_chunk_aborted'); + continue; + } + + let result; + try { + result = normalizeApplyChunkResult(await pushApplyEventAndWait(chunk.batch, pageUrl, chunk.meta)); + } catch (err) { + markChunkEntriesFailed(failedByEntry, chunk, err.message || 'chat_agent_error'); + aborted = true; + continue; + } + + for (const file of result.files) files.add(file); + notes.push(...result.notes); + + const chunkFailedIds = new Set(); + for (const item of result.failed) { + const entryId = item.entryId || item.id; + if (!entryId) continue; + chunkFailedIds.add(entryId); + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { + entryId, + reason: item.reason || item.message || 'failed', + candidates: Array.isArray(item.candidates) ? item.candidates : [], + }); + } + } + + if (result.status === 'error') { + markChunkEntriesFailed(failedByEntry, chunk, result.message || firstFailureReason(result) || 'chat_agent_error'); + aborted = true; + continue; + } + + const reportedAppliedIds = new Set(result.appliedEntryIds); + for (const entryId of reportedAppliedIds) { + if (!chunk.entryIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + appliedOpsByEntry.set(entryId, (appliedOpsByEntry.get(entryId) || 0) + (chunk.opCountsByEntry.get(entryId) || 0)); + } + + for (const entryId of chunk.entryIds) { + if (reportedAppliedIds.has(entryId) || chunkFailedIds.has(entryId)) continue; + if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + } + + const appliedEntryIds = []; + for (const [entryId, expectedOps] of expectedOpsByEntry.entries()) { + if (failedByEntry.has(entryId)) continue; + if ((appliedOpsByEntry.get(entryId) || 0) === expectedOps && expectedOps > 0) { + appliedEntryIds.push(entryId); + } else if (!failedByEntry.has(entryId)) { + failedByEntry.set(entryId, { entryId, reason: 'not_reported_applied', candidates: [] }); + } + } + + const failed = [...failedByEntry.values()]; + return { + status: failed.length === 0 ? 'done' : appliedEntryIds.length > 0 ? 'partial' : 'error', + appliedEntryIds, + failed, + files: [...files], + notes, + }; +} + +function normalizeApplyChunkResult(result) { + const status = result?.status === 'partial' ? 'partial' : result?.status === 'error' ? 'error' : 'done'; + return { + status, + message: typeof result?.message === 'string' ? result.message : null, + appliedEntryIds: Array.isArray(result?.appliedEntryIds) ? result.appliedEntryIds.filter((id) => typeof id === 'string') : [], + failed: Array.isArray(result?.failed) ? result.failed.filter(Boolean) : [], + files: Array.isArray(result?.files) ? result.files.filter((file) => typeof file === 'string') : [], + notes: Array.isArray(result?.notes) ? result.notes.filter((note) => typeof note === 'string') : [], + }; +} + +function manualApplyResultShapeHint(eventId = 'EVENT_ID') { + return `Use live-poll.mjs --reply ${eventId} done --data '{"status":"done","appliedEntryIds":["ENTRY_ID"],"failed":[],"files":["src/page.html"],"notes":[]}'`; +} + +function invalidManualApplyResult(reason, eventId, extra = {}) { + return { + ok: false, + body: { + error: 'invalid_manual_apply_result', + reason, + hint: manualApplyResultShapeHint(eventId), + ...extra, + }, + }; +} + +function validateManualApplyResultMessage(msg, deferred) { + let data = msg?.data; + const eventId = msg?.id || deferred?.event?.id || 'EVENT_ID'; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return invalidManualApplyResult('missing_result_data', eventId); + } + if ('entries' in data || 'ops' in data) { + return invalidManualApplyResult('summary_result_not_allowed', eventId); + } + if (!['done', 'partial', 'error'].includes(data.status)) { + return invalidManualApplyResult('invalid_status', eventId, { status: data.status ?? null }); + } + + for (const key of ['appliedEntryIds', 'failed', 'files', 'notes']) { + if (!Array.isArray(data[key])) { + return invalidManualApplyResult(`${key}_must_be_array`, eventId); + } + } + + for (const [index, value] of data.appliedEntryIds.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('appliedEntryIds_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.files.entries()) { + if (typeof value !== 'string' || !value) { + return invalidManualApplyResult('files_must_contain_strings', eventId, { index }); + } + } + for (const [index, value] of data.notes.entries()) { + if (typeof value !== 'string') { + return invalidManualApplyResult('notes_must_contain_strings', eventId, { index }); + } + } + for (const [index, item] of data.failed.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + return invalidManualApplyResult('failed_must_contain_objects', eventId, { index }); + } + if (typeof item.entryId !== 'string' || !item.entryId) { + return invalidManualApplyResult('failed_entryId_required', eventId, { index }); + } + if (typeof item.reason !== 'string' || !item.reason) { + return invalidManualApplyResult('failed_reason_required', eventId, { index }); + } + } + + const eventEntryIds = new Set((deferred?.batch?.entries || []).map((entry) => entry.id).filter(Boolean)); + for (const entryId of data.appliedEntryIds) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(entryId)) { + return invalidManualApplyResult('applied_entry_id_not_in_event', eventId, { entryId }); + } + } + for (const item of data.failed) { + if (eventEntryIds.size > 0 && !eventEntryIds.has(item.entryId)) { + return invalidManualApplyResult('failed_entry_id_not_in_event', eventId, { entryId: item.entryId }); + } + } + + if (data.status === 'done') { + if (data.failed.length > 0) { + return invalidManualApplyResult('done_result_has_failed_entries', eventId); + } + if (countManualApplyOps(deferred?.batch) > 0 && data.appliedEntryIds.length === 0) { + return invalidManualApplyResult('done_result_missing_applied_entry_ids', eventId); + } + } + if (data.status === 'partial' && data.appliedEntryIds.length === 0 && data.failed.length === 0) { + return invalidManualApplyResult('partial_result_has_no_entries', eventId); + } + if (data.status === 'error' && data.appliedEntryIds.length > 0) { + return invalidManualApplyResult('error_result_has_applied_entries', eventId); + } + + return { + ok: true, + result: { + status: data.status, + message: typeof data.message === 'string' ? data.message : undefined, + appliedEntryIds: data.appliedEntryIds, + failed: data.failed, + files: data.files, + notes: data.notes, + }, + }; +} + +function firstFailureReason(result) { + const first = Array.isArray(result?.failed) ? result.failed.find(Boolean) : null; + return first?.reason || first?.message || null; +} + +function markChunkEntriesFailed(failedByEntry, chunk, reason) { + for (const entryId of chunk.entryIds) { + if (failedByEntry.has(entryId)) continue; + failedByEntry.set(entryId, { entryId, reason, candidates: [] }); + } +} + +function splitManualApplyBatch(batch, maxOps) { + const totalOpCount = countManualApplyOps(batch); + if (totalOpCount <= maxOps) { + return [{ + batch, + meta: null, + entryIds: new Set((batch?.entries || []).map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: new Map((batch?.entries || []).map((entry) => [entry.id, Array.isArray(entry.ops) ? entry.ops.length : 0])), + }]; + } + + const rawChunks = []; + let current = createManualApplyChunkBuilder(); + for (const entry of batch?.entries || []) { + const ops = entry.ops || []; + if (ops.length <= maxOps) { + if (current.opCount > 0 && current.opCount + ops.length > maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) addOpToManualApplyChunk(current, entry, op); + continue; + } + if (current.opCount > 0) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + for (const op of ops) { + if (current.opCount >= maxOps) { + rawChunks.push(current); + current = createManualApplyChunkBuilder(); + } + addOpToManualApplyChunk(current, entry, op); + } + } + if (current.opCount > 0) rawChunks.push(current); + + return rawChunks.map((chunk, index) => ({ + batch: { + ...batch, + count: chunk.opCount, + entries: chunk.entries, + ops: chunk.ops, + candidates: filterManualApplyChunkCandidates(batch, chunk.refsByEntry), + context: { + ...(batch?.context || {}), + totalEntries: chunk.entries.length, + totalOps: chunk.opCount, + chunkIndex: index + 1, + chunkTotal: rawChunks.length, + totalApplyOps: totalOpCount, + }, + }, + meta: { + index: index + 1, + total: rawChunks.length, + opCount: chunk.opCount, + totalOpCount, + }, + entryIds: new Set(chunk.entries.map((entry) => entry.id).filter(Boolean)), + opCountsByEntry: chunk.opCountsByEntry, + })); +} + +function createManualApplyChunkBuilder() { + return { + entries: [], + entryById: new Map(), + entryIds: new Set(), + ops: [], + refsByEntry: new Map(), + opCountsByEntry: new Map(), + opCount: 0, + }; +} + +function addOpToManualApplyChunk(chunk, entry, op) { + let chunkEntry = chunk.entryById.get(entry.id); + if (!chunkEntry) { + chunkEntry = { ...entry, ops: [] }; + chunk.entryById.set(entry.id, chunkEntry); + chunk.entryIds.add(entry.id); + chunk.entries.push(chunkEntry); + } + chunkEntry.ops.push(op); + chunk.ops.push({ ...op, entryId: op.entryId || entry.id }); + if (!chunk.refsByEntry.has(entry.id)) chunk.refsByEntry.set(entry.id, new Set()); + if (op.ref) chunk.refsByEntry.get(entry.id).add(op.ref); + chunk.opCountsByEntry.set(entry.id, (chunk.opCountsByEntry.get(entry.id) || 0) + 1); + chunk.opCount += 1; +} + +function filterManualApplyChunkCandidates(batch, refsByEntry) { + return (batch?.candidates || []).filter((candidate) => { + const refs = refsByEntry.get(candidate.entryId); + if (!refs) return false; + if (!candidate.ref) return true; + return refs.has(candidate.ref); + }); +} + +function resolveApplyDeferred(eventId, body) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.resolve(body); + return true; +} + +function rejectApplyDeferred(eventId, reason) { + const deferred = state.pendingApplyDeferreds.get(eventId); + if (!deferred) return false; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + removeManualApplyEvidence(deferred.event?.evidencePath); + deferred.reject(new Error(reason || 'chat_agent_error')); + return true; +} + +function snapshotApplyEventFiles(batch) { + const snapshot = new Map(); + for (const relativeFile of collectManualApplyFiles(batch)) { + const absolute = path.resolve(process.cwd(), relativeFile); + try { + snapshot.set(relativeFile, { + exists: fs.existsSync(absolute), + content: fs.existsSync(absolute) ? fs.readFileSync(absolute, 'utf-8') : '', + }); + } catch { + // If a file cannot be read before dispatch, do not attempt late rollback. + } + } + return snapshot; +} + +function manualApplyTransactionPath(cwd = process.cwd()) { + return path.join(getLiveDir(cwd), 'manual-edit-apply-transaction.json'); +} + +function readManualApplyTransaction(cwd = process.cwd()) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { + return null; + } +} + +function writeManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, batch }) { + const file = manualApplyTransactionPath(cwd); + const files = collectManualApplyFiles(batch); + const transaction = { + version: 1, + id: randomUUID().replace(/-/g, '').slice(0, 8), + createdAt: new Date().toISOString(), + pageUrl, + entryIds: (batch?.entries || []).map((entry) => entry.id).filter(Boolean), + files: files.map((relativeFile) => { + const absolute = path.resolve(cwd, relativeFile); + const exists = fs.existsSync(absolute); + return { + file: relativeFile, + exists, + content: exists ? fs.readFileSync(absolute, 'utf-8') : '', + }; + }), + }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, JSON.stringify(transaction, null, 2) + '\n', 'utf-8'); + fs.renameSync(`${file}.tmp`, file); + return transaction; +} + +function clearManualApplyTransaction(cwd = process.cwd(), transactionId = null) { + const file = manualApplyTransactionPath(cwd); + if (!fs.existsSync(file)) return false; + if (transactionId) { + const existing = readManualApplyTransaction(cwd); + if (existing?.id && existing.id !== transactionId) return false; + } + try { + fs.unlinkSync(file); + return true; + } catch { + return false; + } +} + +function rollbackManualApplyTransaction({ cwd = process.cwd(), pageUrl = null, reason = 'manual_edit_transaction_rollback' } = {}) { + const transaction = readManualApplyTransaction(cwd); + if (!transaction) return null; + if (pageUrl && transaction.pageUrl && transaction.pageUrl !== pageUrl) return null; + + let pendingIds = new Set(); + try { + const buffer = readManualEditsBuffer(cwd); + pendingIds = new Set((buffer.entries || []).map((entry) => entry.id).filter(Boolean)); + } catch { + pendingIds = new Set(transaction.entryIds || []); + } + const shouldRollback = (transaction.entryIds || []).some((id) => pendingIds.has(id)); + if (!shouldRollback) { + clearManualApplyTransaction(cwd, transaction.id); + return { id: transaction.id, reason, rolledBackFiles: [], rollbackFailures: [], skipped: 'entries_not_pending' }; + } + + const rolledBackFiles = []; + const rollbackFailures = []; + for (const item of transaction.files || []) { + const relativeFile = normalizeProjectFile(item.file); + if (!relativeFile) continue; + const absolute = path.resolve(cwd, relativeFile); + try { + if (item.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, item.content || '', 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + clearManualApplyTransaction(cwd, transaction.id); + recordManualEditActivity('manual_edit_transaction_rolled_back', { + id: transaction.id, + pageUrl: transaction.pageUrl || null, + reason, + entryIds: transaction.entryIds || [], + rolledBackFiles: rolledBackFiles.map(summarizeManualLogFile).filter(Boolean), + rollbackFailures: summarizeManualDiagnostics(rollbackFailures), + }); + return { id: transaction.id, reason, rolledBackFiles, rollbackFailures }; +} + +function collectManualApplyFiles(batch, extraFiles = []) { + const files = []; + for (const entry of batch?.entries || []) { + for (const op of entry.ops || []) files.push(op.sourceHint?.file); + } + for (const candidate of batch?.candidates || []) { + files.push(candidate.sourceHint?.relativeFile, candidate.sourceHint?.file); + for (const item of candidate.textMatches || []) files.push(item.file); + for (const item of candidate.objectKeyMatches || []) files.push(item.file); + for (const item of candidate.locatorMatches || []) files.push(item.file); + for (const item of candidate.contextTextMatches || []) files.push(item.file); + } + files.push(...(extraFiles || [])); + return [...new Set(files)] + .map((file) => normalizeProjectFile(file)) + .filter(Boolean); +} + +function normalizeProjectFile(file) { + if (!file || typeof file !== 'string') return null; + const absolute = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file); + const relative = path.relative(process.cwd(), absolute); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null; + return relative; +} + +function rollbackApplySnapshot(batch, rollbackSnapshot, extraFiles = [], reason = 'manual_edit_apply_snapshot_rollback') { + const scope = collectManualApplyFiles(batch, extraFiles); + const rolledBackFiles = []; + const rollbackFailures = []; + for (const relativeFile of scope) { + const before = rollbackSnapshot?.get(relativeFile); + if (!before) continue; + const absolute = path.resolve(process.cwd(), relativeFile); + try { + if (before.exists) { + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, before.content, 'utf-8'); + } else if (fs.existsSync(absolute)) { + fs.rmSync(absolute); + } + rolledBackFiles.push(relativeFile); + } catch (err) { + rollbackFailures.push({ file: relativeFile, reason: 'restore_failed', message: err.message || String(err) }); + } + } + return { rolledBackFiles, rollbackFailures }; +} + +function rollbackTimedOutApplyReply(msg) { + const details = state.timedOutApplyIds.get(msg.id); + if (!details) return { rolledBackFiles: [], rollbackFailures: [] }; + state.timedOutApplyIds.delete(msg.id); + return rollbackApplySnapshot(details.batch, details.rollbackSnapshot, msg.data?.files || [], 'stale_manual_edit_apply_reply'); +} + // Cap per-annotation upload size. A full 1920×1080 PNG is typically <1 MB; // cap at 10 MB to guard against runaway writes from a misbehaving client. const MAX_ANNOTATION_BYTES = 10 * 1024 * 1024; function enqueueEvent(event) { if (!event || (event.id && state.pendingEvents.some((entry) => entry.event?.id === event.id && entry.event?.type === event.type))) return; - state.pendingEvents.push({ event, leaseUntil: 0 }); + state.pendingEvents.push({ event, leaseUntil: 0, seq: state.nextEventSeq++ }); flushPendingPolls(); } @@ -91,7 +883,11 @@ function restorePendingEventsFromStore() { } function findAvailablePendingEvent(now = Date.now()) { - return state.pendingEvents.find((entry) => !entry.leaseUntil || entry.leaseUntil <= now); + for (const entry of state.pendingEvents) { + if (entry.leaseUntil && entry.leaseUntil > now) continue; + return entry; + } + return null; } function leaseEvent(entry, leaseMs) { @@ -108,9 +904,96 @@ function acknowledgePendingEvent(id) { if (!id) return false; const idx = state.pendingEvents.findIndex((entry) => entry.event?.id === id); if (idx === -1) return false; + const acknowledged = state.pendingEvents[idx].event; state.pendingEvents.splice(idx, 1); scheduleLeaseFlush(); - return true; + return acknowledged; +} + +function manualApplyReplyCommand(eventOrId = 'EVENT_ID') { + const id = typeof eventOrId === 'string' ? eventOrId : eventOrId?.id || 'EVENT_ID'; + return `live-poll.mjs --reply ${id} done --data ''`; +} + +function buildManualApplyAgentAction(eventOrId = 'EVENT_ID') { + return { + kind: 'manual_edit_apply', + required: 'apply_source_edits_then_reply', + replyCommand: manualApplyReplyCommand(eventOrId), + warning: 'Polling only leases this work item; it does not commit source edits.', + }; +} + +function summarizeManualApplyEvent(event = {}, batch = event.batch) { + const entries = Array.isArray(batch?.entries) ? batch.entries : []; + const opCount = entries.reduce((sum, entry) => sum + (Array.isArray(entry.ops) ? entry.ops.length : 0), 0); + return { + pageUrl: event.pageUrl || null, + chunk: event.chunk || null, + entryCount: entries.length, + opCount, + files: collectManualApplyFiles(batch), + }; +} + +function summarizePendingEventForStatus(entry) { + const event = entry.event || {}; + const summary = { + id: event.id, + type: event.type, + leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), + leaseUntil: entry.leaseUntil || null, + }; + if (event.type === 'manual_edit_apply') { + summary.pageUrl = event.pageUrl || null; + summary.chunk = event.chunk || null; + summary.repair = event.repair || null; + summary.evidencePath = event.evidencePath || null; + summary.agentAction = event.agentAction || buildManualApplyAgentAction(event); + summary.manualApplySummary = summarizeManualApplyEvent(event, state.pendingApplyDeferreds.get(event.id)?.batch || event.batch); + } + return summary; +} + +function cancelPendingManualApplyEvents(pageUrl, reason = 'manual_edit_discarded') { + const canceledById = new Map(); + const shouldCancel = (event) => event?.type === 'manual_edit_apply' && (!pageUrl || event.pageUrl === pageUrl); + + for (let i = state.pendingEvents.length - 1; i >= 0; i -= 1) { + const event = state.pendingEvents[i]?.event; + if (!shouldCancel(event)) continue; + state.pendingEvents.splice(i, 1); + removeManualApplyEvidence(event.evidencePath); + canceledById.set(event.id, { + id: event.id, + pageUrl: event.pageUrl, + entryCount: event.batch?.entries?.length || 0, + }); + } + + for (const [eventId, deferred] of [...state.pendingApplyDeferreds.entries()]) { + if (!shouldCancel(deferred.event)) continue; + state.pendingApplyDeferreds.delete(eventId); + clearTimeout(deferred.timer); + const rollback = rollbackApplySnapshot(deferred.batch, deferred.rollbackSnapshot, [], reason); + tombstoneTimedOutApplyId(eventId, { + batch: deferred.batch, + rollbackSnapshot: deferred.rollbackSnapshot, + reason, + }); + removeManualApplyEvidence(deferred.event?.evidencePath); + canceledById.set(eventId, { + id: eventId, + pageUrl: deferred.pageUrl, + entryCount: deferred.batch?.entries?.length || 0, + rolledBackFiles: rollback.rolledBackFiles, + rollbackFailures: rollback.rollbackFailures, + }); + deferred.reject(new Error(reason)); + } + + if (canceledById.size > 0) flushPendingPolls(); + return [...canceledById.values()]; } function scheduleLeaseFlush() { @@ -167,6 +1050,96 @@ function broadcast(msg) { } } +function recordManualEditActivity(type, details = {}) { + const entry = { + seq: state.nextManualEditSeq++, + type, + ts: new Date().toISOString(), + ...details, + }; + state.manualEditActivity = entry; + if (DEBUG_MANUAL_EDIT_EVENTS) { + try { + const filePath = path.join(getLiveDir(process.cwd()), 'manual-edit-events.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, JSON.stringify(entry) + '\n'); + } catch { + /* diagnostics are best-effort; never block live mode on observability */ + } + } + broadcast(entry); + return entry; +} + +function getManualEditStatus() { + try { + const { totalCount, perPage } = countPendingByPage(process.cwd()); + return { totalCount, perPage, lastActivity: state.manualEditActivity }; + } catch (err) { + return { + totalCount: null, + perPage: {}, + lastActivity: state.manualEditActivity, + error: err.message, + }; + } +} + +function summarizePendingManualEditBatch(pageUrl = null) { + try { + const buffer = readManualEditsBuffer(process.cwd()); + const entries = (buffer.entries || []) + .filter((entry) => !pageUrl || entry.pageUrl === pageUrl); + return { + pendingEntryCount: entries.length, + pendingOpCount: entries.reduce((sum, entry) => sum + (entry.ops?.length || 0), 0), + }; + } catch (err) { + return { pendingSummaryError: err.message || String(err) }; + } +} + +function summarizeManualApplyFailures(failed) { + if (!Array.isArray(failed)) return []; + return failed.slice(0, 20).map((item) => ({ + id: item.id || item.entryId || null, + reason: item.reason || item.message || 'failed', + message: compactManualLogText(item.message, 300), + files: Array.isArray(item.files) ? item.files.slice(0, 12).map(summarizeManualLogFile).filter(Boolean) : undefined, + checks: summarizeManualDiagnostics(item.checks), + failures: summarizeManualDiagnostics(item.failures), + candidates: summarizeManualDiagnostics(item.candidates), + })); +} + +function summarizeManualDiagnostics(items) { + if (!Array.isArray(items) || items.length === 0) return undefined; + return items.slice(0, 12).map((item) => ({ + reason: item.reason || item.kind || undefined, + detail: compactManualLogText(item.detail, 220), + message: compactManualLogText(item.message, 300), + file: summarizeManualLogFile(item.file || item.relativeFile), + line: item.line || undefined, + ref: compactManualLogText(item.ref, 180), + marker: compactManualLogText(item.marker, 120), + files: Array.isArray(item.files) ? item.files.slice(0, 8).map(summarizeManualLogFile).filter(Boolean) : undefined, + })); +} + +function summarizeManualLogFile(file) { + if (!file || typeof file !== 'string') return undefined; + if (!path.isAbsolute(file)) return file; + const relative = path.relative(process.cwd(), file); + return relative && !relative.startsWith('..') && !path.isAbsolute(relative) ? relative : file; +} + +function compactManualLogText(value, max = 200) { + if (typeof value !== 'string') return undefined; + const normalized = value.replace(/\s+/g, ' ').trim(); + if (normalized.length <= max) return normalized; + return normalized.slice(0, max) + `... [truncated ${normalized.length - max} chars]`; +} + // --------------------------------------------------------------------------- // Load scripts // --------------------------------------------------------------------------- @@ -215,7 +1188,6 @@ function statOrNull(filePath) { try { return fs.statSync(filePath); } catch { return null; } } -// --------------------------------------------------------------------------- // HTTP request handler // --------------------------------------------------------------------------- @@ -352,14 +1324,10 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { status: 'ok', port: state.port, connectedClients: state.sseClients.size, + pendingEvents: state.pendingEvents.map((entry) => summarizePendingEventForStatus(entry)), agentPolling: agentPollingConnected(), - pendingEvents: state.pendingEvents.map((entry) => ({ - id: entry.event?.id, - type: entry.event?.type, - leased: !!(entry.leaseUntil && entry.leaseUntil > Date.now()), - leaseUntil: entry.leaseUntil || null, - })), activeSessions: sessions, + manualEdits: getManualEditStatus(), })); return; } @@ -487,6 +1455,335 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } + // --- Manual copy edits: Save stages entries, Apply commits the staged + // page batch through the local AI copy-edit runner. + if (p === '/manual-edit-stash' && req.method === 'POST') { + let body = ''; + req.on('data', (c) => { body += c; }); + req.on('end', () => { + let msg; + try { msg = JSON.parse(body); } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + if (msg.token !== state.token) { + res.writeHead(401, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + const error = validateEvent({ ...msg, type: 'manual_edits' }); + if (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error })); + return; + } + try { + stageManualEditEntry(process.cwd(), { + id: msg.id, + pageUrl: msg.pageUrl, + element: msg.element, + ops: msg.ops, + }); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stash_write_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const pendingCount = perPage[msg.pageUrl] || 0; + recordManualEditActivity('manual_edit_stashed', { + id: msg.id, + pageUrl: msg.pageUrl, + opCount: msg.ops.length, + pendingCount, + totalCount, + hintedFileCount: new Set((msg.ops || []).map((op) => summarizeManualLogFile(op.sourceHint?.file)).filter(Boolean)).size, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, pendingCount, totalCount, perPage })); + }); + return; + } + + // GET /manual-edit-stash?pageUrl= → { count, totalCount, perPage, entries } + if (p === '/manual-edit-stash' && req.method === 'GET') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl') || ''; + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const buffer = readManualEditsBuffer(process.cwd()); + const entriesForPage = pageUrl ? buffer.entries.filter((e) => e.pageUrl === pageUrl) : buffer.entries; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + count: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + entries: entriesForPage, + })); + return; + } + + // POST /manual-edit-commit?pageUrl= → ask the AI to apply the staged page batch. + if (p === '/manual-edit-commit' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + const asyncMode = /^(1|true|yes)$/i.test(url.searchParams.get('async') || ''); + const repairOnly = /^(1|true|yes)$/i.test(url.searchParams.get('repair') || ''); + const existingTransaction = readManualApplyTransaction(process.cwd()); + if (repairOnly && !existingTransaction) { + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_repair_transaction_missing' })); + return; + } + const recoveredTransaction = repairOnly ? null : rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_recovered_abandoned_transaction', + }); + const before = getManualEditStatus(); + const pendingCount = pageUrl ? (before.perPage[pageUrl] || 0) : before.totalCount; + recordManualEditActivity('manual_edit_commit_started', { + pageUrl, + repairOnly, + pendingCount, + totalCount: before.totalCount, + recoveredTransaction: recoveredTransaction ? { + id: recoveredTransaction.id, + reason: recoveredTransaction.reason, + skipped: recoveredTransaction.skipped, + rolledBackFiles: recoveredTransaction.rolledBackFiles, + rollbackFailures: summarizeManualDiagnostics(recoveredTransaction.rollbackFailures), + } : null, + ...summarizePendingManualEditBatch(pageUrl), + }); + if (asyncMode) { + res.writeHead(202, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + status: 'started', + pendingCount, + totalCount: before.totalCount, + perPage: before.perPage, + })); + } + (async () => { + let result; + let routedProvider = 'subprocess'; + let transaction = null; + let commitBatch = null; + try { + if (pendingCount > 0) { + const transactionBatch = buildManualEditEvidence({ cwd: process.cwd(), pageUrl }); + commitBatch = transactionBatch; + if (!repairOnly && countManualApplyOps(transactionBatch) > 0) { + transaction = writeManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + batch: transactionBatch, + }); + } else if (repairOnly && existingTransaction) { + transaction = existingTransaction; + } + } + const requestedMode = (process.env.IMPECCABLE_LIVE_COPY_AGENT || 'auto').trim().toLowerCase(); + const useChatRoute = requestedMode === 'chat' + || (requestedMode === 'auto' && chatAgentLikelyActive()); + if (useChatRoute) { + routedProvider = 'chat'; + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider: 'chat', + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + applyBatchToSource: (batch, context) => pushApplyBatchInChunksAndWait(batch, pageUrl, context), + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } else { + const timeoutMs = Number(process.env.IMPECCABLE_LIVE_COPY_AGENT_TIMEOUT_MS || 120000); + const provider = ['codex', 'claude', 'mock'].includes(requestedMode) ? requestedMode : undefined; + result = await commitManualEdits({ + cwd: process.cwd(), + pageUrl, + provider, + env: process.env, + timeoutMs, + chatAvailable: chatAgentLikelyActive, + repairOnly, + transactionId: transaction?.id || existingTransaction?.id || null, + batch: commitBatch, + }); + } + } catch (err) { + if (transaction) { + rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_commit_exception', + }); + } + const message = err.stderr?.toString?.() || err.message; + recordManualEditActivity('manual_edit_commit_failed', { + pageUrl, + provider: routedProvider, + error: 'manual_edit_commit_failed', + message, + transactionId: transaction?.id || null, + }); + if (!asyncMode) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: 'manual_edit_commit_failed', + message, + })); + } + return; + } finally { + if (transaction) { + const shouldKeepTransaction = result?.needsManualDecision === true; + if (!shouldKeepTransaction) clearManualApplyTransaction(process.cwd(), transaction.id); + } + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + if (result?.needsManualDecision) { + recordManualEditActivity('manual_edit_repair_needs_decision', { + pageUrl, + provider: routedProvider, + transactionId: transaction?.id || existingTransaction?.id || null, + repair: result.repair || null, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } else { + recordManualEditActivity('manual_edit_commit_done', { + pageUrl, + provider: routedProvider, + reason: result.reason || null, + repair: result.repair || null, + appliedCount: Array.isArray(result.applied) ? result.applied.length : 0, + failedCount: Array.isArray(result.failed) ? result.failed.length : 0, + failed: summarizeManualApplyFailures(result.failed), + files: Array.isArray(result.files) ? result.files.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + warnings: summarizeManualDiagnostics(result.warnings), + rolledBackFiles: Array.isArray(result.rolledBackFiles) ? result.rolledBackFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : [], + rollbackFailures: summarizeManualDiagnostics(result.rollbackFailures), + unreportedFiles: Array.isArray(result.unreportedFiles) ? result.unreportedFiles.slice(0, 20).map(summarizeManualLogFile).filter(Boolean) : undefined, + noteCount: Array.isArray(result.notes) ? result.notes.length : 0, + cleared: result.cleared || 0, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + }); + } + if (!asyncMode) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ...result, totalCount, perPage })); + } + })(); + return; + } + + // POST /manual-edit-repair-decision → user resolves an exhausted repair loop. + if (p === '/manual-edit-repair-decision' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + let payload = {}; + try { payload = body ? JSON.parse(body) : {}; } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON' })); + return; + } + const token = payload.token || url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = payload.pageUrl || url.searchParams.get('pageUrl') || null; + const action = String(payload.action || url.searchParams.get('action') || '').trim().toLowerCase(); + if (action !== 'rollback') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'unsupported_manual_edit_repair_decision', action })); + return; + } + const rollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_user_requested_rollback', + }); + const { totalCount, perPage } = countPendingByPage(process.cwd()); + const response = { + action, + pageUrl, + rollback, + remainingCount: pageUrl ? (perPage[pageUrl] || 0) : totalCount, + totalCount, + perPage, + }; + recordManualEditActivity('manual_edit_repair_rollback_done', response); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + }); + return; + } + + // POST /manual-edit-discard?pageUrl= → drops entries (all if no pageUrl) + if (p === '/manual-edit-discard' && req.method === 'POST') { + const token = url.searchParams.get('token'); + if (token !== state.token) { res.writeHead(401); res.end('Unauthorized'); return; } + const pageUrl = url.searchParams.get('pageUrl'); + let discarded; + let discardedEntries = []; + let canceledApplyEvents = []; + let transactionRollback = null; + try { + const buffer = readManualEditsBuffer(process.cwd()); + transactionRollback = rollbackManualApplyTransaction({ + cwd: process.cwd(), + pageUrl, + reason: 'manual_edit_discarded', + }); + if (pageUrl) { + discardedEntries = buffer.entries.filter((entry) => entry.pageUrl === pageUrl); + discarded = removeManualEditEntries(process.cwd(), (entry) => entry.pageUrl === pageUrl); + } else { + discardedEntries = buffer.entries; + discarded = truncateManualEditsBuffer(process.cwd()); + } + canceledApplyEvents = cancelPendingManualApplyEvents(pageUrl); + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'discard_failed', message: err.message })); + return; + } + const { totalCount, perPage } = countPendingByPage(process.cwd()); + recordManualEditActivity('manual_edit_discarded', { + pageUrl, + discarded, + canceledApplyIds: canceledApplyEvents.map((event) => event.id), + transactionRollback: transactionRollback ? { + id: transactionRollback.id, + rolledBackFiles: transactionRollback.rolledBackFiles?.map(summarizeManualLogFile).filter(Boolean) || [], + rollbackFailures: summarizeManualDiagnostics(transactionRollback.rollbackFailures), + skipped: transactionRollback.skipped, + } : undefined, + totalCount, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ discarded, entries: discardedEntries, canceledApplyEvents, totalCount, perPage })); + return; + } + + // Defense in depth: redirect any stragglers from the old /manual-edit endpoint. + if (p === '/manual-edit' && req.method === 'POST') { + res.writeHead(410, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: '/manual-edit is removed; use /manual-edit-stash and /manual-edit-commit for staged copy edits.' })); + return; + } + // --- Browser→server events (replaces WebSocket messages) --- if (p === '/events' && req.method === 'POST') { let body = ''; @@ -503,6 +1800,18 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + // Defense in depth: manual copy edits must use the staged stash/apply + // endpoints. The direct Save event path is disabled in the browser. + if (msg.type === 'manual_edits') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edits must POST to /manual-edit-stash, not /events' })); + return; + } + if (msg.type === 'manual_edit_apply') { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'manual_edit_apply is disabled; use /manual-edit-stash then /manual-edit-commit' })); + return; + } const error = validateEvent(msg); if (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); @@ -518,7 +1827,9 @@ function createRequestHandler({ detectScript, sessionPath, livePath }) { return; } } - if (msg.type !== 'checkpoint') enqueueEvent(msg); + if (msg.type !== 'checkpoint') { + enqueueEvent(msg); + } res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ ok: true })); }); @@ -560,6 +1871,7 @@ function handlePollGet(req, res, url) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } + state.lastPollAt = Date.now(); const timeout = parseInt(url.searchParams.get('timeout') || DEFAULT_POLL_TIMEOUT, 10); const leaseMs = parseInt(url.searchParams.get('leaseMs') || '30000', 10); const available = findAvailablePendingEvent(); @@ -578,6 +1890,7 @@ function handlePollGet(req, res, url) { }, timeout); function resolve(event) { clearTimeout(timer); + state.lastPollAt = Date.now(); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(event)); } @@ -607,8 +1920,74 @@ function handlePollPost(req, res) { res.end(JSON.stringify({ error: 'Unauthorized' })); return; } - acknowledgePendingEvent(msg.id); - if (state.sessionStore && msg.id) { + const pendingApplyDeferred = state.pendingApplyDeferreds.get(msg.id); + if (pendingApplyDeferred) { + const validation = validateManualApplyResultMessage(msg, pendingApplyDeferred); + if (!validation.ok) { + recordManualEditActivity('manual_edit_apply_reply_invalid', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + reason: validation.body?.reason || validation.body?.error || 'invalid_manual_apply_result', + status: msg.data?.status || null, + }); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(validation.body)); + return; + } + recordManualEditActivity('manual_edit_apply_reply_received', { + id: msg.id, + pageUrl: pendingApplyDeferred.pageUrl, + chunk: pendingApplyDeferred.event?.chunk || null, + repair: pendingApplyDeferred.event?.repair || null, + status: validation.result.status, + appliedCount: validation.result.appliedEntryIds.length, + failed: summarizeManualApplyFailures(validation.result.failed), + fileCount: validation.result.files.length, + noteCount: validation.result.notes.length, + }); + resolveApplyDeferred(msg.id, validation.result); + acknowledgePendingEvent(msg.id); + flushPendingPolls(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + return; + } + if (state.timedOutApplyIds.has(msg.id)) { + const rollback = rollbackTimedOutApplyReply(msg); + recordManualEditActivity('manual_edit_apply_stale_reply_rejected', { + id: msg.id, + rolledBackFileCount: rollback.rolledBackFiles?.length || 0, + rollbackFailureCount: rollback.rollbackFailures?.length || 0, + }); + res.writeHead(409, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'stale_manual_edit_apply_reply', ...rollback })); + return; + } + const acknowledgedEvent = acknowledgePendingEvent(msg.id); + let skipJournalReply = false; + let existingSession = null; + if (!acknowledgedEvent && state.sessionStore && msg.id) { + try { + existingSession = state.sessionStore.getSnapshot(msg.id, { includeCompleted: true }); + if (!existingSession?.updatedAt) existingSession = null; + skipJournalReply = existingSession?.phase === 'completed' || existingSession?.phase === 'discarded'; + } catch { /* fall through and record the reply normally */ } + } + if (!acknowledgedEvent && !existingSession) { + recordManualEditActivity('manual_edit_poll_reply_unknown', { + id: msg.id || null, + type: msg.type || null, + }); + res.writeHead(msg.id ? 404 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + error: msg.id ? 'unknown_poll_reply_id' : 'missing_poll_reply_id', + id: msg.id, + })); + return; + } + if (state.sessionStore && msg.id && !skipJournalReply) { try { const eventType = msg.type === 'steer_done' ? 'steer_done' @@ -624,6 +2003,7 @@ function handlePollPost(req, res) { id: msg.id, file: msg.file, message: msg.message, + sourceEventType: acknowledgedEvent?.type, carbonize: msg.data?.carbonize === true, }); } catch { /* keep reply path best-effort; browser still needs SSE */ } @@ -686,6 +2066,9 @@ Endpoints: /annotation POST raw image/png to stage a variant screenshot /events SSE stream (server→browser) + POST (browser→server) /poll Long-poll for agent CLI + /manual-edit-stash Stage browser copy edits + /manual-edit-commit Apply staged browser copy edits + /manual-edit-discard Discard staged browser copy edits /source Raw source file reader (no-HMR fallback) /status Durable recovery status (token-protected) /health Health check`); @@ -775,7 +2158,12 @@ if (existingRecord?.info) { state.token = randomUUID(); state.sessionStore = createLiveSessionStore({ cwd: process.cwd() }); +rollbackManualApplyTransaction({ + cwd: process.cwd(), + reason: 'manual_edit_server_start_recovered_abandoned_transaction', +}); restorePendingEventsFromStore(); +pruneStaleManualApplyEvidence(process.cwd()); const portArg = args.find(a => a.startsWith('--port=')); state.port = portArg ? parseInt(portArg.split('=')[1], 10) : await findOpenPort(); // Annotation screenshots live in the project root so the agent's Read tool @@ -793,7 +2181,8 @@ httpServer.listen(state.port, '127.0.0.1', () => { const url = `http://localhost:${state.port}`; console.log(`\nImpeccable live server running on ${url}`); console.log(`Token: ${state.token}\n`); - console.log(`Inject: + + +`; + const file = join(tmp, 'Layout.astro'); + writeFileSync(file, original); + + const cfgPath = join(tmp, 'config.json'); + writeFileSync(cfgPath, JSON.stringify({ + files: ['Layout.astro'], + insertBefore: '', + commentSyntax: 'html', + })); + + runInject(tmp, cfgPath, ['--port', '8400']); + const afterInject = readFileSync(file, 'utf-8'); + + assert.equal((afterInject.match(/impeccable-live-start/g) || []).length, 1, 'reinjection should leave one live block'); + assert.match(afterInject, /