import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { buildCopyEditBatchPrompt, chooseCopyEditAgent, describeNoProviderError, extractRunnerErrorMessage, parseCopyEditBatchResult, runCopyEditBatchAgent, runCopyEditPostApplyChecks, } from '../skill/scripts/live-copy-edit-agent.mjs'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; describe('live-copy-edit-agent', () => { it('builds a batch prompt with duplicate-card context and candidate evidence', () => { const prompt = buildCopyEditBatchPrompt({ pageUrl: '/', entries: [{ id: 'cards', pageUrl: '/', element: { tagName: 'div', classes: ['foundation-card'], textContent: 'Color & Contrast 29 Accessibility' }, ops: [{ ref: 'body>main>section#foundation>div.foundation-grid>div:nth-of-type(2)>span.foundation-card-label:nth-of-type(1)', contextRef: 'body>main>section#foundation>div.foundation-grid>div:nth-of-type(2)', tag: 'span', classes: ['foundation-card-label'], originalText: 'Color & Contrast', newText: 'Color!!!', nearbyEditableTexts: [{ text: '29' }, { text: 'Accessibility' }], }], }], candidates: [{ entryId: 'cards', ref: 'body>main>section#foundation>div.foundation-grid>div:nth-of-type(2)>span.foundation-card-label:nth-of-type(1)', textMatches: [{ file: 'site/scripts/data.js', line: 25 }], objectKeyMatches: [{ file: 'site/scripts/components/foundation-animations.js', line: 3 }], }], }, { cwd: '/tmp/project' }); assert.match(prompt, /staged copy-edit batch applier/); assert.match(prompt, /The user already clicked Apply\. Do not ask what to do with the staged edits/); assert.match(prompt, /Apply all staged edits in one coherent batch/); assert.match(prompt, /"entryId": "cards"/); assert.match(prompt, /foundation-card-label/); assert.match(prompt, /site\/scripts\/data\.js/); assert.match(prompt, /Preserve numeric, boolean, array, and object model data/); assert.match(prompt, /do not replace the underlying typed model declaration/); assert.match(prompt, /impeccable:manual-edit-validate/); assert.match(prompt, /Return ONLY JSON/); }); it('bounds and whitelists operation context in batch prompts', () => { const huge = 'Z'.repeat(50_000); const prompt = buildCopyEditBatchPrompt({ pageUrl: '/', entries: [{ id: 'bounded', pageUrl: '/', ops: [{ classes: [huge], originalText: 'Old', newText: 'New', sourceHint: { file: 'src/App.jsx', loc: '12:3', nested: { payload: huge }, }, nearbyEditableTexts: [{ ref: 'body>main>span', tag: 'span', classes: ['label'], text: huge, extra: huge, }], contextHints: [huge], }], }], }); const serializedBatch = prompt.split('Staged copy-edit batch:\n').pop(); const op = JSON.parse(serializedBatch).entries[0].ops[0]; assert.ok(prompt.length < 20_000, `expected compact prompt, got ${prompt.length} characters`); assert.ok(op.classes[0].length < 400); assert.deepEqual(op.sourceHint, { file: 'src/App.jsx', loc: '12:3', line: 12, column: 3, }); assert.deepEqual(Object.keys(op.nearbyEditableTexts[0]).sort(), ['classes', 'ref', 'tag', 'text']); assert.ok(op.nearbyEditableTexts[0].text.length < 400); assert.ok(op.contextHints[0].length < 400); }); it('bounds batch repair, candidate, and element context', () => { const huge = 'Z'.repeat(250_000); const prompt = buildCopyEditBatchPrompt({ pageUrl: '/', repair: { status: 'needs_decision', attempt: 2, maxAttempts: 3, reason: 'source_verification_failed', pageUrl: '/pricing', transactionId: huge, failures: [{ entryId: 'bounded', message: huge, candidates: [{ file: huge, line: 12, kind: 'text' }], failures: [{ ref: huge, reason: huge }], checks: [{ file: huge, reason: huge }], extra: huge, }], files: [huge], extra: huge, }, entries: [{ id: 'bounded', element: { ref: huge, tagName: huge, id: huge, classes: [huge], textContent: huge }, ops: [{ originalText: 'Old', newText: 'New', sourceHint: { file: 'src/App.jsx', line: null, column: null }, }], }], candidates: [{ entryId: 'bounded', ref: huge, sourceHint: { file: huge, line: null, extra: huge }, textMatches: [{ file: huge, reason: huge, extra: huge }], extra: huge, }], }); const serializedBatch = prompt.split('Staged copy-edit batch:\n').pop(); const compact = JSON.parse(serializedBatch); assert.ok(prompt.length < 25_000, `expected compact prompt, got ${prompt.length} characters`); assert.deepEqual(Object.keys(compact.repair).sort(), [ 'failures', 'files', 'attempt', 'maxAttempts', 'pageUrl', 'reason', 'status', 'transactionId', ].sort()); assert.equal(compact.repair.attempt, 2); assert.equal(compact.repair.reason, 'source_verification_failed'); assert.ok(compact.repair.transactionId.length < 400); assert.ok(compact.repair.failures[0].message.length < 400); assert.equal(compact.repair.failures[0].entryId, 'bounded'); assert.ok(compact.repair.failures[0].candidates[0].file.length < 400); assert.ok(compact.repair.failures[0].failures[0].ref.length < 400); assert.ok(compact.repair.failures[0].checks[0].file.length < 400); assert.deepEqual(Object.keys(compact.candidates[0]).sort(), [ 'entryId', 'ref', 'sourceHint', 'textMatches', ]); assert.ok(compact.candidates[0].ref.length < 400); assert.ok(compact.candidates[0].sourceHint.file.length < 400); assert.ok(compact.entries[0].element.ref.length < 400); assert.ok(compact.entries[0].element.classes[0].length < 400); assert.equal(compact.entries[0].ops[0].sourceHint.line, null); assert.equal(compact.entries[0].ops[0].sourceHint.column, null); }); it('parses partial batch results', () => { assert.deepEqual( parseCopyEditBatchResult('{"status":"partial","appliedEntryIds":["a"],"failed":[{"entryId":"b","reason":"ambiguous"}],"files":["src/page.js"]}'), { status: 'partial', message: null, appliedEntryIds: ['a'], failed: [{ entryId: 'b', reason: 'ambiguous', candidates: [] }], files: ['src/page.js'], notes: [], warnings: [], }, ); }); it('preserves structured warnings in batch results', () => { assert.deepEqual( parseCopyEditBatchResult('{"status":"done","appliedEntryIds":["a"],"files":["src/page.js"],"warnings":[{"reason":"repair_followup","file":"src/page.js"},"plain warning"]}'), { status: 'done', message: null, appliedEntryIds: ['a'], failed: [], files: ['src/page.js'], notes: [], warnings: [ { reason: 'repair_followup', file: 'src/page.js' }, { message: 'plain warning' }, ], }, ); }); it('flags invalid JS and actual leftover carbonize markers in post-apply checks', () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-agent-checks-')); try { fs.mkdirSync(path.join(tmp, 'src'), { recursive: true }); fs.writeFileSync(path.join(tmp, 'src', 'bad.js'), 'const value = ;\n'); fs.writeFileSync(path.join(tmp, 'src', 'page.html'), '\n

Hi

\n'); const checks = runCopyEditPostApplyChecks({ cwd: tmp, files: ['src/bad.js', 'src/page.html'] }); assert.equal(checks.ok, false); assert.equal(checks.failures.some((item) => item.reason === 'leftover_impeccable_marker'), true); assert.equal(checks.failures.some((item) => item.reason === 'invalid_js'), true); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } }); it('flags invalid JSX syntax in post-apply checks', () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-agent-jsx-checks-')); try { fs.mkdirSync(path.join(tmp, 'src'), { recursive: true }); fs.writeFileSync(path.join(tmp, 'src', 'App.jsx'), 'export default function App() { return

Broken

; }\n'); const checks = runCopyEditPostApplyChecks({ cwd: tmp, files: ['src/App.jsx'] }); assert.equal(checks.ok, false); assert.equal(checks.failures.some((item) => item.reason === 'invalid_source_syntax'), true); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } }); it('flags invalid JSON in post-apply checks', () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-agent-json-checks-')); try { fs.mkdirSync(path.join(tmp, 'src'), { recursive: true }); fs.writeFileSync(path.join(tmp, 'src', 'data.json'), '{"title":"New",}\n'); const checks = runCopyEditPostApplyChecks({ cwd: tmp, files: ['src/data.json'] }); assert.equal(checks.ok, false); assert.equal(checks.failures.some((item) => item.reason === 'invalid_json'), true); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } }); it('does not flag live-mode marker words when they only appear as source literals', () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-agent-literals-')); try { fs.mkdirSync(path.join(tmp, 'src'), { recursive: true }); fs.writeFileSync( path.join(tmp, 'src', 'live-helper.js'), [ 'const words = "impeccable-carbonize- data-impeccable-variant IMPECCABLE_VARIANT impeccable-live-variant";', 'const selector = "[data-impeccable-variant=\\"1\\"]";', 'const example = "
demo
";', 'export { words, selector, example };', '', ].join('\n'), ); const checks = runCopyEditPostApplyChecks({ cwd: tmp, files: ['src/live-helper.js'] }); assert.equal(checks.ok, true); assert.deepEqual(checks.failures, []); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } }); it('respects off mode before trying local AI commands', () => { assert.equal(chooseCopyEditAgent({ env: { IMPECCABLE_LIVE_COPY_AGENT: 'off' } }), null); assert.equal(chooseCopyEditAgent({ env: { IMPECCABLE_LIVE_COPY_AGENT: 'false' } }), null); assert.equal(chooseCopyEditAgent({ env: { IMPECCABLE_LIVE_COPY_AGENT: 'mock' } }), 'mock'); }); it('surfaces Claude CLI auth errors from is_error JSON output', () => { const output = '{"type":"result","subtype":"success","is_error":true,"duration_ms":36,' + '"result":"Not logged in ยท Please run /login","stop_reason":"stop_sequence","session_id":"abc"}'; const hint = extractRunnerErrorMessage(output, 'claude'); assert.ok(hint, 'expected extractRunnerErrorMessage to return a hint'); assert.match(hint, /claude CLI:/); assert.match(hint, /Not logged in/); }); it('falls back to last non-empty line when no JSON is recognizable', () => { const hint = extractRunnerErrorMessage('warm-up...\nsome noise\nfatal: provider unreachable\n', 'codex'); assert.equal(hint, 'codex: fatal: provider unreachable'); }); it('returns null when there is nothing useful to surface', () => { assert.equal(extractRunnerErrorMessage('', 'claude'), null); assert.equal(extractRunnerErrorMessage(' \n \n', 'claude'), null); }); it('describeNoProviderError pinpoints which CLI needs which action', () => { const claudeInstalledUnauthed = describeNoProviderError({ exists: (cmd) => cmd === 'claude', authed: () => false, env: {}, }); assert.match(claudeInstalledUnauthed, /Claude CLI: installed but not selected/); assert.match(claudeInstalledUnauthed, /subprocess may be unable to read/); assert.match(claudeInstalledUnauthed, /claude setup-token/); assert.match(claudeInstalledUnauthed, /CLAUDE_CODE_OAUTH_TOKEN/); assert.match(claudeInstalledUnauthed, /ANTHROPIC_API_KEY/); assert.match(claudeInstalledUnauthed, /Codex CLI: not installed/); const tokenSetButInvalid = describeNoProviderError({ exists: (cmd) => cmd === 'claude', authed: () => false, env: { CLAUDE_CODE_OAUTH_TOKEN: 'sk-ant-oat01-broken' }, }); assert.match(tokenSetButInvalid, /CLAUDE_CODE_OAUTH_TOKEN is set but the CLI still rejected it/); const nothingInstalled = describeNoProviderError({ exists: () => false, authed: () => false, env: {} }); assert.match(nothingInstalled, /Claude CLI: not installed/); assert.match(nothingInstalled, /Codex CLI: not installed/); assert.match(nothingInstalled, /IMPECCABLE_LIVE_COPY_AGENT=mock/); const codexInstalled = describeNoProviderError({ exists: (cmd) => cmd === 'codex', authed: () => false, env: {}, }); assert.match(codexInstalled, /Codex CLI: installed/); assert.match(codexInstalled, /codex login/); }); it('auto mode picks the first authenticated provider via injected authCheck', () => { const claudeUnauthedCodexAuthed = (cmd) => cmd === 'codex'; assert.equal( chooseCopyEditAgent({ env: { IMPECCABLE_LIVE_COPY_AGENT: 'auto' }, authCheck: claudeUnauthedCodexAuthed }), 'codex', ); const onlyClaudeAuthed = (cmd) => cmd === 'claude'; assert.equal( chooseCopyEditAgent({ env: { IMPECCABLE_LIVE_COPY_AGENT: 'auto' }, authCheck: onlyClaudeAuthed }), 'claude', ); const noneAuthed = () => false; assert.equal( chooseCopyEditAgent({ env: {}, authCheck: noneAuthed }), null, ); }); it('auto mode falls back to chat when no CLI is authenticated and chat is available', () => { const noneAuthed = () => false; assert.equal( chooseCopyEditAgent({ env: { IMPECCABLE_LIVE_COPY_AGENT: 'auto' }, authCheck: noneAuthed, chatAvailable: () => true, }), 'chat', ); assert.equal( chooseCopyEditAgent({ env: { IMPECCABLE_LIVE_COPY_AGENT: 'auto' }, authCheck: noneAuthed, chatAvailable: () => false, }), null, ); // Explicit chat mode honors chatAvailable. assert.equal( chooseCopyEditAgent({ env: { IMPECCABLE_LIVE_COPY_AGENT: 'chat' }, chatAvailable: () => true, }), 'chat', ); assert.equal( chooseCopyEditAgent({ env: { IMPECCABLE_LIVE_COPY_AGENT: 'chat' }, chatAvailable: () => false, }), null, ); // CLI providers still preferred when authenticated. assert.equal( chooseCopyEditAgent({ env: { IMPECCABLE_LIVE_COPY_AGENT: 'auto' }, authCheck: (cmd) => cmd === 'codex', chatAvailable: () => true, }), 'codex', ); }); it('runCopyEditBatchAgent with provider=chat delegates to applyBatchToSource', async () => { const batch = { pageUrl: '/', entries: [{ id: 'a1b2c3d4', pageUrl: '/', element: {}, ops: [] }], candidates: [], }; let receivedBatch = null; const fakeApply = async (b) => { receivedBatch = b; return { status: 'done', appliedEntryIds: ['a1b2c3d4'], failed: [], files: ['site/pages/index.astro'], notes: ['ok'], }; }; const result = await runCopyEditBatchAgent(batch, { provider: 'chat', applyBatchToSource: fakeApply, }); assert.equal(receivedBatch, batch); assert.equal(result.status, 'done'); assert.deepEqual(result.appliedEntryIds, ['a1b2c3d4']); assert.deepEqual(result.files, ['site/pages/index.astro']); assert.deepEqual(result.notes, ['ok']); }); it('runCopyEditBatchAgent with provider=chat rejects when no callback is supplied', async () => { await assert.rejects( runCopyEditBatchAgent({ entries: [], candidates: [] }, { provider: 'chat' }), /chat provider requires applyBatchToSource/, ); }); it('passes large Claude prompts on stdin instead of argv', async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-agent-claude-stdin-')); try { const fakeClaude = path.join(tmp, 'claude'); fs.writeFileSync(fakeClaude, [ '#!/usr/bin/env node', "let input = '';", "process.stdin.setEncoding('utf8');", "process.stdin.on('data', (chunk) => { input += chunk; });", "process.stdin.on('end', () => {", " const promptLeakedToArgv = process.argv.slice(2).some((arg) => arg.includes('large-prompt-sentinel'));", " if (!input.includes('large-prompt-sentinel') || promptLeakedToArgv) process.exit(2);", " process.stdout.write(JSON.stringify({ status: 'done', appliedEntryIds: ['large'], files: [], notes: [] }));", '});', '', ].join('\n')); fs.chmodSync(fakeClaude, 0o755); const result = await runCopyEditBatchAgent({ pageUrl: '/', entries: [{ id: 'large', pageUrl: '/', ops: [{ originalText: 'Old', newText: `large-prompt-sentinel${'x'.repeat(1_100_000)}` }], }], }, { provider: 'claude', outDir: path.join(tmp, 'out'), timeoutMs: 5_000, env: { ...process.env, PATH: `${tmp}${path.delimiter}${process.env.PATH || ''}`, }, }); assert.equal(result.status, 'done'); assert.deepEqual(result.appliedEntryIds, ['large']); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } }); it('describeNoProviderError mentions starting impeccable live when chat is the missing piece', () => { const noChatPolling = describeNoProviderError({ exists: () => false, authed: () => false, chatAvailable: () => false, env: {}, }); assert.match(noChatPolling, /Chat: no Impeccable live session is currently polling/); assert.match(noChatPolling, /Start Impeccable live/); }); });