[codex] Improve CI test coverage (#212)

* Improve CI test coverage

* Stabilize live E2E harness

* Shard live E2E CI

* Cache live E2E CI dependencies

* Stabilize live E2E smoke CI

* Update generated live browser bundles

* Tighten live E2E smoke runtime

* Prevent live E2E smoke hangs

* Stabilize live E2E CI coverage

* Fix stale accept DOM cleanup

* Regenerate live browser outputs

.github/skills/impeccable/scripts/live-browser.js

@@ -4503,15 +4503,17 @@
     if (origContent.id) {
       liveEl = document.getElementById(origContent.id);
     } else if (cls) {
-      const candidates = document.querySelectorAll(tag + '.' + cls.split(' ')[0]);
+      const candidates = [...document.getElementsByTagName(tag)];
       for (const c of candidates) {
         if (c.className === cls && !own(c)) { liveEl = c; break; }
       }
       if (!liveEl) {
-        const expectedClasses = String(cls).split(/\s+/).filter(Boolean);
-        for (const c of candidates) {
-          if (own(c)) continue;
-          if (expectedClasses.every((name) => c.classList.contains(name))) { liveEl = c; break; }
+        const expectedClasses = String(cls).split(/\s+/).filter((name) => /^[A-Za-z_-][\w-]*$/.test(name));
+        if (expectedClasses.length > 0) {
+          for (const c of candidates) {
+            if (own(c)) continue;
+            if (expectedClasses.every((name) => c.classList.contains(name))) { liveEl = c; break; }
+          }
         }
       }
     }
@@ -4880,7 +4882,7 @@
         const block = startIdx !== -1 && endIdx !== -1 && endIdx > startIdx
           ? html.slice(startIdx + startMark.length, endIdx).trim()
           : html;
-        const doc = parser.parseFromString(block, 'text/html');
+        const doc = parser.parseFromString(normalizeSourceFallbackBlock(block, filePath), 'text/html');
         srcWrapper = doc.querySelector('[data-impeccable-variants="' + sessionId + '"]');
         if (!srcWrapper) {
           console.warn('[impeccable] Variant wrapper not found in source file.');
@@ -4949,6 +4951,44 @@
       });
   }
 
+  function normalizeSourceFallbackBlock(block, filePath) {
+    if (!/\.[cm]?[jt]sx$/i.test(String(filePath || ''))) return block;
+    return String(block)
+      .replace(
+        /<style\b([^>]*)>\s*\{\s*`([\s\S]*?)`\s*\}\s*<\/style>/g,
+        (_match, attrs, css) => '<style' + attrs + '>' + css + '</style>',
+      )
+      .replace(/\bclassName\s*=\s*\{\s*`([^`]*?)`\s*\}/g, (_match, value) => {
+        const literalClasses = value.replace(/\$\{[^}]*\}/g, ' ').replace(/\s+/g, ' ').trim();
+        return literalClasses ? 'class="' + escapeHtml(literalClasses) + '"' : '';
+      })
+      .replace(/\bclassName\s*=/g, 'class=')
+      .replace(/\sstyle=\{\{([\s\S]*?)\}\}/g, (_match, body) => {
+        const css = jsxStyleObjectToCss(body);
+        return css ? ' style="' + escapeHtml(css) + '"' : '';
+      });
+  }
+
+  function jsxStyleObjectToCss(body) {
+    const declarations = [];
+    const re = /(["'][^"']+["']|[A-Za-z_$][\w$-]*)\s*:\s*(?:"([^"]*)"|'([^']*)'|(-?\d+(?:\.\d+)?))/g;
+    let match;
+    while ((match = re.exec(String(body || '')))) {
+      const prop = jsxStylePropToCss(match[1]);
+      const value = match[2] ?? match[3] ?? match[4] ?? '';
+      if (!prop || value === '') continue;
+      declarations.push(prop + ': ' + value);
+    }
+    return declarations.join('; ');
+  }
+
+  function jsxStylePropToCss(prop) {
+    let out = String(prop || '').trim().replace(/^["']|["']$/g, '');
+    if (!out) return '';
+    if (out.startsWith('--')) return out;
+    return out.replace(/[A-Z]/g, (ch) => '-' + ch.toLowerCase()).replace(/^-ms-/, '-ms-');
+  }
+
   function buildSvelteExpressionTextMap(sourceOriginal, liveOriginal) {
     const map = new Map();
     if (!sourceOriginal || !liveOriginal) return map;
@@ -5422,7 +5462,11 @@
           }
           // Source fallback when HMR did not land variants in this tab.
           if (msg.file && msg.id && state === 'GENERATING' && msg.id === currentSessionId) {
-            injectVariantsFromSource(msg.file, msg.id);
+            setTimeout(() => {
+              if (arrivedVariants >= expectedVariants && expectedVariants > 0) return;
+              if (state !== 'GENERATING' || msg.id !== currentSessionId) return;
+              injectVariantsFromSource(msg.file, msg.id);
+            }, 750);
             break;
           }
           // Variants are in source but not in the DOM yet. Common when the
@@ -5514,8 +5558,11 @@
   function sendEvent(msg, opts) {
     msg.token = TOKEN;
     function handleFailure(err) {
-      console.error('[impeccable] Failed to send event:', err);
-      if (opts && opts.throwOnError) throw err;
+      if (opts && opts.throwOnError) {
+        console.error('[impeccable] Failed to send event:', err);
+        throw err;
+      }
+      console.debug('[impeccable] Dropped optional live event:', err);
       return null;
     }
     return fetch('http://localhost:' + PORT + '/events', {
@@ -6843,12 +6890,16 @@ void main() {
   function ensureAcceptedDomClean(pending) {
     const sessionId = pending?.id;
     const variantId = pending?.variant;
-    const wrapper = document.querySelector('[data-impeccable-variants="' + sessionId + '"]');
+    const wrapper = findAcceptedRuntimeWrapper(sessionId);
     const accepted = wrapper?.querySelector?.('[data-impeccable-variant="' + variantId + '"]');
     if (!wrapper) {
       restoreAcceptedDomFromSnapshot(pending);
       return;
     }
+    if (acceptedDomAlreadyClean(pending)) {
+      wrapper.remove();
+      return;
+    }
     if (!accepted) {
       wrapper.remove();
       restoreAcceptedDomFromSnapshot(pending);
@@ -6862,6 +6913,12 @@ void main() {
     wrapper.remove();
   }
 
+  function findAcceptedRuntimeWrapper(sessionId) {
+    if (!sessionId) return null;
+    return document.querySelector('[data-impeccable-variants="' + sessionId + '"]')
+      || document.querySelector('[data-impeccable-carbonize="' + sessionId + '"]');
+  }
+
   function restoreAcceptedDomFromSnapshot(pending) {
     if (acceptedDomAlreadyClean(pending)) return;
     if (!pending?.acceptedHtml) {

.github/workflows/ci.yml

@@ -5,15 +5,49 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  test:
+  changes:
     runs-on: ubuntu-latest
-    
+    outputs:
+      core: ${{ steps.plan.outputs.core }}
+      detector: ${{ steps.plan.outputs.detector }}
+      live: ${{ steps.plan.outputs.live }}
+      framework: ${{ steps.plan.outputs.framework }}
+      cli_remote_e2e: ${{ steps.plan.outputs.cli_remote_e2e }}
+      live_e2e: ${{ steps.plan.outputs.live_e2e }}
+      live_e2e_accept_cleanup: ${{ steps.plan.outputs.live_e2e_accept_cleanup }}
+      skill_behavior: ${{ steps.plan.outputs.skill_behavior }}
+      live_svelte_adapter_deepseek: ${{ steps.plan.outputs.live_svelte_adapter_deepseek }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
-        
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Detect test plan
+        id: plan
+        env:
+          GITHUB_EVENT_BEFORE: ${{ github.event.before }}
+        run: node scripts/ci-test-plan.mjs
+
+  test:
+    runs-on: ubuntu-latest
+    needs: changes
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
       - name: Setup Node
         uses: actions/setup-node@v6
         with:
@@ -27,18 +61,358 @@ jobs:
       - name: Install dependencies
         run: bun install
 
+      - name: Run core tests
+        run: bun run test:core
+
       - name: Install Puppeteer browser
+        if: needs.changes.outputs.detector == 'true'
         run: bunx puppeteer browsers install chrome
 
-      - name: Run tests
-        run: bun run test
-        
+      - name: Run detector tests
+        if: needs.changes.outputs.detector == 'true'
+        run: bun run test:detector
+
+      - name: Run live unit tests
+        if: needs.changes.outputs.live == 'true'
+        run: bun run test:live
+
+      - name: Run framework fixture tests
+        if: needs.changes.outputs.framework == 'true'
+        run: bun run test:framework
+
+      - name: Rebuild browser detector
+        if: needs.changes.outputs.detector == 'true'
+        run: bun run build:browser
+
+      - name: Rebuild extension detector
+        if: needs.changes.outputs.detector == 'true'
+        run: bun run build:extension
+
       - name: Build
         run: bun run build
-        
+
+      - name: Verify generated tracked outputs
+        run: git diff --exit-code -- .agents .claude .cursor .gemini .github/skills plugin cli/engine/detect-antipatterns-browser.js
+
       - name: Upload build artifacts
         uses: actions/upload-artifact@v7
         with:
           name: impeccable-dist
           path: dist/
           retention-days: 7
+
+  cli-remote-e2e:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.cli_remote_e2e == 'true'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Run remote CLI E2E smoke
+        run: bun run test:cli-remote-e2e
+
+  live-e2e-smoke:
+    name: live-e2e smoke (${{ matrix.group }})
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.live_e2e == 'true' && github.event_name != 'workflow_dispatch'
+    timeout-minutes: 15
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - group: platform
+            fixtures: astro-vite7,nextjs-app-router,vite8-sveltekit
+          - group: react
+            fixtures: vite8-react-css-modules,vite8-react-insert,vite8-react-plain
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Cache fixture npm downloads
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-fixture-npm-${{ hashFiles('tests/framework-fixtures/**/files/package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-fixture-npm-
+
+      - name: Cache Playwright Chromium
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('package.json', 'bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-playwright-chromium-
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Install Playwright Chromium
+        run: npx playwright install chromium
+
+      - name: Run live E2E tests
+        run: bun run test:live-e2e
+        env:
+          IMPECCABLE_E2E_ONLY: ${{ matrix.fixtures }}
+          IMPECCABLE_E2E_SCENARIOS: core
+          IMPECCABLE_E2E_TEST_TIMEOUT_MS: 180000
+          IMPECCABLE_E2E_INSTALL_TIMEOUT_MS: 120000
+          IMPECCABLE_E2E_DEV_READY_TIMEOUT_MS: 60000
+          IMPECCABLE_E2E_ARTIFACT_DIR: test-results/live-e2e/${{ matrix.group }}
+
+      - name: Upload live E2E failure artifacts
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: live-e2e-smoke-${{ matrix.group }}-artifacts
+          path: test-results/live-e2e
+          if-no-files-found: ignore
+
+  live-e2e-full:
+    name: live-e2e full (${{ matrix.group }})
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.live_e2e == 'true' && github.event_name == 'workflow_dispatch'
+    timeout-minutes: 25
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - group: platform
+            fixtures: astro-vite7,nextjs-app-router,vite8-sveltekit
+          - group: react-a
+            fixtures: vite8-https,vite8-react-base-path,vite8-react-csp-meta,vite8-react-css-modules,vite8-react-emotion
+          - group: react-b
+            fixtures: vite8-react-insert,vite8-react-mapped-list,vite8-react-modal,vite8-react-plain
+          - group: stateful
+            fixtures: vite8-react-radix-dialog,vite8-react-router-spa,vite8-react-styled-components,vite8-react-tabs
+          - group: styling
+            fixtures: vite8-react-tailwindv3,vite8-react-tailwindv4,vite8-react-ts,vite8-react-tsx-repeated-aside,vite8-react-unocss,vite8-react-vanilla-extract
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Cache fixture npm downloads
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-fixture-npm-${{ hashFiles('tests/framework-fixtures/**/files/package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-fixture-npm-
+
+      - name: Cache Playwright Chromium
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('package.json', 'bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-playwright-chromium-
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Install Playwright Chromium
+        run: npx playwright install chromium
+
+      - name: Run live E2E tests
+        run: bun run test:live-e2e
+        env:
+          IMPECCABLE_E2E_ONLY: ${{ matrix.fixtures }}
+          IMPECCABLE_E2E_TEST_TIMEOUT_MS: 300000
+          IMPECCABLE_E2E_INSTALL_TIMEOUT_MS: 180000
+          IMPECCABLE_E2E_DEV_READY_TIMEOUT_MS: 120000
+          IMPECCABLE_E2E_ARTIFACT_DIR: test-results/live-e2e/${{ matrix.group }}
+
+      - name: Upload live E2E failure artifacts
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: live-e2e-full-${{ matrix.group }}-artifacts
+          path: test-results/live-e2e
+          if-no-files-found: ignore
+
+  live-e2e-accept-cleanup:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.live_e2e_accept_cleanup == 'true'
+    timeout-minutes: 15
+    env:
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+    steps:
+      - name: Skip without provider key
+        if: ${{ env.ANTHROPIC_API_KEY == '' && env.DEEPSEEK_API_KEY == '' }}
+        run: echo "Skipping provider-backed accept-cleanup regression because no provider API key is configured."
+
+      - name: Checkout repository
+        if: ${{ env.ANTHROPIC_API_KEY != '' || env.DEEPSEEK_API_KEY != '' }}
+        uses: actions/checkout@v6
+
+      - name: Setup Node
+        if: ${{ env.ANTHROPIC_API_KEY != '' || env.DEEPSEEK_API_KEY != '' }}
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Setup Bun
+        if: ${{ env.ANTHROPIC_API_KEY != '' || env.DEEPSEEK_API_KEY != '' }}
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Cache fixture npm downloads
+        if: ${{ env.ANTHROPIC_API_KEY != '' || env.DEEPSEEK_API_KEY != '' }}
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-fixture-npm-${{ hashFiles('tests/framework-fixtures/**/files/package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-fixture-npm-
+
+      - name: Cache Playwright Chromium
+        if: ${{ env.ANTHROPIC_API_KEY != '' || env.DEEPSEEK_API_KEY != '' }}
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('package.json', 'bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-playwright-chromium-
+
+      - name: Install dependencies
+        if: ${{ env.ANTHROPIC_API_KEY != '' || env.DEEPSEEK_API_KEY != '' }}
+        run: bun install
+
+      - name: Install Playwright Chromium
+        if: ${{ env.ANTHROPIC_API_KEY != '' || env.DEEPSEEK_API_KEY != '' }}
+        run: npx playwright install chromium
+
+      - name: Run accept cleanup regression
+        if: ${{ env.ANTHROPIC_API_KEY != '' || env.DEEPSEEK_API_KEY != '' }}
+        run: |
+          if [ -n "$DEEPSEEK_API_KEY" ]; then
+            export IMPECCABLE_E2E_LLM_PROVIDER=deepseek
+          else
+            export IMPECCABLE_E2E_LLM_PROVIDER=anthropic
+          fi
+          bun run test:live-e2e-accept-cleanup
+
+  live-svelte-adapter-deepseek:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.live_svelte_adapter_deepseek == 'true'
+    timeout-minutes: 25
+    env:
+      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+    steps:
+      - name: Skip without DeepSeek key
+        if: ${{ env.DEEPSEEK_API_KEY == '' }}
+        run: echo "Skipping Svelte adapter DeepSeek sweep because DEEPSEEK_API_KEY is not configured."
+
+      - name: Checkout repository
+        if: ${{ env.DEEPSEEK_API_KEY != '' }}
+        uses: actions/checkout@v6
+
+      - name: Setup Node
+        if: ${{ env.DEEPSEEK_API_KEY != '' }}
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Setup Bun
+        if: ${{ env.DEEPSEEK_API_KEY != '' }}
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Cache fixture npm downloads
+        if: ${{ env.DEEPSEEK_API_KEY != '' }}
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-fixture-npm-${{ hashFiles('tests/framework-fixtures/**/files/package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-fixture-npm-
+
+      - name: Cache Playwright Chromium
+        if: ${{ env.DEEPSEEK_API_KEY != '' }}
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-chromium-${{ hashFiles('package.json', 'bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-playwright-chromium-
+
+      - name: Install dependencies
+        if: ${{ env.DEEPSEEK_API_KEY != '' }}
+        run: bun install
+
+      - name: Install Playwright Chromium
+        if: ${{ env.DEEPSEEK_API_KEY != '' }}
+        run: npx playwright install chromium
+
+      - name: Run Svelte adapter DeepSeek sweep
+        if: ${{ env.DEEPSEEK_API_KEY != '' }}
+        run: bun run test:live-svelte-adapter-deepseek
+
+  skill-behavior:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.skill_behavior == 'true' && github.event_name != 'pull_request'
+    env:
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      GOOGLE_CLOUD_API_KEY: ${{ secrets.GOOGLE_CLOUD_API_KEY }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Run skill behavior tests
+        run: bun run test:skill-behavior