pbakaus_impeccable/scripts/run-tests.mjs

#!/usr/bin/env node
import { spawn } from 'node:child_process';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { DEFAULT_SUITES, OPT_IN_SUITES, SUITES, expandSuites } from './test-suites.mjs';
import { createGroupShutdown, trackChildExit } from './lib/process-group.mjs';
import {
  REPO_ENV,
  REPO_PATH_ENV,
  RUN_ID_ENV,
  alive,
  findLiveServers,
  killLiveServers,
  makeRunId,
  repoMarker,
} from './lib/live-server-processes.mjs';

const REPO_ROOT = path.resolve(fileURLToPath(new URL('..', import.meta.url)));

// Global wall-clock backstop for any one command. Even with per-test timeouts
// and client-side network deadlines in place, a wedged tool or an orphaned
// grandchild can keep a runner alive forever; this cap guarantees the sweep
// terminates. Per-suite `wallClockMs` overrides it; the env var overrides both.
const DEFAULT_WALL_CLOCK_MS = Number(process.env.IMPECCABLE_TEST_WALL_CLOCK_MS) || 1_200_000;

const args = process.argv.slice(2);

if (args.includes('--help') || args.includes('-h')) {
  printHelp();
  process.exit(0);
}

if (args.includes('--list')) {
  printSuites();
  process.exit(0);
}

if (args.includes('--cleanup')) {
  process.exit(cleanupRepoServers());
}

/**
 * Suite commands run in their own process group, which buys two things: the
 * wall-clock cap can SIGKILL a wedged tree whole (the runner, its per-file
 * `node --test` workers, and any grandchildren or browsers they left open),
 * and a Ctrl-C can end that same tree deterministically instead of orphaning
 * it. Nothing in this file may use spawnSync: a blocked event loop cannot run
 * the signal handlers that make either guarantee, and cannot reap the child it
 * is waiting on either.
 */
const shutdown = createGroupShutdown();

for (const sig of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
  process.on(sig, () => { void shutdown.onSignal(exitCodeForSignal(sig)); });
}
// Nothing can be awaited here, so this is the one path that does not wait.
process.on('exit', () => shutdown.onExit());

/** The shell convention for "killed by signal N": 130 SIGINT, 143 SIGTERM, 129 SIGHUP. */
function exitCodeForSignal(sig) {
  return 128 + (os.constants.signals[sig] ?? 0);
}

const requestedSuites = args.filter((arg) => !arg.startsWith('-'));
let suites;
try {
  suites = expandSuites(requestedSuites);
} catch (err) {
  console.error(err.message);
  process.exit(1);
}

await main();

async function main() {
  for (const suiteName of suites) {
    const suite = SUITES[suiteName];
    console.log(`\n## test:${suiteName}`);
    console.log(suite.description);
    for (const command of suite.commands) {
      await runCommand(command, suiteName);
    }
  }
}

async function runCommand(command, suiteName) {
  const runId = makeRunId(REPO_ROOT);
  const env = {
    ...process.env,
    [RUN_ID_ENV]: runId,
    // The hash is what matching uses; the path rides along for a human reading
    // `ps -E` output and is never matched on.
    [REPO_ENV]: repoMarker(REPO_ROOT),
    [REPO_PATH_ENV]: REPO_ROOT,
    ...(command.env || {}),
  };
  const wallClockMs = command.wallClockMs ?? DEFAULT_WALL_CLOCK_MS;

  if (command.runner === 'bun') {
    await runProcess('bun', ['test', ...command.files], { env, wallClockMs });
  } else if (command.runner === 'node') {
    // One invocation for the whole file list: node --test runs each file in
    // its own child process regardless, so isolation is unchanged, but the
    // runner-per-file spawn overhead is gone and files execute concurrently.
    // Measured on the live suite (38 files): 52s serial-per-file vs 18s
    // batched at concurrency 4. Suites can pin `concurrency: 1` if their
    // tests ever contend for a shared resource.
    const nodeArgs = ['--test', `--test-concurrency=${command.concurrency ?? 4}`];
    if (command.timeoutMs) nodeArgs.push(`--test-timeout=${command.timeoutMs}`);
    if (command.forceExit) nodeArgs.push('--test-force-exit');
    nodeArgs.push(...command.files);
    await runProcess(process.execPath, nodeArgs, { env, wallClockMs });
  } else {
    throw new Error(`Unsupported test runner "${command.runner}"`);
  }

  await assertNoLeakedServers(runId, suiteName);
}

function runProcess(cmd, args, { env, wallClockMs }) {
  console.log(`$ ${formatCommand(cmd, args)}`);
  return new Promise((resolve) => {
    const child = spawn(cmd, args, {
      // Own process group: the wall-clock cap and the shutdown handler can
      // then take down the runner, every test file it forked, and anything
      // those forked, in one signal.
      detached: true,
      // stdin is deliberately not inherited. A detached child is a background
      // process group on the terminal, and a background read of the tty stops
      // the process with SIGTTIN. No suite reads the runner's stdin.
      stdio: ['ignore', 'inherit', 'inherit'],
      env,
    });
    // Registered before the handlers below, so `hasExited` is already set by
    // the time they run and a shutdown mid-exit does not signal a dead pid.
    const running = shutdown.track(trackChildExit(child));

    let timedOut = false;
    const timer = wallClockMs
      ? setTimeout(() => {
          timedOut = true;
          console.error(
            `\n[run-tests] wall-clock cap of ${wallClockMs}ms exceeded for "${formatCommand(cmd, args)}"; ` +
            'killing the process group (SIGKILL).',
          );
          // A test blocked in a synchronous spawnSync cannot be reached by
          // node's --test-timeout, so this is the guaranteed end of the tree.
          // No graceful phase: the cap has already been generous.
          try { process.kill(-running.child.pid, 'SIGKILL'); }
          catch { try { running.child.kill('SIGKILL'); } catch { /* already gone */ } }
        }, wallClockMs)
      : null;

    child.on('error', (err) => {
      if (timer) clearTimeout(timer);
      shutdown.release();
      console.error(err.message);
      process.exit(1);
    });
    child.on('exit', (code, signal) => {
      if (timer) clearTimeout(timer);
      shutdown.release();
      if (shutdown.shuttingDown) return;
      if (timedOut) {
        // A wedged suite is one of the ways servers are left behind, so sweep
        // before reporting rather than walking away from them.
        assertNoLeakedServers(env[RUN_ID_ENV], null).finally(() => process.exit(1));
        return;
      }
      if (signal) {
        console.error(`[run-tests] "${formatCommand(cmd, args)}" killed by signal ${signal}`);
        assertNoLeakedServers(env[RUN_ID_ENV], null).finally(() => process.exit(1));
        return;
      }
      if (code !== 0) {
        // Leaked servers are still worth reporting on a failing suite: a
        // failure before teardown is one of the ways they are left behind.
        assertNoLeakedServers(env[RUN_ID_ENV], null).finally(() => process.exit(code || 1));
        return;
      }
      resolve();
    });
  });
}

/**
 * Fail the run when a suite left live servers behind.
 *
 * The whole point of the guard is that a leak shows up in the run that caused
 * it rather than as a wedged port days later, so it is an error, not a warning.
 * The leaked servers are killed either way, so the next suite still gets its
 * ports.
 */
async function assertNoLeakedServers(runId, suiteName) {
  if (!runId || process.env.IMPECCABLE_SKIP_LEAK_CHECK === '1') return;
  // A server asked to stop needs a moment to actually go.
  let leaked = [];
  for (let attempt = 0; attempt < 10; attempt += 1) {
    leaked = findLiveServers({ runId });
    if (!leaked.length) return;
    await new Promise((r) => setTimeout(r, 200));
  }

  killLiveServers(leaked);
  const label = suiteName ? `test:${suiteName}` : 'the suite';
  console.error(`\nLeaked live servers: ${label} left ${leaked.length} live server process(es) running.`);
  for (const { pid, command } of leaked) console.error(`  pid ${pid}  ${command}`);
  console.error('They have been killed. A live server outliving its suite means a teardown path');
  console.error('was skipped; see tests/lib/live-servers.mjs for how servers are meant to be tracked.');
  console.error('Set IMPECCABLE_SKIP_LEAK_CHECK=1 to bypass this check.');
  process.exit(1);
}

/**
 * `bun run test:cleanup`: kill live servers this checkout's tests left behind.
 *
 * Scoped to servers carrying this checkout's `IMPECCABLE_TEST_REPO` marker, so
 * a live session the developer started themselves in this same repo is not a
 * candidate. A server from a run that predates the marker is not found here and
 * has to be killed by hand.
 */
function cleanupRepoServers() {
  const leaked = findLiveServers({ repo: REPO_ROOT });
  if (!leaked.length) {
    console.log('No leftover live servers from this repo\'s test runs.');
    return 0;
  }
  for (const { pid, command } of leaked) console.log(`killing pid ${pid}  ${command}`);
  killLiveServers(leaked);
  const survivors = leaked.filter(({ pid }) => alive(pid));
  if (survivors.length) {
    console.error(`Could not kill ${survivors.length} process(es): ${survivors.map((p) => p.pid).join(', ')}`);
    return 1;
  }
  console.log(`Killed ${leaked.length} leftover live server process(es).`);
  return 0;
}

function formatCommand(cmd, args) {
  const bin = cmd === process.execPath ? 'node' : cmd;
  return [bin, ...args].join(' ');
}

function printHelp() {
  console.log(`Usage: node scripts/run-tests.mjs [suite...]

Aliases:
  default     ${DEFAULT_SUITES.join(', ')}
  all-local   ${DEFAULT_SUITES.join(', ')}
  all         ${[...DEFAULT_SUITES, ...OPT_IN_SUITES].join(', ')}

Run with --list to see suite contents.
Run with --cleanup to kill live servers a previous run left behind.`);
}

function printSuites() {
  for (const [name, suite] of Object.entries(SUITES)) {
    const marker = suite.optIn ? ' (opt-in)' : '';
    console.log(`\n${name}${marker}`);
    console.log(`  ${suite.description}`);
    for (const command of suite.commands) {
      console.log(`  ${command.runner}:`);
      for (const file of command.files) console.log(`    ${file}`);
    }
  }
}