#!/usr/bin/env bun import * as fs from "node:fs/promises"; import * as path from "node:path"; type Mode = | "all" | "workspace" | "native" | "coding-agent-singleton" | "coding-agent-ui" | "coding-agent-runtime" | "coding-agent-native" | "coding-agent-heavy"; type CodingAgentBucket = "singleton" | "ui" | "runtime" | "native"; interface TestCommand { label: string; cwd: string; command: string[]; } type CodingAgentTestPartition = Record; const repoRoot = path.join(import.meta.dir, ".."); const args = process.argv.slice(2); const isDryRun = args.includes("--dry-run"); const requestedMode = args.find(arg => !arg.startsWith("--")) ?? "all"; const validModes = new Set([ "all", "workspace", "native", "coding-agent-singleton", "coding-agent-ui", "coding-agent-runtime", "coding-agent-native", "coding-agent-heavy", ]); // `chunkSize` splits a bucket's file list into that-many-file groups, each run as a // separate `bun --smol test` child process. A fresh process per chunk resets Bun's // heap and reaps any dangling spawned children between groups, keeping peak RSS // under the CI runner's OOM ceiling (a single 170–370-file invocation gets // SIGKILLed at 137). The singleton/global-state bucket is left whole: its suites // co-locate in one process to exercise process-wide state, so they must not split. const codingAgentBucketPlans: Record = { singleton: { label: "singleton/global-state bucket", parallel: 1 }, ui: { label: "UI/TUI bucket", parallel: 1, chunkSize: 10 }, runtime: { label: "runtime/session bucket", parallel: 1, chunkSize: 10 }, native: { label: "native/tooling/browser/unit bucket", parallel: 1, chunkSize: 10 }, }; // Smaller workspace packages stay separate from native/TUI/integration suites so // their short TS suites can run together. CI still downloads the Linux x64 native // addon before this bucket: shared utility barrels may load native-backed modules. // mnemopi is intentionally excluded — its embedding suites depend on a ~270MB // fastembed model absent from CI runners, so they flake/time out under the parallel // bucket; run `bun --cwd=packages/mnemopi test` locally instead. const fastWorkspacePackages = [ "packages/hashline", "packages/wire", "packages/utils", "packages/catalog", "packages/ai", "packages/snapcompact", "packages/agent", ]; // These suites cover the native package, TUI/browser-ish behavior, local servers, // or coding-agent-adjacent benchmark paths. Keep them low-concurrency and in jobs // that have downloaded the Linux x64 native addon artifacts. const nativeAndIntegrationPackages = [ "packages/natives", "packages/tui", "packages/collab-web", "packages/typescript-edit-benchmark", ]; const codingAgentNativePathPatterns = [ /(^|\/)[^/]*(bash|native|browser|cmux|mnemopi|hindsight|memory)[^/]*\.test\.ts$/i, /^test\/[^/]*(ask|gh|irc|task|eval|search|read|write|edit|ast|resolve|sqlite|web-search|fetch|image|ssh|tool)[^/]*\.test\.ts$/, /^test\/core\/python-[^/]*\.test\.ts$/, /^test\/core\/[^/]*executor[^/]*\.test\.ts$/, /^test\/tools\/[^/]*(ask|gh|irc|task|eval|search|read|edit|ast|resolve|sqlite|web-search|fetch|image|ssh)[^/]*\.test\.ts$/, /^test\/tools\/web-scrapers\//, /^test\/web\//, /^test\/ssh\//, /^test\/tools\.test\.ts$/, ]; const codingAgentSingletonPathPatterns = [ /^test\/(settings|config|fast-mode-scope|autocomplete-max-visible)[^/]*\.test\.ts$/, /^test\/[^/]*(singleton|global-state|fake-timer)[^/]*\.test\.ts$/, ]; const codingAgentUiPathPatterns = [ /^test\/modes\//, /^test\/(interactive-mode|main-interactive|input-controller|streaming|status-line|keybindings|editor|hook|theme|setup-wizard|job-renderer|tool-args-reveal|tool-execution)[^/]*\.test\.ts$/, /^src\/modes\/components\//, ]; const codingAgentRuntimePathPatterns = [ /^test\/agent-session[^/]*\.test\.ts$/, /^test\/(acp|mcp|rpc|sdk)[^/]*\.test\.ts$/, /^test\/(session|session-manager|task|collab|internal-urls)\//, /^test\/session[^/]*\.test\.ts$/, /^test\/session-manager[^/]*\.test\.ts$/, /^test\/(extensions?|plugin|autolearn|skills|marketplace|oauth)[^/]*\.test\.ts$/, /^test\/[^/]*oauth[^/]*\.test\.ts$/, /^test\/(extensibility|discovery|tool-discovery|goals|marketplace)\//, /^test\/(model|model-|model-registry|model-resolver|compaction)[^/]*\.test\.ts$/, ]; const codingAgentNativeContentMarkers = [ "@oh-my-pi/pi-natives", "pi-natives", "native", "readImageMetadata", "Bun.spawn", "Bun.spawnSync", "child_process", "Bun.serve", "new Worker", "Worker(", "puppeteer", "bun:sqlite", "Redis", "redis", "WebSocket", ]; const codingAgentSingletonContentMarkers = [ "Settings.init(", "Settings.instance", "resetSettingsForTest", "setAgentDir(", "vi.useFakeTimers(", "vi.useRealTimers(", "vi.stubEnv(", "vi.unstubAllEnvs(", ]; const codingAgentSingletonContentPatterns = [ /(^|[^\w$.])(process\.env|Bun\.env)\.[A-Za-z0-9_]+\s*=/, /(^|[^\w$.])(process\.env|Bun\.env)\[[^\]]+\]\s*=/, /delete\s+(process\.env|Bun\.env)(\.[A-Za-z0-9_]+|\[[^\]]+\])/, /Object\.assign\((process\.env|Bun\.env),/, ]; const codingAgentUiContentMarkers = [ "@oh-my-pi/pi-tui", "InteractiveMode", "InputController", "StatusLine", "ToolExecutionComponent", "render(", "renderToString", ]; const codingAgentRuntimeContentMarkers = [ "AgentSession", "SessionManager", "AuthStorage", "Bun.sleep", "setTimeout(", ]; let codingAgentTestPartitionPromise: Promise | null = null; function shellQuote(value: string): string { if (/^[A-Za-z0-9_./:=@+-]+$/.test(value)) { return value; } return `'${value.replaceAll("'", `'\\''`)}'`; } function workspaceTestCommand(pkg: string, parallel: number, smol = false): TestCommand { return { label: pkg, cwd: pkg, command: ["bun", ...(smol ? ["--smol"] : []), "test", `--parallel=${parallel}`], }; } async function collectTestsUnder(root: string, baseDir: string): Promise { const entries = await fs.readdir(root, { withFileTypes: true }); const files: string[] = []; for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { const filePath = path.join(root, entry.name); if (entry.isDirectory()) { files.push(...(await collectTestsUnder(filePath, baseDir))); continue; } if (!entry.isFile() || !entry.name.endsWith(".test.ts")) { continue; } files.push(path.relative(baseDir, filePath).split(path.sep).join("/")); } return files; } function hasAnyMarker(content: string, markers: string[]): boolean { return markers.some(marker => content.includes(marker)); } function matchesAnyPath(testFile: string, patterns: RegExp[]): boolean { return patterns.some(pattern => pattern.test(testFile)); } function matchesAnyContentPattern(content: string, patterns: RegExp[]): boolean { return patterns.some(pattern => pattern.test(content)); } // Native/tooling tests are classified first because they need the lowest // concurrency; all coding-agent buckets run with the native addon available in CI. function classifyCodingAgentTest(testFile: string, content: string): CodingAgentBucket { if ( matchesAnyPath(testFile, codingAgentNativePathPatterns) || hasAnyMarker(content, codingAgentNativeContentMarkers) ) { return "native"; } if ( matchesAnyPath(testFile, codingAgentUiPathPatterns) || hasAnyMarker(content, codingAgentUiContentMarkers) ) { return "ui"; } if ( matchesAnyPath(testFile, codingAgentSingletonPathPatterns) || hasAnyMarker(content, codingAgentSingletonContentMarkers) || matchesAnyContentPattern(content, codingAgentSingletonContentPatterns) ) { return "singleton"; } if ( matchesAnyPath(testFile, codingAgentRuntimePathPatterns) || hasAnyMarker(content, codingAgentRuntimeContentMarkers) ) { return "runtime"; } return "native"; } async function getCodingAgentTestPartition(): Promise { codingAgentTestPartitionPromise ??= (async () => { const codingAgentDir = path.join(repoRoot, "packages/coding-agent"); const testFiles = [ ...(await collectTestsUnder(path.join(codingAgentDir, "test"), codingAgentDir)), ...(await collectTestsUnder(path.join(codingAgentDir, "src"), codingAgentDir)), ].sort(); const partition: CodingAgentTestPartition = { singleton: [], ui: [], runtime: [], native: [], }; for (const testFile of testFiles) { const content = await Bun.file(path.join(codingAgentDir, testFile)).text(); partition[classifyCodingAgentTest(testFile, content)].push(testFile); } return partition; })(); return codingAgentTestPartitionPromise; } async function codingAgentTestCommands(bucket: CodingAgentBucket): Promise { const partition = await getCodingAgentTestPartition(); const testFiles = partition[bucket]; if (testFiles.length === 0) { throw new Error(`No coding-agent ${bucket} tests matched`); } const plan = codingAgentBucketPlans[bucket]; const chunkSize = plan.chunkSize ?? testFiles.length; const chunkCount = Math.ceil(testFiles.length / chunkSize); const commands: TestCommand[] = []; for (let i = 0; i < testFiles.length; i += chunkSize) { const chunk = testFiles.slice(i, i + chunkSize); const chunkLabel = chunkCount > 1 ? ` chunk ${commands.length + 1}/${chunkCount}` : ""; commands.push({ label: `packages/coding-agent (${plan.label}; ${testFiles.length} files; parallel=${plan.parallel}${chunkLabel}; ${chunk.length} files)`, cwd: "packages/coding-agent", command: ["bun", "--smol", "test", `--parallel=${plan.parallel}`, "--only-failures", ...chunk], }); } return commands; } async function commandsForMode(mode: Mode): Promise { switch (mode) { case "workspace": return [ ...fastWorkspacePackages.map(pkg => workspaceTestCommand(pkg, 8)), { label: "scripts", cwd: ".", command: ["bun", "test", "--parallel=4", "--only-failures", "scripts/ci-concurrency.test.ts"], }, ]; case "native": return nativeAndIntegrationPackages.map(pkg => workspaceTestCommand(pkg, 4, true)); case "coding-agent-singleton": return await codingAgentTestCommands("singleton"); case "coding-agent-ui": return await codingAgentTestCommands("ui"); case "coding-agent-runtime": return await codingAgentTestCommands("runtime"); case "coding-agent-native": return await codingAgentTestCommands("native"); case "coding-agent-heavy": return [ ...(await codingAgentTestCommands("singleton")), ...(await codingAgentTestCommands("ui")), ...(await codingAgentTestCommands("runtime")), ...(await codingAgentTestCommands("native")), ]; case "all": return [ ...(await commandsForMode("workspace")), ...(await commandsForMode("native")), ...(await commandsForMode("coding-agent-heavy")), ]; } } // The omp-kata runner pods inject sccache S3 credentials (`AWS_*`) and config // (`SCCACHE_*`) pod-wide via `envFrom`, GitHub Actions injects `GITHUB_TOKEN`, // and a host may carry provider API keys. Any of these make env-sensitive code // non-deterministic in tests — e.g. leaked AWS creds make `amazon-bedrock` look // authenticated and win the provider startup fallback over `anthropic`. Run the // suites in a hermetic environment with all credential / cloud-config variables // stripped so resolution depends only on the test's own fixtures. const SCRUBBED_ENV_PREFIXES = ["AWS_", "SCCACHE_", "GOOGLE_CLOUD_"]; const SCRUBBED_ENV_NAMES = new Set([ "RUSTC_WRAPPER", "GITHUB_TOKEN", "GH_TOKEN", "COPILOT_GITHUB_TOKEN", "GOOGLE_APPLICATION_CREDENTIALS", "ANTHROPIC_OAUTH_TOKEN", "XAI_OAUTH_TOKEN", ]); function isScrubbedEnvVar(key: string): boolean { if (SCRUBBED_ENV_NAMES.has(key)) { return true; } if (SCRUBBED_ENV_PREFIXES.some(prefix => key.startsWith(prefix))) { return true; } // Any provider credential, e.g. ANTHROPIC_API_KEY / XAI_OAUTH_TOKEN / bedrock bearer. return /_(API_KEY|OAUTH_TOKEN)$/.test(key) || key.includes("BEARER_TOKEN"); } async function runTestCommand(testCommand: TestCommand): Promise { const cwd = path.join(repoRoot, testCommand.cwd); const renderedCommand = testCommand.command.map(shellQuote).join(" "); console.log(`\n==> ${testCommand.label}`); console.log(`$ ${renderedCommand}`); if (isDryRun) { return; } const env: Record = { ...Bun.env, GITHUB_ACTIONS: "" }; for (const key of Object.keys(env)) { if (isScrubbedEnvVar(key)) { delete env[key]; } } const proc = Bun.spawn(testCommand.command, { cwd, env, stdout: "inherit", stderr: "inherit", }); const exitCode = await proc.exited; if (exitCode !== 0) { throw new Error(`${testCommand.label} failed with exit code ${exitCode}: ${renderedCommand}`); } } if (!validModes.has(requestedMode as Mode)) { throw new Error(`Unknown mode ${shellQuote(requestedMode)}. Expected one of: ${[...validModes].join(", ")}`); } for (const testCommand of await commandsForMode(requestedMode as Mode)) { await runTestCommand(testCommand); }