2026-01-23 07:34:50 +00:00
import { spawn } from "node:child_process" ;
2026-02-07 08:27:50 +00:00
import fs from "node:fs" ;
2026-01-23 11:36:28 +00:00
import os from "node:os" ;
2026-03-13 18:36:38 -05:00
import path from "node:path" ;
2026-03-14 11:23:25 -07:00
import { channelTestPrefixes } from "../vitest.channel-paths.mjs" ;
2026-03-18 12:16:07 -07:00
import { isUnitConfigTestFile } from "../vitest.unit-paths.mjs" ;
2026-03-19 17:59:13 -04:00
import {
getProcessTreeRecords ,
parseCompletedTestFileLines ,
sampleProcessTreeRssKb ,
} from "./test-parallel-memory.mjs" ;
2026-03-19 11:01:16 -07:00
import {
appendCapturedOutput ,
hasFatalTestRunOutput ,
resolveTestRunExitCode ,
} from "./test-parallel-utils.mjs" ;
2026-03-18 16:57:27 +00:00
import {
2026-03-20 17:09:46 +00:00
dedupeFilesPreserveOrder ,
2026-03-20 04:27:49 +00:00
loadUnitMemoryHotspotManifest ,
2026-03-18 16:57:27 +00:00
loadTestRunnerBehavior ,
loadUnitTimingManifest ,
2026-03-20 04:43:09 +00:00
selectUnitHeavyFileGroups ,
2026-03-18 16:57:27 +00:00
packFilesByDuration ,
} from "./test-runner-manifest.mjs" ;
2026-01-23 07:34:50 +00:00
2026-02-15 03:35:02 +00:00
// On Windows, `.cmd` launchers can fail with `spawn EINVAL` when invoked without a shell
// (especially under GitHub Actions + Git Bash). Use `shell: true` and let the shell resolve pnpm.
const pnpm = "pnpm" ;
2026-03-18 16:57:27 +00:00
const behaviorManifest = loadTestRunnerBehavior ( ) ;
const existingFiles = ( entries ) =>
entries . map ( ( entry ) => entry . file ) . filter ( ( file ) => fs . existsSync ( file ) ) ;
2026-03-20 04:48:24 +00:00
let tempArtifactDir = null ;
const ensureTempArtifactDir = ( ) => {
if ( tempArtifactDir === null ) {
tempArtifactDir = fs . mkdtempSync ( path . join ( os . tmpdir ( ) , "openclaw-test-parallel-" ) ) ;
}
return tempArtifactDir ;
} ;
const writeTempJsonArtifact = ( name , value ) => {
const filePath = path . join ( ensureTempArtifactDir ( ) , ` ${ name } .json ` ) ;
fs . writeFileSync ( filePath , ` ${ JSON . stringify ( value ) } \n ` , "utf8" ) ;
return filePath ;
} ;
const cleanupTempArtifacts = ( ) => {
if ( tempArtifactDir === null ) {
return ;
}
fs . rmSync ( tempArtifactDir , { recursive : true , force : true } ) ;
tempArtifactDir = null ;
} ;
2026-03-18 12:16:07 -07:00
const existingUnitConfigFiles = ( entries ) => existingFiles ( entries ) . filter ( isUnitConfigTestFile ) ;
2026-03-22 14:35:21 -07:00
const baseThreadPinnedFiles = existingFiles ( behaviorManifest . base ? . threadPinned ? ? [ ] ) ;
const unitForkIsolatedFiles = existingUnitConfigFiles ( behaviorManifest . unit . isolated ) ;
const unitThreadPinnedFiles = existingUnitConfigFiles ( behaviorManifest . unit . threadPinned ) ;
2026-03-22 16:22:04 -07:00
const unitBehaviorOverrideSet = new Set ( [ ... unitForkIsolatedFiles , ... unitThreadPinnedFiles ] ) ;
2026-03-18 16:57:27 +00:00
const channelSingletonFiles = [ ] ;
2026-02-13 04:30:39 +00:00
const children = new Set ( ) ;
const isCI = process . env . CI === "true" || process . env . GITHUB _ACTIONS === "true" ;
const isMacOS = process . platform === "darwin" || process . env . RUNNER _OS === "macOS" ;
const isWindows = process . platform === "win32" || process . env . RUNNER _OS === "Windows" ;
const isWindowsCi = isCI && isWindows ;
2026-02-25 12:16:17 +02:00
const hostCpuCount = os . cpus ( ) . length ;
const hostMemoryGiB = Math . floor ( os . totalmem ( ) / 1024 * * 3 ) ;
// Keep aggressive local defaults for high-memory workstations (Mac Studio class).
const highMemLocalHost = ! isCI && hostMemoryGiB >= 96 ;
const lowMemLocalHost = ! isCI && hostMemoryGiB < 64 ;
2026-02-13 08:15:25 -05:00
const nodeMajor = Number . parseInt ( process . versions . node . split ( "." ) [ 0 ] ? ? "" , 10 ) ;
2026-03-06 17:45:35 -05:00
const rawTestProfile = process . env . OPENCLAW _TEST _PROFILE ? . trim ( ) . toLowerCase ( ) ;
const testProfile =
rawTestProfile === "low" ||
2026-03-19 07:47:07 -05:00
rawTestProfile === "macmini" ||
2026-03-06 17:45:35 -05:00
rawTestProfile === "max" ||
rawTestProfile === "normal" ||
rawTestProfile === "serial"
? rawTestProfile
: "normal" ;
2026-03-19 07:47:07 -05:00
const isMacMiniProfile = testProfile === "macmini" ;
2026-03-20 13:15:55 -04:00
// Vitest executes Node tests through Vite's SSR/module-runner pipeline, so the
// shared unit lane still retains transformed ESM/module state even when the
2026-03-22 12:24:47 -07:00
// tests themselves are not "server rendering" a website. We previously kept
2026-03-22 16:22:04 -07:00
// forks as the default after VM-pool regressions on constrained hosts. On
2026-03-22 12:24:47 -07:00
// 2026-03-22, a direct full-unit threads run finished 1109/1110 green; the sole
// correctness exception stayed on the manifest fork lane, so the wrapper now
2026-03-22 16:22:04 -07:00
// defaults unit runs to threads while preserving explicit fork escapes.
2026-03-22 10:47:52 -07:00
const forceIsolation =
process . env . OPENCLAW _TEST _ISOLATE === "1" || process . env . OPENCLAW _TEST _ISOLATE === "true" ;
const disableIsolation =
! forceIsolation &&
process . env . OPENCLAW _TEST _NO _ISOLATE !== "0" &&
process . env . OPENCLAW _TEST _NO _ISOLATE !== "false" ;
2026-03-19 15:02:48 -07:00
const includeGatewaySuite = process . env . OPENCLAW _TEST _INCLUDE _GATEWAY === "1" ;
const includeExtensionsSuite = process . env . OPENCLAW _TEST _INCLUDE _EXTENSIONS === "1" ;
2026-03-22 12:24:47 -07:00
const parsePoolOverride = ( value , fallback ) => {
if ( value === "threads" || value === "forks" ) {
return value ;
}
return fallback ;
} ;
2026-03-08 18:39:37 +00:00
// Even on low-memory hosts, keep the isolated lane split so files like
// git-commit.test.ts still get the worker/process isolation they require.
const shouldSplitUnitRuns = testProfile !== "serial" ;
2026-03-18 16:57:27 +00:00
let runs = [ ] ;
2026-01-30 03:15:10 +01:00
const shardOverride = Number . parseInt ( process . env . OPENCLAW _TEST _SHARDS ? ? "" , 10 ) ;
2026-02-26 00:33:36 -06:00
const configuredShardCount =
Number . isFinite ( shardOverride ) && shardOverride > 1 ? shardOverride : null ;
const shardCount = configuredShardCount ? ? ( isWindowsCi ? 2 : 1 ) ;
const shardIndexOverride = ( ( ) => {
const parsed = Number . parseInt ( process . env . OPENCLAW _TEST _SHARD _INDEX ? ? "" , 10 ) ;
return Number . isFinite ( parsed ) && parsed > 0 ? parsed : null ;
} ) ( ) ;
2026-03-13 18:36:38 -05:00
const OPTION _TAKES _VALUE = new Set ( [
"-t" ,
"-c" ,
"-r" ,
"--testNamePattern" ,
"--config" ,
"--root" ,
"--dir" ,
"--reporter" ,
"--outputFile" ,
"--pool" ,
"--execArgv" ,
"--vmMemoryLimit" ,
"--maxWorkers" ,
"--environment" ,
"--shard" ,
"--changed" ,
"--sequence" ,
"--inspect" ,
"--inspectBrk" ,
"--testTimeout" ,
"--hookTimeout" ,
"--bail" ,
"--retry" ,
"--diff" ,
"--exclude" ,
"--project" ,
"--slowTestThreshold" ,
"--teardownTimeout" ,
"--attachmentsDir" ,
"--mode" ,
"--api" ,
"--browser" ,
"--maxConcurrency" ,
"--mergeReports" ,
"--configLoader" ,
"--experimental" ,
] ) ;
const SINGLE _RUN _ONLY _FLAGS = new Set ( [ "--coverage" , "--outputFile" , "--mergeReports" ] ) ;
2026-02-26 00:33:36 -06:00
if ( shardIndexOverride !== null && shardCount <= 1 ) {
console . error (
` [test-parallel] OPENCLAW_TEST_SHARD_INDEX= ${ String (
shardIndexOverride ,
) } requires OPENCLAW_TEST_SHARDS>1. ` ,
) ;
process . exit ( 2 ) ;
}
if ( shardIndexOverride !== null && shardIndexOverride > shardCount ) {
console . error (
` [test-parallel] OPENCLAW_TEST_SHARD_INDEX= ${ String (
shardIndexOverride ,
) } exceeds OPENCLAW_TEST_SHARDS= ${ String ( shardCount ) } . ` ,
) ;
process . exit ( 2 ) ;
}
2026-02-06 23:18:19 -03:00
const windowsCiArgs = isWindowsCi ? [ "--dangerouslyIgnoreUnhandledErrors" ] : [ ] ;
2026-02-12 17:59:44 +00:00
const silentArgs =
process . env . OPENCLAW _TEST _SHOW _PASSED _LOGS === "1" ? [ ] : [ "--silent=passed-only" ] ;
2026-02-07 20:02:32 -08:00
const rawPassthroughArgs = process . argv . slice ( 2 ) ;
const passthroughArgs =
rawPassthroughArgs [ 0 ] === "--" ? rawPassthroughArgs . slice ( 1 ) : rawPassthroughArgs ;
2026-03-13 18:36:38 -05:00
const parsePassthroughArgs = ( args ) => {
const fileFilters = [ ] ;
const optionArgs = [ ] ;
let consumeNextAsOptionValue = false ;
for ( const arg of args ) {
if ( consumeNextAsOptionValue ) {
optionArgs . push ( arg ) ;
consumeNextAsOptionValue = false ;
continue ;
}
if ( arg === "--" ) {
optionArgs . push ( arg ) ;
continue ;
}
if ( arg . startsWith ( "-" ) ) {
optionArgs . push ( arg ) ;
consumeNextAsOptionValue = ! arg . includes ( "=" ) && OPTION _TAKES _VALUE . has ( arg ) ;
continue ;
}
fileFilters . push ( arg ) ;
}
return { fileFilters , optionArgs } ;
} ;
const { fileFilters : passthroughFileFilters , optionArgs : passthroughOptionArgs } =
parsePassthroughArgs ( passthroughArgs ) ;
2026-03-19 07:47:07 -05:00
const passthroughMetadataFlags = new Set ( [ "-h" , "--help" , "--listTags" , "--clearCache" ] ) ;
const passthroughMetadataOnly =
passthroughArgs . length > 0 &&
passthroughFileFilters . length === 0 &&
passthroughOptionArgs . every ( ( arg ) => {
if ( ! arg . startsWith ( "-" ) ) {
return false ;
}
const [ flag ] = arg . split ( "=" , 1 ) ;
return passthroughMetadataFlags . has ( flag ) ;
} ) ;
2026-03-18 08:58:29 -07:00
const countExplicitEntryFilters = ( entryArgs ) => {
const { fileFilters } = parsePassthroughArgs ( entryArgs . slice ( 2 ) ) ;
return fileFilters . length > 0 ? fileFilters . length : null ;
} ;
2026-03-19 14:02:19 -07:00
const getExplicitEntryFilters = ( entryArgs ) => parsePassthroughArgs ( entryArgs . slice ( 2 ) ) . fileFilters ;
2026-03-13 18:36:38 -05:00
const passthroughRequiresSingleRun = passthroughOptionArgs . some ( ( arg ) => {
if ( ! arg . startsWith ( "-" ) ) {
return false ;
}
const [ flag ] = arg . split ( "=" , 1 ) ;
return SINGLE _RUN _ONLY _FLAGS . has ( flag ) ;
} ) ;
const baseConfigPrefixes = [ "src/agents/" , "src/auto-reply/" , "src/commands/" , "test/" , "ui/" ] ;
const normalizeRepoPath = ( value ) => value . split ( path . sep ) . join ( "/" ) ;
const walkTestFiles = ( rootDir ) => {
if ( ! fs . existsSync ( rootDir ) ) {
return [ ] ;
}
const entries = fs . readdirSync ( rootDir , { withFileTypes : true } ) ;
const files = [ ] ;
for ( const entry of entries ) {
const fullPath = path . join ( rootDir , entry . name ) ;
if ( entry . isDirectory ( ) ) {
files . push ( ... walkTestFiles ( fullPath ) ) ;
continue ;
}
if ( ! entry . isFile ( ) ) {
continue ;
}
if (
fullPath . endsWith ( ".test.ts" ) ||
fullPath . endsWith ( ".live.test.ts" ) ||
fullPath . endsWith ( ".e2e.test.ts" )
) {
files . push ( normalizeRepoPath ( fullPath ) ) ;
}
}
return files ;
} ;
const allKnownTestFiles = [
... new Set ( [
... walkTestFiles ( "src" ) ,
... walkTestFiles ( "extensions" ) ,
... walkTestFiles ( "test" ) ,
... walkTestFiles ( path . join ( "ui" , "src" , "ui" ) ) ,
] ) ,
] ;
2026-03-22 12:24:47 -07:00
const defaultUnitPool = parsePoolOverride ( process . env . OPENCLAW _TEST _UNIT _DEFAULT _POOL , "threads" ) ;
const isTargetedIsolatedUnitFile = ( fileFilter ) =>
2026-03-22 23:26:21 +00:00
unitForkIsolatedFiles . includes ( fileFilter ) || unitMemoryIsolatedFiles . includes ( fileFilter ) ;
2026-03-13 18:36:38 -05:00
const inferTarget = ( fileFilter ) => {
2026-03-22 12:24:47 -07:00
const isolated = isTargetedIsolatedUnitFile ( fileFilter ) ;
if ( isUnitConfigTestFile ( fileFilter ) ) {
return { owner : "unit" , isolated } ;
}
2026-03-13 18:36:38 -05:00
if ( fileFilter . endsWith ( ".live.test.ts" ) ) {
return { owner : "live" , isolated } ;
}
if ( fileFilter . endsWith ( ".e2e.test.ts" ) ) {
return { owner : "e2e" , isolated } ;
}
2026-03-14 11:23:25 -07:00
if ( channelTestPrefixes . some ( ( prefix ) => fileFilter . startsWith ( prefix ) ) ) {
return { owner : "channels" , isolated } ;
}
2026-03-13 18:36:38 -05:00
if ( fileFilter . startsWith ( "extensions/" ) ) {
return { owner : "extensions" , isolated } ;
}
if ( fileFilter . startsWith ( "src/gateway/" ) ) {
return { owner : "gateway" , isolated } ;
}
if ( baseConfigPrefixes . some ( ( prefix ) => fileFilter . startsWith ( prefix ) ) ) {
return { owner : "base" , isolated } ;
}
if ( fileFilter . startsWith ( "src/" ) ) {
return { owner : "unit" , isolated } ;
}
return { owner : "base" , isolated } ;
} ;
2026-03-18 16:57:27 +00:00
const unitTimingManifest = loadUnitTimingManifest ( ) ;
2026-03-20 04:27:49 +00:00
const unitMemoryHotspotManifest = loadUnitMemoryHotspotManifest ( ) ;
2026-03-18 16:57:27 +00:00
const parseEnvNumber = ( name , fallback ) => {
const parsed = Number . parseInt ( process . env [ name ] ? ? "" , 10 ) ;
return Number . isFinite ( parsed ) && parsed >= 0 ? parsed : fallback ;
} ;
2026-03-18 18:19:12 +00:00
const allKnownUnitFiles = allKnownTestFiles . filter ( ( file ) => {
2026-03-18 12:16:07 -07:00
return isUnitConfigTestFile ( file ) ;
2026-03-18 18:19:12 +00:00
} ) ;
2026-03-18 16:57:27 +00:00
const defaultHeavyUnitFileLimit =
2026-03-19 07:47:07 -05:00
testProfile === "serial"
? 0
: isMacMiniProfile
? 90
: testProfile === "low"
2026-03-20 21:30:44 +00:00
? 36
2026-03-19 07:47:07 -05:00
: highMemLocalHost
? 80
: 60 ;
2026-03-18 16:57:27 +00:00
const defaultHeavyUnitLaneCount =
2026-03-19 07:47:07 -05:00
testProfile === "serial"
? 0
: isMacMiniProfile
? 6
: testProfile === "low"
2026-03-20 21:30:44 +00:00
? 4
2026-03-19 07:47:07 -05:00
: highMemLocalHost
? 5
: 4 ;
2026-03-18 16:57:27 +00:00
const heavyUnitFileLimit = parseEnvNumber (
"OPENCLAW_TEST_HEAVY_UNIT_FILE_LIMIT" ,
defaultHeavyUnitFileLimit ,
) ;
const heavyUnitLaneCount = parseEnvNumber (
"OPENCLAW_TEST_HEAVY_UNIT_LANES" ,
defaultHeavyUnitLaneCount ,
) ;
const heavyUnitMinDurationMs = parseEnvNumber ( "OPENCLAW_TEST_HEAVY_UNIT_MIN_MS" , 1200 ) ;
2026-03-20 04:27:49 +00:00
const defaultMemoryHeavyUnitFileLimit =
2026-03-20 05:00:11 +00:00
testProfile === "serial" ? 0 : isCI ? 64 : testProfile === "low" ? 8 : 16 ;
2026-03-20 04:27:49 +00:00
const memoryHeavyUnitFileLimit = parseEnvNumber (
"OPENCLAW_TEST_MEMORY_HEAVY_UNIT_FILE_LIMIT" ,
defaultMemoryHeavyUnitFileLimit ,
) ;
const memoryHeavyUnitMinDeltaKb = parseEnvNumber (
"OPENCLAW_TEST_MEMORY_HEAVY_UNIT_MIN_KB" ,
unitMemoryHotspotManifest . defaultMinDeltaKb ,
) ;
2026-03-20 04:43:09 +00:00
const { memoryHeavyFiles : memoryHeavyUnitFiles , timedHeavyFiles : timedHeavyUnitFiles } =
shouldSplitUnitRuns
? selectUnitHeavyFileGroups ( {
2026-03-18 16:57:27 +00:00
candidates : allKnownUnitFiles ,
2026-03-20 04:43:09 +00:00
behaviorOverrides : unitBehaviorOverrideSet ,
timedLimit : heavyUnitFileLimit ,
timedMinDurationMs : heavyUnitMinDurationMs ,
memoryLimit : memoryHeavyUnitFileLimit ,
memoryMinDeltaKb : memoryHeavyUnitMinDeltaKb ,
2026-03-18 16:57:27 +00:00
timings : unitTimingManifest ,
2026-03-20 04:27:49 +00:00
hotspots : unitMemoryHotspotManifest ,
} )
2026-03-20 04:43:09 +00:00
: {
memoryHeavyFiles : [ ] ,
timedHeavyFiles : [ ] ,
} ;
2026-03-22 14:35:21 -07:00
const unitMemoryIsolatedFiles = dedupeFilesPreserveOrder (
2026-03-20 17:09:46 +00:00
memoryHeavyUnitFiles ,
2026-03-22 16:22:04 -07:00
unitBehaviorOverrideSet ,
2026-03-20 17:09:46 +00:00
) ;
2026-03-20 04:43:09 +00:00
const unitSchedulingOverrideSet = new Set ( [ ... unitBehaviorOverrideSet , ... memoryHeavyUnitFiles ] ) ;
2026-03-18 16:57:27 +00:00
const unitFastExcludedFiles = [
2026-03-20 04:43:09 +00:00
... new Set ( [ ... unitSchedulingOverrideSet , ... timedHeavyUnitFiles , ... channelSingletonFiles ] ) ,
2026-03-20 04:27:49 +00:00
] ;
2026-03-18 16:57:27 +00:00
const estimateUnitDurationMs = ( file ) =>
unitTimingManifest . files [ file ] ? . durationMs ? ? unitTimingManifest . defaultDurationMs ;
2026-03-21 14:56:29 -07:00
const splitFilesByDurationBudget = ( files , targetDurationMs , estimateDurationMs ) => {
if ( ! Number . isFinite ( targetDurationMs ) || targetDurationMs <= 0 || files . length <= 1 ) {
return [ files ] ;
}
const batches = [ ] ;
let currentBatch = [ ] ;
let currentDurationMs = 0 ;
for ( const file of files ) {
const durationMs = estimateDurationMs ( file ) ;
if ( currentBatch . length > 0 && currentDurationMs + durationMs > targetDurationMs ) {
batches . push ( currentBatch ) ;
currentBatch = [ ] ;
currentDurationMs = 0 ;
}
currentBatch . push ( file ) ;
currentDurationMs += durationMs ;
}
if ( currentBatch . length > 0 ) {
batches . push ( currentBatch ) ;
}
return batches ;
} ;
2026-03-20 05:08:39 +00:00
const unitFastExcludedFileSet = new Set ( unitFastExcludedFiles ) ;
const unitFastCandidateFiles = allKnownUnitFiles . filter (
( file ) => ! unitFastExcludedFileSet . has ( file ) ,
) ;
2026-03-22 23:26:21 +00:00
const extensionSharedCandidateFiles = allKnownTestFiles . filter ( ( file ) =>
file . startsWith ( "extensions/" ) ,
) ;
2026-03-19 23:29:22 -07:00
const defaultUnitFastLaneCount = isCI && ! isWindows ? 3 : 1 ;
2026-03-20 05:08:39 +00:00
const unitFastLaneCount = Math . max (
1 ,
parseEnvNumber ( "OPENCLAW_TEST_UNIT_FAST_LANES" , defaultUnitFastLaneCount ) ,
) ;
2026-03-21 14:56:29 -07:00
const defaultUnitFastBatchTargetMs = isCI && ! isWindows ? 45_000 : 0 ;
const unitFastBatchTargetMs = parseEnvNumber (
"OPENCLAW_TEST_UNIT_FAST_BATCH_TARGET_MS" ,
defaultUnitFastBatchTargetMs ,
) ;
2026-03-19 23:29:22 -07:00
// Heap snapshots on current main show long-lived unit-fast workers retaining
// transformed Vitest/Vite module graphs rather than app objects. Multiple
// bounded unit-fast lanes only help if we also recycle them serially instead
// of keeping several transform-heavy workers resident at the same time.
2026-03-20 05:08:39 +00:00
const unitFastBuckets =
unitFastLaneCount > 1
? packFilesByDuration ( unitFastCandidateFiles , unitFastLaneCount , estimateUnitDurationMs )
: [ unitFastCandidateFiles ] ;
2026-03-21 14:56:29 -07:00
const unitFastEntries = unitFastBuckets . flatMap ( ( files , index ) => {
const laneName = unitFastBuckets . length === 1 ? "unit-fast" : ` unit-fast- ${ String ( index + 1 ) } ` ;
const recycledBatches = splitFilesByDurationBudget (
files ,
unitFastBatchTargetMs ,
estimateUnitDurationMs ,
) ;
return recycledBatches
. filter ( ( batch ) => batch . length > 0 )
. map ( ( batch , batchIndex ) => ( {
name : recycledBatches . length === 1 ? laneName : ` ${ laneName } -batch- ${ String ( batchIndex + 1 ) } ` ,
serialPhase : "unit-fast" ,
env : {
OPENCLAW _VITEST _INCLUDE _FILE : writeTempJsonArtifact (
` vitest-unit-fast-include- ${ String ( index + 1 ) } - ${ String ( batchIndex + 1 ) } ` ,
batch ,
) ,
} ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
2026-03-22 12:24:47 -07:00
` --pool= ${ defaultUnitPool } ` ,
2026-03-21 14:56:29 -07:00
... ( disableIsolation ? [ "--isolate=false" ] : [ ] ) ,
] ,
} ) ) ;
} ) ;
2026-03-18 16:57:27 +00:00
const heavyUnitBuckets = packFilesByDuration (
timedHeavyUnitFiles ,
heavyUnitLaneCount ,
estimateUnitDurationMs ,
) ;
const unitHeavyEntries = heavyUnitBuckets . map ( ( files , index ) => ( {
name : ` unit-heavy- ${ String ( index + 1 ) } ` ,
args : [ "vitest" , "run" , "--config" , "vitest.unit.config.ts" , "--pool=forks" , ... files ] ,
} ) ) ;
2026-03-20 20:43:32 +00:00
const unitThreadEntries =
2026-03-22 14:35:21 -07:00
unitThreadPinnedFiles . length > 0
2026-03-20 20:43:32 +00:00
? [
{
name : "unit-threads" ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
"--pool=threads" ,
2026-03-22 14:35:21 -07:00
... unitThreadPinnedFiles ,
2026-03-20 20:43:32 +00:00
] ,
} ,
]
: [ ] ;
2026-03-22 14:35:21 -07:00
const unitIsolatedEntries = unitForkIsolatedFiles . map ( ( file ) => ( {
name : ` unit- ${ path . basename ( file , ".test.ts" ) } -isolated ` ,
args : [ "vitest" , "run" , "--config" , "vitest.unit.config.ts" , "--pool=forks" , file ] ,
} ) ) ;
2026-03-18 16:57:27 +00:00
const baseRuns = [
... ( shouldSplitUnitRuns
? [
2026-03-20 05:08:39 +00:00
... unitFastEntries ,
2026-03-22 14:35:21 -07:00
... unitIsolatedEntries ,
2026-03-18 16:57:27 +00:00
... unitHeavyEntries ,
2026-03-22 14:35:21 -07:00
... unitMemoryIsolatedFiles . map ( ( file ) => ( {
name : ` unit- ${ path . basename ( file , ".test.ts" ) } -memory-isolated ` ,
2026-03-22 23:26:21 +00:00
args : [ "vitest" , "run" , "--config" , "vitest.unit.config.ts" , "--pool=forks" , file ] ,
2026-03-18 16:57:27 +00:00
} ) ) ,
2026-03-20 20:43:32 +00:00
... unitThreadEntries ,
2026-03-18 16:57:27 +00:00
... channelSingletonFiles . map ( ( file ) => ( {
name : ` ${ path . basename ( file , ".test.ts" ) } -channels-isolated ` ,
args : [ "vitest" , "run" , "--config" , "vitest.channels.config.ts" , "--pool=forks" , file ] ,
} ) ) ,
]
: [
{
name : "unit" ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
2026-03-22 16:22:04 -07:00
"--pool=forks" ,
2026-03-18 16:57:27 +00:00
... ( disableIsolation ? [ "--isolate=false" ] : [ ] ) ,
] ,
} ,
] ) ,
... ( includeExtensionsSuite
? [
{
name : "extensions" ,
2026-03-22 12:28:55 -07:00
env :
extensionSharedCandidateFiles . length > 0
? {
OPENCLAW _VITEST _INCLUDE _FILE : writeTempJsonArtifact (
"vitest-extensions-include" ,
extensionSharedCandidateFiles ,
) ,
}
: undefined ,
2026-03-22 23:26:21 +00:00
args : [ "vitest" , "run" , "--config" , "vitest.extensions.config.ts" ] ,
2026-03-18 16:57:27 +00:00
} ,
]
: [ ] ) ,
... ( includeGatewaySuite
? [
{
name : "gateway" ,
args : [ "vitest" , "run" , "--config" , "vitest.gateway.config.ts" , "--pool=forks" ] ,
} ,
]
: [ ] ) ,
] ;
runs = baseRuns ;
const formatEntrySummary = ( entry ) => {
const explicitFilters = countExplicitEntryFilters ( entry . args ) ? ? 0 ;
return ` ${ entry . name } filters= ${ String ( explicitFilters || "all" ) } maxWorkers= ${ String (
maxWorkersForRun ( entry . name ) ? ? "default" ,
) } ` ;
} ;
2026-03-13 18:36:38 -05:00
const resolveFilterMatches = ( fileFilter ) => {
const normalizedFilter = normalizeRepoPath ( fileFilter ) ;
if ( fs . existsSync ( fileFilter ) ) {
const stats = fs . statSync ( fileFilter ) ;
if ( stats . isFile ( ) ) {
return [ normalizedFilter ] ;
}
if ( stats . isDirectory ( ) ) {
const prefix = normalizedFilter . endsWith ( "/" ) ? normalizedFilter : ` ${ normalizedFilter } / ` ;
return allKnownTestFiles . filter ( ( file ) => file . startsWith ( prefix ) ) ;
}
}
if ( /[*?[\]{}]/ . test ( normalizedFilter ) ) {
return allKnownTestFiles . filter ( ( file ) => path . matchesGlob ( file , normalizedFilter ) ) ;
}
return allKnownTestFiles . filter ( ( file ) => file . includes ( normalizedFilter ) ) ;
} ;
2026-03-22 14:35:21 -07:00
const isThreadPinnedUnitFile = ( fileFilter ) => unitThreadPinnedFiles . includes ( fileFilter ) ;
const isBaseThreadPinnedFile = ( fileFilter ) => baseThreadPinnedFiles . includes ( fileFilter ) ;
2026-03-13 18:36:38 -05:00
const createTargetedEntry = ( owner , isolated , filters ) => {
const name = isolated ? ` ${ owner } -isolated ` : owner ;
const forceForks = isolated ;
if ( owner === "unit" ) {
return {
name ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.unit.config.ts" ,
2026-03-22 12:24:47 -07:00
` --pool= ${ forceForks ? "forks" : defaultUnitPool } ` ,
2026-03-13 18:36:38 -05:00
... ( disableIsolation ? [ "--isolate=false" ] : [ ] ) ,
... filters ,
] ,
} ;
}
2026-03-18 03:39:02 +00:00
if ( owner === "unit-threads" ) {
return {
name ,
args : [ "vitest" , "run" , "--config" , "vitest.unit.config.ts" , "--pool=threads" , ... filters ] ,
} ;
}
2026-03-22 11:59:01 -07:00
if ( owner === "base-threads" ) {
return {
name ,
args : [ "vitest" , "run" , "--config" , "vitest.config.ts" , "--pool=threads" , ... filters ] ,
} ;
}
2026-03-13 18:36:38 -05:00
if ( owner === "extensions" ) {
return {
name ,
2026-03-22 16:22:04 -07:00
args : [ "vitest" , "run" , "--config" , "vitest.extensions.config.ts" , ... filters ] ,
2026-03-13 18:36:38 -05:00
} ;
}
if ( owner === "gateway" ) {
return {
name ,
args : [ "vitest" , "run" , "--config" , "vitest.gateway.config.ts" , "--pool=forks" , ... filters ] ,
} ;
}
if ( owner === "channels" ) {
return {
name ,
2026-03-22 16:22:04 -07:00
args : [ "vitest" , "run" , "--config" , "vitest.channels.config.ts" , ... filters ] ,
2026-03-13 18:36:38 -05:00
} ;
}
if ( owner === "live" ) {
return {
name ,
args : [ "vitest" , "run" , "--config" , "vitest.live.config.ts" , ... filters ] ,
} ;
}
if ( owner === "e2e" ) {
return {
name ,
args : [ "vitest" , "run" , "--config" , "vitest.e2e.config.ts" , ... filters ] ,
} ;
}
return {
name ,
args : [
"vitest" ,
"run" ,
"--config" ,
"vitest.config.ts" ,
... ( forceForks ? [ "--pool=forks" ] : [ ] ) ,
... filters ,
] ,
} ;
} ;
2026-03-22 14:35:21 -07:00
const formatPerFileEntryName = ( owner , file ) => {
const baseName = path
. basename ( file )
. replace ( /\.live\.test\.ts$/u , "" )
. replace ( /\.e2e\.test\.ts$/u , "" )
. replace ( /\.test\.ts$/u , "" ) ;
return ` ${ owner } - ${ baseName } ` ;
} ;
const createPerFileTargetedEntry = ( file ) => {
const target = inferTarget ( file ) ;
const owner = isThreadPinnedUnitFile ( file )
? "unit-threads"
2026-03-22 16:22:04 -07:00
: isBaseThreadPinnedFile ( file )
2026-03-22 23:26:21 +00:00
? "base-threads"
: target . owner ;
2026-03-22 14:35:21 -07:00
return {
... createTargetedEntry ( owner , target . isolated , [ file ] ) ,
name : ` ${ formatPerFileEntryName ( owner , file ) } ${ target . isolated ? "-isolated" : "" } ` ,
} ;
} ;
2026-03-13 18:36:38 -05:00
const targetedEntries = ( ( ) => {
if ( passthroughFileFilters . length === 0 ) {
return [ ] ;
}
const groups = passthroughFileFilters . reduce ( ( acc , fileFilter ) => {
const matchedFiles = resolveFilterMatches ( fileFilter ) ;
if ( matchedFiles . length === 0 ) {
2026-03-17 06:53:29 +00:00
const normalizedFile = normalizeRepoPath ( fileFilter ) ;
const target = inferTarget ( normalizedFile ) ;
2026-03-22 14:35:21 -07:00
const owner = isThreadPinnedUnitFile ( normalizedFile )
2026-03-18 03:39:02 +00:00
? "unit-threads"
2026-03-22 16:22:04 -07:00
: isBaseThreadPinnedFile ( normalizedFile )
2026-03-22 23:26:21 +00:00
? "base-threads"
: target . owner ;
2026-03-17 06:53:29 +00:00
const key = ` ${ owner } : ${ target . isolated ? "isolated" : "default" } ` ;
2026-03-13 18:36:38 -05:00
const files = acc . get ( key ) ? ? [ ] ;
2026-03-17 06:53:29 +00:00
files . push ( normalizedFile ) ;
2026-03-13 18:36:38 -05:00
acc . set ( key , files ) ;
return acc ;
}
for ( const matchedFile of matchedFiles ) {
const target = inferTarget ( matchedFile ) ;
2026-03-22 14:35:21 -07:00
const owner = isThreadPinnedUnitFile ( matchedFile )
2026-03-18 03:39:02 +00:00
? "unit-threads"
2026-03-22 16:22:04 -07:00
: isBaseThreadPinnedFile ( matchedFile )
2026-03-22 23:26:21 +00:00
? "base-threads"
: target . owner ;
2026-03-17 06:53:29 +00:00
const key = ` ${ owner } : ${ target . isolated ? "isolated" : "default" } ` ;
2026-03-13 18:36:38 -05:00
const files = acc . get ( key ) ? ? [ ] ;
files . push ( matchedFile ) ;
acc . set ( key , files ) ;
}
return acc ;
} , new Map ( ) ) ;
return Array . from ( groups , ( [ key , filters ] ) => {
const [ owner , mode ] = key . split ( ":" ) ;
2026-03-22 14:35:21 -07:00
const uniqueFilters = [ ... new Set ( filters ) ] ;
if ( mode === "isolated" ) {
return uniqueFilters . map ( ( file ) => createPerFileTargetedEntry ( file ) ) ;
}
return [ createTargetedEntry ( owner , false , uniqueFilters ) ] ;
} ) . flat ( ) ;
2026-03-13 18:36:38 -05:00
} ) ( ) ;
2026-03-18 03:39:02 +00:00
// Node 25 local runs still show cross-process worker shutdown contention even
// after moving the known heavy files into singleton lanes.
const topLevelParallelEnabled =
2026-03-19 07:47:07 -05:00
testProfile !== "low" &&
testProfile !== "serial" &&
! ( ! isCI && nodeMajor >= 25 ) &&
! isMacMiniProfile ;
2026-03-22 10:47:52 -07:00
const defaultTopLevelParallelLimit = disableIsolation
? isCI
? isWindows
? 2
: 4
: highMemLocalHost
? Math . min ( 16 , hostCpuCount )
: lowMemLocalHost
? Math . min ( 8 , hostCpuCount )
: Math . min ( 12 , hostCpuCount )
: testProfile === "serial"
2026-03-20 01:36:12 +00:00
? 1
: testProfile === "low"
2026-03-20 21:30:44 +00:00
? lowMemLocalHost
? 2
: 3
2026-03-20 01:36:12 +00:00
: testProfile === "max"
? 5
: highMemLocalHost
? 4
: lowMemLocalHost
? 2
: 3 ;
const topLevelParallelLimit = Math . max (
1 ,
parseEnvNumber ( "OPENCLAW_TEST_TOP_LEVEL_CONCURRENCY" , defaultTopLevelParallelLimit ) ,
) ;
2026-01-30 03:15:10 +01:00
const overrideWorkers = Number . parseInt ( process . env . OPENCLAW _TEST _WORKERS ? ? "" , 10 ) ;
2026-01-31 21:21:09 +09:00
const resolvedOverride =
Number . isFinite ( overrideWorkers ) && overrideWorkers > 0 ? overrideWorkers : null ;
2026-02-23 20:48:05 +02:00
const parallelGatewayEnabled =
2026-03-19 07:47:07 -05:00
! isMacMiniProfile &&
( process . env . OPENCLAW _TEST _PARALLEL _GATEWAY === "1" || ( ! isCI && highMemLocalHost ) ) ;
2026-02-23 20:48:05 +02:00
// Keep gateway serial by default except when explicitly requested or on high-memory local hosts.
2026-02-12 17:59:44 +00:00
const keepGatewaySerial =
isWindowsCi ||
process . env . OPENCLAW _TEST _SERIAL _GATEWAY === "1" ||
2026-02-15 07:40:13 -06:00
testProfile === "serial" ||
2026-02-23 20:48:05 +02:00
! parallelGatewayEnabled ;
2026-02-12 17:59:44 +00:00
const parallelRuns = keepGatewaySerial ? runs . filter ( ( entry ) => entry . name !== "gateway" ) : runs ;
const serialRuns = keepGatewaySerial ? runs . filter ( ( entry ) => entry . name === "gateway" ) : [ ] ;
2026-03-19 23:29:22 -07:00
const serialPrefixRuns = parallelRuns . filter ( ( entry ) => entry . serialPhase ) ;
const deferredParallelRuns = parallelRuns . filter ( ( entry ) => ! entry . serialPhase ) ;
2026-02-22 21:59:13 +00:00
const baseLocalWorkers = Math . max ( 4 , Math . min ( 16 , hostCpuCount ) ) ;
const loadAwareDisabledRaw = process . env . OPENCLAW _TEST _LOAD _AWARE ? . trim ( ) . toLowerCase ( ) ;
const loadAwareDisabled = loadAwareDisabledRaw === "0" || loadAwareDisabledRaw === "false" ;
const loadRatio =
! isCI && ! loadAwareDisabled && process . platform !== "win32" && hostCpuCount > 0
? os . loadavg ( ) [ 0 ] / hostCpuCount
: 0 ;
// Keep the fast-path unchanged on normal load; only throttle under extreme host pressure.
const extremeLoadScale = loadRatio >= 1.1 ? 0.75 : loadRatio >= 1 ? 0.85 : 1 ;
const localWorkers = Math . max ( 4 , Math . min ( 16 , Math . floor ( baseLocalWorkers * extremeLoadScale ) ) ) ;
2026-02-15 07:40:13 -06:00
const defaultWorkerBudget =
testProfile === "low"
? {
unit : 2 ,
unitIsolated : 1 ,
2026-02-25 12:16:17 +02:00
extensions : 4 ,
2026-02-15 07:40:13 -06:00
gateway : 1 ,
}
2026-03-19 07:47:07 -05:00
: isMacMiniProfile
2026-02-15 07:40:13 -06:00
? {
2026-03-19 07:47:07 -05:00
unit : 3 ,
2026-02-15 07:40:13 -06:00
unitIsolated : 1 ,
extensions : 1 ,
gateway : 1 ,
}
2026-03-19 07:47:07 -05:00
: testProfile === "serial"
2026-02-15 07:40:13 -06:00
? {
2026-03-19 07:47:07 -05:00
unit : 1 ,
unitIsolated : 1 ,
extensions : 1 ,
gateway : 1 ,
2026-02-15 07:40:13 -06:00
}
2026-03-19 07:47:07 -05:00
: testProfile === "max"
2026-02-23 20:48:05 +02:00
? {
2026-03-19 07:47:07 -05:00
unit : localWorkers ,
unitIsolated : Math . min ( 4 , localWorkers ) ,
extensions : Math . max ( 1 , Math . min ( 6 , Math . floor ( localWorkers / 2 ) ) ) ,
gateway : Math . max ( 1 , Math . min ( 2 , Math . floor ( localWorkers / 4 ) ) ) ,
2026-02-23 20:48:05 +02:00
}
2026-03-19 07:47:07 -05:00
: highMemLocalHost
2026-02-23 20:48:05 +02:00
? {
2026-03-19 07:47:07 -05:00
// After peeling measured hotspots into dedicated lanes, the shared
// unit-fast lane shuts down more reliably with a slightly smaller
// worker fan-out than the old "max it out" local default.
unit : Math . max ( 4 , Math . min ( 10 , Math . floor ( ( localWorkers * 5 ) / 8 ) ) ) ,
unitIsolated : Math . max ( 1 , Math . min ( 2 , Math . floor ( localWorkers / 6 ) || 1 ) ) ,
2026-02-23 20:48:05 +02:00
extensions : Math . max ( 1 , Math . min ( 4 , Math . floor ( localWorkers / 4 ) ) ) ,
2026-03-19 07:47:07 -05:00
gateway : Math . max ( 2 , Math . min ( 6 , Math . floor ( localWorkers / 2 ) ) ) ,
}
: lowMemLocalHost
? {
// Sub-64 GiB local hosts are prone to OOM with large vmFork runs.
unit : 2 ,
unitIsolated : 1 ,
extensions : 4 ,
gateway : 1 ,
}
: {
// 64-95 GiB local hosts: conservative split with some parallel headroom.
unit : Math . max ( 2 , Math . min ( 8 , Math . floor ( localWorkers / 2 ) ) ) ,
unitIsolated : 1 ,
extensions : Math . max ( 1 , Math . min ( 4 , Math . floor ( localWorkers / 4 ) ) ) ,
gateway : 1 ,
} ;
2026-02-07 07:57:50 +00:00
2026-01-25 07:22:36 -05:00
// Keep worker counts predictable for local runs; trim macOS CI workers to avoid worker crashes/OOM.
2026-01-25 10:18:51 +05:30
// In CI on linux/windows, prefer Vitest defaults to avoid cross-test interference from lower worker counts.
2026-02-07 07:57:50 +00:00
const maxWorkersForRun = ( name ) => {
if ( resolvedOverride ) {
return resolvedOverride ;
}
if ( isCI && ! isMacOS ) {
return null ;
}
if ( isCI && isMacOS ) {
return 1 ;
}
2026-03-22 16:22:04 -07:00
if ( name . endsWith ( "-threads" ) ) {
2026-03-18 16:57:27 +00:00
return 1 ;
}
2026-03-22 14:35:21 -07:00
if ( name . endsWith ( "-isolated" ) ) {
2026-03-18 16:57:27 +00:00
return 1 ;
}
2026-03-22 14:35:21 -07:00
if ( name . startsWith ( "unit-heavy-" ) ) {
2026-02-15 07:40:13 -06:00
return defaultWorkerBudget . unitIsolated ;
2026-02-13 04:30:39 +00:00
}
2026-02-07 07:57:50 +00:00
if ( name === "extensions" ) {
2026-02-15 07:40:13 -06:00
return defaultWorkerBudget . extensions ;
2026-02-07 07:57:50 +00:00
}
if ( name === "gateway" ) {
2026-02-15 07:40:13 -06:00
return defaultWorkerBudget . gateway ;
2026-02-07 07:57:50 +00:00
}
2026-02-15 07:40:13 -06:00
return defaultWorkerBudget . unit ;
2026-02-07 07:57:50 +00:00
} ;
2026-01-23 07:34:50 +00:00
2026-01-24 11:16:41 +00:00
const WARNING _SUPPRESSION _FLAGS = [
"--disable-warning=ExperimentalWarning" ,
"--disable-warning=DEP0040" ,
"--disable-warning=DEP0060" ,
2026-02-13 13:28:23 +00:00
"--disable-warning=MaxListenersExceededWarning" ,
2026-01-24 11:16:41 +00:00
] ;
2026-02-15 05:06:58 +00:00
const DEFAULT _CI _MAX _OLD _SPACE _SIZE _MB = 4096 ;
const maxOldSpaceSizeMb = ( ( ) => {
// CI can hit Node heap limits (especially on large suites). Allow override, default to 4GB.
const raw = process . env . OPENCLAW _TEST _MAX _OLD _SPACE _SIZE _MB ? ? "" ;
const parsed = Number . parseInt ( raw , 10 ) ;
if ( Number . isFinite ( parsed ) && parsed > 0 ) {
return parsed ;
}
if ( isCI && ! isWindows ) {
return DEFAULT _CI _MAX _OLD _SPACE _SIZE _MB ;
}
return null ;
} ) ( ) ;
2026-03-18 16:57:27 +00:00
const formatElapsedMs = ( elapsedMs ) =>
elapsedMs >= 1000 ? ` ${ ( elapsedMs / 1000 ) . toFixed ( 1 ) } s ` : ` ${ Math . round ( elapsedMs ) } ms ` ;
2026-03-19 14:02:19 -07:00
const formatMemoryKb = ( rssKb ) =>
rssKb >= 1024 * * 2
? ` ${ ( rssKb / 1024 * * 2 ) . toFixed ( 2 ) } GiB `
: rssKb >= 1024
? ` ${ ( rssKb / 1024 ) . toFixed ( 1 ) } MiB `
: ` ${ rssKb } KiB ` ;
const formatMemoryDeltaKb = ( rssKb ) =>
` ${ rssKb >= 0 ? "+" : "-" } ${ formatMemoryKb ( Math . abs ( rssKb ) ) } ` ;
const rawMemoryTrace = process . env . OPENCLAW _TEST _MEMORY _TRACE ? . trim ( ) . toLowerCase ( ) ;
const memoryTraceEnabled =
process . platform !== "win32" &&
( rawMemoryTrace === "1" ||
rawMemoryTrace === "true" ||
( rawMemoryTrace !== "0" && rawMemoryTrace !== "false" && isCI ) ) ;
const memoryTracePollMs = Math . max ( 250 , parseEnvNumber ( "OPENCLAW_TEST_MEMORY_TRACE_POLL_MS" , 1000 ) ) ;
const memoryTraceTopCount = Math . max ( 1 , parseEnvNumber ( "OPENCLAW_TEST_MEMORY_TRACE_TOP_COUNT" , 6 ) ) ;
2026-03-19 17:59:13 -04:00
const heapSnapshotIntervalMs = Math . max (
0 ,
parseEnvNumber ( "OPENCLAW_TEST_HEAPSNAPSHOT_INTERVAL_MS" , 0 ) ,
) ;
const heapSnapshotMinIntervalMs = 5000 ;
const heapSnapshotEnabled =
2026-03-19 15:05:56 -07:00
process . platform !== "win32" && heapSnapshotIntervalMs >= heapSnapshotMinIntervalMs ;
2026-03-19 17:59:13 -04:00
const heapSnapshotSignal = process . env . OPENCLAW _TEST _HEAPSNAPSHOT _SIGNAL ? . trim ( ) || "SIGUSR2" ;
const heapSnapshotBaseDir = heapSnapshotEnabled
? path . resolve (
process . env . OPENCLAW _TEST _HEAPSNAPSHOT _DIR ? . trim ( ) ||
path . join ( os . tmpdir ( ) , ` openclaw-heapsnapshots- ${ Date . now ( ) } ` ) ,
)
: null ;
const ensureNodeOptionFlag = ( nodeOptions , flagPrefix , nextValue ) =>
nodeOptions . includes ( flagPrefix ) ? nodeOptions : ` ${ nodeOptions } ${ nextValue } ` . trim ( ) ;
const isNodeLikeProcess = ( command ) => / ( ? : ^ | \ / ) node ( ? : $ | \ . exe$ ) / iu . test ( command ) ;
2026-02-15 05:06:58 +00:00
2026-01-27 16:39:28 +00:00
const runOnce = ( entry , extraArgs = [ ] ) =>
2026-01-23 07:34:50 +00:00
new Promise ( ( resolve ) => {
2026-03-18 16:57:27 +00:00
const startedAt = Date . now ( ) ;
2026-02-07 07:57:50 +00:00
const maxWorkers = maxWorkersForRun ( entry . name ) ;
2026-03-22 16:22:04 -07:00
const entryArgs = entry . args ;
2026-03-19 14:02:19 -07:00
const explicitEntryFilters = getExplicitEntryFilters ( entryArgs ) ;
2026-01-27 16:39:28 +00:00
const args = maxWorkers
2026-02-07 08:27:50 +00:00
? [
2026-02-25 12:16:17 +02:00
... entryArgs ,
2026-02-07 08:27:50 +00:00
"--maxWorkers" ,
String ( maxWorkers ) ,
2026-02-12 17:59:44 +00:00
... silentArgs ,
2026-02-07 08:27:50 +00:00
... windowsCiArgs ,
... extraArgs ,
]
2026-03-01 13:03:06 -08:00
: [ ... entryArgs , ... silentArgs , ... windowsCiArgs , ... extraArgs ] ;
2026-03-18 16:57:27 +00:00
console . log (
` [test-parallel] start ${ entry . name } workers= ${ maxWorkers ? ? "default" } filters= ${ String (
countExplicitEntryFilters ( entryArgs ) ? ? "all" ,
) } ` ,
) ;
2026-01-24 11:16:41 +00:00
const nodeOptions = process . env . NODE _OPTIONS ? ? "" ;
const nextNodeOptions = WARNING _SUPPRESSION _FLAGS . reduce (
( acc , flag ) => ( acc . includes ( flag ) ? acc : ` ${ acc } ${ flag } ` . trim ( ) ) ,
nodeOptions ,
) ;
2026-03-19 17:59:13 -04:00
const heapSnapshotDir =
heapSnapshotBaseDir === null ? null : path . join ( heapSnapshotBaseDir , entry . name ) ;
let resolvedNodeOptions =
2026-02-15 05:06:58 +00:00
maxOldSpaceSizeMb && ! nextNodeOptions . includes ( "--max-old-space-size=" )
2026-03-19 17:59:13 -04:00
? ` ${ nextNodeOptions } --max-old-space-size= ${ maxOldSpaceSizeMb } ` . trim ( )
: nextNodeOptions ;
if ( heapSnapshotEnabled && heapSnapshotDir ) {
try {
fs . mkdirSync ( heapSnapshotDir , { recursive : true } ) ;
} catch ( err ) {
2026-03-19 15:05:56 -07:00
console . error (
` [test-parallel] failed to create heap snapshot dir ${ heapSnapshotDir } : ${ String ( err ) } ` ,
) ;
2026-03-19 17:59:13 -04:00
resolve ( 1 ) ;
return ;
}
resolvedNodeOptions = ensureNodeOptionFlag (
resolvedNodeOptions ,
"--diagnostic-dir=" ,
` --diagnostic-dir= ${ heapSnapshotDir } ` ,
) ;
resolvedNodeOptions = ensureNodeOptionFlag (
resolvedNodeOptions ,
"--heapsnapshot-signal=" ,
` --heapsnapshot-signal= ${ heapSnapshotSignal } ` ,
) ;
}
2026-03-19 09:52:00 -07:00
let output = "" ;
2026-03-19 11:01:16 -07:00
let fatalSeen = false ;
let childError = null ;
2026-02-15 03:35:02 +00:00
let child ;
2026-03-19 14:02:19 -07:00
let pendingLine = "" ;
let memoryPollTimer = null ;
2026-03-19 17:59:13 -04:00
let heapSnapshotTimer = null ;
2026-03-19 14:02:19 -07:00
const memoryFileRecords = [ ] ;
let initialTreeSample = null ;
let latestTreeSample = null ;
let peakTreeSample = null ;
2026-03-19 17:59:13 -04:00
let heapSnapshotSequence = 0 ;
2026-03-19 14:02:19 -07:00
const updatePeakTreeSample = ( sample , reason ) => {
if ( ! sample ) {
return ;
}
if ( ! peakTreeSample || sample . rssKb > peakTreeSample . rssKb ) {
peakTreeSample = { ... sample , reason } ;
}
} ;
2026-03-19 17:59:13 -04:00
const triggerHeapSnapshot = ( reason ) => {
if ( ! heapSnapshotEnabled || ! child ? . pid || ! heapSnapshotDir ) {
return ;
}
const records = getProcessTreeRecords ( child . pid ) ? ? [ ] ;
const targetPids = records
. filter ( ( record ) => record . pid !== process . pid && isNodeLikeProcess ( record . command ) )
. map ( ( record ) => record . pid ) ;
if ( targetPids . length === 0 ) {
return ;
}
heapSnapshotSequence += 1 ;
let signaledCount = 0 ;
for ( const pid of targetPids ) {
try {
process . kill ( pid , heapSnapshotSignal ) ;
signaledCount += 1 ;
} catch {
// Process likely exited between ps sampling and signal delivery.
}
}
if ( signaledCount > 0 ) {
console . log (
` [test-parallel][heap] ${ entry . name } seq= ${ String ( heapSnapshotSequence ) } reason= ${ reason } signaled= ${ String (
signaledCount ,
) } / ${ String ( targetPids . length ) } dir= ${ heapSnapshotDir } ` ,
) ;
}
} ;
2026-03-19 14:02:19 -07:00
const captureTreeSample = ( reason ) => {
if ( ! memoryTraceEnabled || ! child ? . pid ) {
return null ;
}
const sample = sampleProcessTreeRssKb ( child . pid ) ;
if ( ! sample ) {
return null ;
}
latestTreeSample = sample ;
if ( ! initialTreeSample ) {
initialTreeSample = sample ;
}
updatePeakTreeSample ( sample , reason ) ;
return sample ;
} ;
const logMemoryTraceForText = ( text ) => {
if ( ! memoryTraceEnabled ) {
return ;
}
const combined = ` ${ pendingLine } ${ text } ` ;
const lines = combined . split ( /\r?\n/u ) ;
pendingLine = lines . pop ( ) ? ? "" ;
const completedFiles = parseCompletedTestFileLines ( lines . join ( "\n" ) ) ;
for ( const completedFile of completedFiles ) {
const sample = captureTreeSample ( completedFile . file ) ;
if ( ! sample ) {
continue ;
}
const previousRssKb =
memoryFileRecords . length > 0
? ( memoryFileRecords . at ( - 1 ) ? . rssKb ? ? initialTreeSample ? . rssKb ? ? sample . rssKb )
: ( initialTreeSample ? . rssKb ? ? sample . rssKb ) ;
const deltaKb = sample . rssKb - previousRssKb ;
const record = {
... completedFile ,
rssKb : sample . rssKb ,
processCount : sample . processCount ,
deltaKb ,
} ;
memoryFileRecords . push ( record ) ;
console . log (
` [test-parallel][mem] ${ entry . name } file= ${ record . file } rss= ${ formatMemoryKb (
record . rssKb ,
) } delta= ${ formatMemoryDeltaKb ( record . deltaKb ) } peak= ${ formatMemoryKb (
peakTreeSample ? . rssKb ? ? record . rssKb ,
) } procs= ${ record . processCount } ${ record . durationMs ? ` duration= ${ formatElapsedMs ( record . durationMs ) } ` : "" } ` ,
) ;
}
} ;
const logMemoryTraceSummary = ( ) => {
if ( ! memoryTraceEnabled ) {
return ;
}
captureTreeSample ( "close" ) ;
const fallbackRecord =
memoryFileRecords . length === 0 &&
explicitEntryFilters . length === 1 &&
latestTreeSample &&
initialTreeSample
? [
{
file : explicitEntryFilters [ 0 ] ,
deltaKb : latestTreeSample . rssKb - initialTreeSample . rssKb ,
} ,
]
: [ ] ;
const totalDeltaKb =
initialTreeSample && latestTreeSample
? latestTreeSample . rssKb - initialTreeSample . rssKb
: 0 ;
const topGrowthFiles = [ ... memoryFileRecords , ... fallbackRecord ]
. filter ( ( record ) => record . deltaKb > 0 && typeof record . file === "string" )
. toSorted ( ( left , right ) => right . deltaKb - left . deltaKb )
. slice ( 0 , memoryTraceTopCount )
. map ( ( record ) => ` ${ record . file } : ${ formatMemoryDeltaKb ( record . deltaKb ) } ` ) ;
console . log (
` [test-parallel][mem] summary ${ entry . name } files= ${ memoryFileRecords . length } peak= ${ formatMemoryKb (
peakTreeSample ? . rssKb ? ? 0 ,
) } totalDelta= ${ formatMemoryDeltaKb ( totalDeltaKb ) } peakAt= ${
peakTreeSample ? . reason ? ? "n/a"
} top= ${ topGrowthFiles . length > 0 ? topGrowthFiles . join ( ", " ) : "none" } ` ,
) ;
} ;
2026-02-15 03:35:02 +00:00
try {
child = spawn ( pnpm , args , {
2026-03-19 09:52:00 -07:00
stdio : [ "inherit" , "pipe" , "pipe" ] ,
2026-03-20 04:48:24 +00:00
env : {
... process . env ,
... entry . env ,
VITEST _GROUP : entry . name ,
NODE _OPTIONS : resolvedNodeOptions ,
} ,
2026-02-15 03:35:02 +00:00
shell : isWindows ,
} ) ;
2026-03-19 14:02:19 -07:00
captureTreeSample ( "spawn" ) ;
if ( memoryTraceEnabled ) {
memoryPollTimer = setInterval ( ( ) => {
captureTreeSample ( "poll" ) ;
} , memoryTracePollMs ) ;
}
2026-03-19 17:59:13 -04:00
if ( heapSnapshotEnabled ) {
heapSnapshotTimer = setInterval ( ( ) => {
triggerHeapSnapshot ( "interval" ) ;
} , heapSnapshotIntervalMs ) ;
}
2026-02-15 03:35:02 +00:00
} catch ( err ) {
console . error ( ` [test-parallel] spawn failed: ${ String ( err ) } ` ) ;
resolve ( 1 ) ;
return ;
}
2026-01-23 07:34:50 +00:00
children . add ( child ) ;
2026-03-19 09:52:00 -07:00
child . stdout ? . on ( "data" , ( chunk ) => {
const text = chunk . toString ( ) ;
2026-03-19 11:01:16 -07:00
fatalSeen || = hasFatalTestRunOutput ( ` ${ output } ${ text } ` ) ;
2026-03-19 09:52:00 -07:00
output = appendCapturedOutput ( output , text ) ;
2026-03-19 14:02:19 -07:00
logMemoryTraceForText ( text ) ;
2026-03-19 09:52:00 -07:00
process . stdout . write ( chunk ) ;
} ) ;
child . stderr ? . on ( "data" , ( chunk ) => {
const text = chunk . toString ( ) ;
2026-03-19 11:01:16 -07:00
fatalSeen || = hasFatalTestRunOutput ( ` ${ output } ${ text } ` ) ;
2026-03-19 09:52:00 -07:00
output = appendCapturedOutput ( output , text ) ;
2026-03-19 14:02:19 -07:00
logMemoryTraceForText ( text ) ;
2026-03-19 09:52:00 -07:00
process . stderr . write ( chunk ) ;
} ) ;
2026-02-15 03:35:02 +00:00
child . on ( "error" , ( err ) => {
2026-03-19 11:01:16 -07:00
childError = err ;
2026-02-15 03:35:02 +00:00
console . error ( ` [test-parallel] child error: ${ String ( err ) } ` ) ;
} ) ;
2026-03-19 09:52:00 -07:00
child . on ( "close" , ( code , signal ) => {
2026-03-19 14:02:19 -07:00
if ( memoryPollTimer ) {
clearInterval ( memoryPollTimer ) ;
}
2026-03-19 17:59:13 -04:00
if ( heapSnapshotTimer ) {
clearInterval ( heapSnapshotTimer ) ;
}
2026-01-23 07:34:50 +00:00
children . delete ( child ) ;
2026-03-19 11:01:16 -07:00
const resolvedCode = resolveTestRunExitCode ( { code , signal , output , fatalSeen , childError } ) ;
2026-03-19 14:02:19 -07:00
logMemoryTraceSummary ( ) ;
2026-03-18 16:57:27 +00:00
console . log (
2026-03-19 09:52:00 -07:00
` [test-parallel] done ${ entry . name } code= ${ String ( resolvedCode ) } elapsed= ${ formatElapsedMs ( Date . now ( ) - startedAt ) } ` ,
2026-03-18 16:57:27 +00:00
) ;
2026-03-19 09:52:00 -07:00
resolve ( resolvedCode ) ;
2026-01-23 07:34:50 +00:00
} ) ;
} ) ;
2026-03-13 18:36:38 -05:00
const run = async ( entry , extraArgs = [ ] ) => {
2026-03-18 08:58:29 -07:00
const explicitFilterCount = countExplicitEntryFilters ( entry . args ) ;
2026-03-18 12:16:07 -07:00
// Vitest requires the shard count to stay strictly below the number of
// resolved test files, so explicit-filter lanes need a `< fileCount` cap.
2026-03-18 08:58:29 -07:00
const effectiveShardCount =
2026-03-18 12:16:07 -07:00
explicitFilterCount === null
? shardCount
: Math . min ( shardCount , Math . max ( 1 , explicitFilterCount - 1 ) ) ;
2026-03-18 08:58:29 -07:00
if ( effectiveShardCount <= 1 ) {
if ( shardIndexOverride !== null && shardIndexOverride > effectiveShardCount ) {
return 0 ;
}
2026-03-13 18:36:38 -05:00
return runOnce ( entry , extraArgs ) ;
2026-01-31 21:29:14 +09:00
}
2026-02-26 00:33:36 -06:00
if ( shardIndexOverride !== null ) {
2026-03-18 08:58:29 -07:00
if ( shardIndexOverride > effectiveShardCount ) {
return 0 ;
}
return runOnce ( entry , [
"--shard" ,
` ${ shardIndexOverride } / ${ effectiveShardCount } ` ,
... extraArgs ,
] ) ;
2026-02-26 00:33:36 -06:00
}
2026-03-18 08:58:29 -07:00
for ( let shardIndex = 1 ; shardIndex <= effectiveShardCount ; shardIndex += 1 ) {
2026-01-27 16:39:28 +00:00
// eslint-disable-next-line no-await-in-loop
2026-03-18 08:58:29 -07:00
const code = await runOnce ( entry , [
"--shard" ,
` ${ shardIndex } / ${ effectiveShardCount } ` ,
... extraArgs ,
] ) ;
2026-01-31 21:29:14 +09:00
if ( code !== 0 ) {
return code ;
}
2026-01-27 16:39:28 +00:00
}
return 0 ;
} ;
2026-03-19 07:47:07 -05:00
const runEntriesWithLimit = async ( entries , extraArgs = [ ] , concurrency = 1 ) => {
if ( entries . length === 0 ) {
return undefined ;
}
const normalizedConcurrency = Math . max ( 1 , Math . floor ( concurrency ) ) ;
if ( normalizedConcurrency <= 1 ) {
for ( const entry of entries ) {
// eslint-disable-next-line no-await-in-loop
const code = await run ( entry , extraArgs ) ;
if ( code !== 0 ) {
return code ;
}
}
return undefined ;
}
let nextIndex = 0 ;
let firstFailure ;
const worker = async ( ) => {
while ( firstFailure === undefined ) {
const entryIndex = nextIndex ;
nextIndex += 1 ;
if ( entryIndex >= entries . length ) {
return ;
}
const code = await run ( entries [ entryIndex ] , extraArgs ) ;
if ( code !== 0 && firstFailure === undefined ) {
firstFailure = code ;
}
}
} ;
const workerCount = Math . min ( normalizedConcurrency , entries . length ) ;
await Promise . all ( Array . from ( { length : workerCount } , ( ) => worker ( ) ) ) ;
return firstFailure ;
} ;
2026-03-13 18:36:38 -05:00
const runEntries = async ( entries , extraArgs = [ ] ) => {
2026-03-06 17:45:35 -05:00
if ( topLevelParallelEnabled ) {
2026-03-20 01:36:12 +00:00
// Keep a bounded number of top-level Vitest processes in flight. As the
// singleton lane list grows, unbounded Promise.all scheduling turns
// isolation into cross-process contention and can reintroduce timeouts.
return runEntriesWithLimit ( entries , extraArgs , topLevelParallelLimit ) ;
2026-03-06 17:45:35 -05:00
}
2026-03-19 07:47:07 -05:00
return runEntriesWithLimit ( entries , extraArgs ) ;
2026-03-06 17:45:35 -05:00
} ;
2026-01-23 07:34:50 +00:00
const shutdown = ( signal ) => {
for ( const child of children ) {
child . kill ( signal ) ;
}
} ;
process . on ( "SIGINT" , ( ) => shutdown ( "SIGINT" ) ) ;
process . on ( "SIGTERM" , ( ) => shutdown ( "SIGTERM" ) ) ;
2026-03-20 04:48:24 +00:00
process . on ( "exit" , cleanupTempArtifacts ) ;
2026-01-23 07:34:50 +00:00
2026-03-18 16:57:27 +00:00
if ( process . env . OPENCLAW _TEST _LIST _LANES === "1" ) {
const entriesToPrint = targetedEntries . length > 0 ? targetedEntries : runs ;
for ( const entry of entriesToPrint ) {
console . log ( formatEntrySummary ( entry ) ) ;
}
process . exit ( 0 ) ;
}
2026-03-19 07:47:07 -05:00
if ( passthroughMetadataOnly ) {
const exitCode = await runOnce (
{
name : "vitest-meta" ,
args : [ "vitest" , "run" ] ,
} ,
passthroughOptionArgs ,
) ;
process . exit ( exitCode ) ;
}
2026-03-13 18:36:38 -05:00
if ( targetedEntries . length > 0 ) {
if ( passthroughRequiresSingleRun && targetedEntries . length > 1 ) {
console . error (
"[test-parallel] The provided Vitest args require a single run, but the selected test filters span multiple wrapper configs. Run one target/config at a time." ,
) ;
process . exit ( 2 ) ;
}
const targetedParallelRuns = keepGatewaySerial
? targetedEntries . filter ( ( entry ) => entry . name !== "gateway" )
: targetedEntries ;
const targetedSerialRuns = keepGatewaySerial
? targetedEntries . filter ( ( entry ) => entry . name === "gateway" )
: [ ] ;
const failedTargetedParallel = await runEntries ( targetedParallelRuns , passthroughOptionArgs ) ;
if ( failedTargetedParallel !== undefined ) {
process . exit ( failedTargetedParallel ) ;
}
for ( const entry of targetedSerialRuns ) {
// eslint-disable-next-line no-await-in-loop
const code = await run ( entry , passthroughOptionArgs ) ;
if ( code !== 0 ) {
process . exit ( code ) ;
2026-02-15 03:35:02 +00:00
}
2026-03-13 18:36:38 -05:00
}
process . exit ( 0 ) ;
}
if ( passthroughRequiresSingleRun && passthroughOptionArgs . length > 0 ) {
console . error (
"[test-parallel] The provided Vitest args require a single run. Use the dedicated npm script for that workflow (for example `pnpm test:coverage`) or target a single test file/filter." ,
) ;
process . exit ( 2 ) ;
2026-02-06 18:03:03 -08:00
}
2026-03-19 23:29:22 -07:00
if ( serialPrefixRuns . length > 0 ) {
const failedSerialPrefix = await runEntriesWithLimit ( serialPrefixRuns , passthroughOptionArgs , 1 ) ;
if ( failedSerialPrefix !== undefined ) {
process . exit ( failedSerialPrefix ) ;
}
2026-03-20 21:07:56 +00:00
const deferredRunConcurrency = isMacMiniProfile ? 3 : testProfile === "low" ? 2 : undefined ;
2026-03-19 23:29:22 -07:00
const failedDeferredParallel = isMacMiniProfile
2026-03-20 21:07:56 +00:00
? await runEntriesWithLimit ( deferredParallelRuns , passthroughOptionArgs , deferredRunConcurrency )
: deferredRunConcurrency
? await runEntriesWithLimit (
deferredParallelRuns ,
passthroughOptionArgs ,
deferredRunConcurrency ,
)
: await runEntries ( deferredParallelRuns , passthroughOptionArgs ) ;
2026-03-19 23:29:22 -07:00
if ( failedDeferredParallel !== undefined ) {
process . exit ( failedDeferredParallel ) ;
}
} else if ( isMacMiniProfile && targetedEntries . length === 0 ) {
2026-03-20 05:08:39 +00:00
const unitFastEntriesForMacMini = parallelRuns . filter ( ( entry ) =>
entry . name . startsWith ( "unit-fast" ) ,
) ;
for ( const entry of unitFastEntriesForMacMini ) {
// eslint-disable-next-line no-await-in-loop
const unitFastCode = await run ( entry , passthroughOptionArgs ) ;
2026-03-19 07:47:07 -05:00
if ( unitFastCode !== 0 ) {
process . exit ( unitFastCode ) ;
}
}
2026-03-20 05:08:39 +00:00
const deferredEntries = parallelRuns . filter ( ( entry ) => ! entry . name . startsWith ( "unit-fast" ) ) ;
2026-03-19 07:47:07 -05:00
const failedMacMiniParallel = await runEntriesWithLimit (
deferredEntries ,
passthroughOptionArgs ,
3 ,
) ;
if ( failedMacMiniParallel !== undefined ) {
process . exit ( failedMacMiniParallel ) ;
}
} else {
const failedParallel = await runEntries ( parallelRuns , passthroughOptionArgs ) ;
if ( failedParallel !== undefined ) {
process . exit ( failedParallel ) ;
}
2026-01-23 11:36:28 +00:00
}
for ( const entry of serialRuns ) {
// eslint-disable-next-line no-await-in-loop
2026-03-13 18:36:38 -05:00
const code = await run ( entry , passthroughOptionArgs ) ;
2026-01-23 11:36:28 +00:00
if ( code !== 0 ) {
process . exit ( code ) ;
}
}
process . exit ( 0 ) ;