|
|
import { spawn } from "node:child_process";
|
|
|
import fs from "node:fs";
|
|
|
import os from "node:os";
|
|
|
import path from "node:path";
|
|
|
import {
|
|
|
getProcessTreeRecords,
|
|
|
parseCompletedTestFileLines,
|
|
|
sampleProcessTreeRssKb,
|
|
|
} from "../test-parallel-memory.mjs";
|
|
|
import {
|
|
|
appendCapturedOutput,
|
|
|
formatCapturedOutputTail,
|
|
|
hasFatalTestRunOutput,
|
|
|
resolveTestRunExitCode,
|
|
|
} from "../test-parallel-utils.mjs";
|
|
|
import { countExplicitEntryFilters, getExplicitEntryFilters } from "./vitest-args.mjs";
|
|
|
|
|
|
const countUnitEntryFilters = (unit) => {
|
|
|
const explicitFilterCount = countExplicitEntryFilters(unit.args);
|
|
|
if (explicitFilterCount !== null) {
|
|
|
return explicitFilterCount;
|
|
|
}
|
|
|
if (Array.isArray(unit.includeFiles) && unit.includeFiles.length > 0) {
|
|
|
return unit.includeFiles.length;
|
|
|
}
|
|
|
return null;
|
|
|
};
|
|
|
|
|
|
export function resolvePnpmCommandInvocation(options = {}) {
|
|
|
const npmExecPath = typeof options.npmExecPath === "string" ? options.npmExecPath.trim() : "";
|
|
|
if (npmExecPath && path.isAbsolute(npmExecPath)) {
|
|
|
const npmExecBase = path.basename(npmExecPath).toLowerCase();
|
|
|
if (npmExecBase.startsWith("pnpm")) {
|
|
|
return {
|
|
|
command: options.nodeExecPath || process.execPath,
|
|
|
args: [npmExecPath],
|
|
|
};
|
|
|
}
|
|
|
}
|
|
|
|
|
|
if (options.platform === "win32") {
|
|
|
return {
|
|
|
command: options.comSpec || "cmd.exe",
|
|
|
args: ["/d", "/s", "/c", "pnpm.cmd"],
|
|
|
};
|
|
|
}
|
|
|
|
|
|
return {
|
|
|
command: "pnpm",
|
|
|
args: [],
|
|
|
};
|
|
|
}
|
|
|
|
|
|
const sanitizeArtifactName = (value) => {
|
|
|
const normalized = value
|
|
|
.trim()
|
|
|
.replace(/[^a-z0-9._-]+/giu, "-")
|
|
|
.replace(/^-+|-+$/gu, "");
|
|
|
return normalized || "artifact";
|
|
|
};
|
|
|
|
|
|
const DEFAULT_CI_MAX_OLD_SPACE_SIZE_MB = 4096;
|
|
|
const WARNING_SUPPRESSION_FLAGS = [
|
|
|
"--disable-warning=ExperimentalWarning",
|
|
|
"--disable-warning=DEP0040",
|
|
|
"--disable-warning=DEP0060",
|
|
|
"--disable-warning=MaxListenersExceededWarning",
|
|
|
];
|
|
|
|
|
|
const formatElapsedMs = (elapsedMs) =>
|
|
|
elapsedMs >= 1000 ? `${(elapsedMs / 1000).toFixed(1)}s` : `${Math.round(elapsedMs)}ms`;
|
|
|
const formatMemoryKb = (rssKb) =>
|
|
|
rssKb >= 1024 ** 2
|
|
|
? `${(rssKb / 1024 ** 2).toFixed(2)}GiB`
|
|
|
: rssKb >= 1024
|
|
|
? `${(rssKb / 1024).toFixed(1)}MiB`
|
|
|
: `${rssKb}KiB`;
|
|
|
const formatMemoryDeltaKb = (rssKb) =>
|
|
|
`${rssKb >= 0 ? "+" : "-"}${formatMemoryKb(Math.abs(rssKb))}`;
|
|
|
|
|
|
const extractFailedTestFiles = (output) => {
|
|
|
const failureFiles = new Set();
|
|
|
const pattern = /^\s*❯\s+([^\s(][^(]*?\.(?:test|spec)\.[cm]?[jt]sx?)/gmu;
|
|
|
for (const match of output.matchAll(pattern)) {
|
|
|
const file = match[1]?.trim();
|
|
|
if (file) {
|
|
|
failureFiles.add(file);
|
|
|
}
|
|
|
}
|
|
|
return [...failureFiles];
|
|
|
};
|
|
|
|
|
|
const classifyRunResult = ({ resolvedCode, signal, fatalSeen, childError, failedTestFiles }) => {
|
|
|
if (resolvedCode === 0) {
|
|
|
return "pass";
|
|
|
}
|
|
|
if (childError || signal || fatalSeen || failedTestFiles.length === 0) {
|
|
|
return "infra-failure";
|
|
|
}
|
|
|
return "test-failure";
|
|
|
};
|
|
|
|
|
|
const formatRunLabel = (result) =>
|
|
|
`unit=${result.unitId}${result.shardLabel ? ` shard=${result.shardLabel}` : ""}`;
|
|
|
|
|
|
const buildFinalRunReport = (results) => {
|
|
|
const failedResults = results.filter((result) => result.exitCode !== 0);
|
|
|
const failedUnits = new Set(failedResults.map((result) => result.unitId));
|
|
|
const failedTestFiles = new Set(failedResults.flatMap((result) => result.failedTestFiles ?? []));
|
|
|
const infraFailures = failedResults.filter((result) => result.classification === "infra-failure");
|
|
|
return {
|
|
|
exitCode: failedResults.length > 0 ? 1 : 0,
|
|
|
results,
|
|
|
summary: {
|
|
|
failedRunCount: failedResults.length,
|
|
|
failedUnitCount: failedUnits.size,
|
|
|
failedTestFileCount: failedTestFiles.size,
|
|
|
infraFailureCount: infraFailures.length,
|
|
|
},
|
|
|
};
|
|
|
};
|
|
|
|
|
|
const printFinalRunSummary = (plan, report, reportArtifactPath) => {
|
|
|
console.log(
|
|
|
`[test-parallel] summary failurePolicy=${plan.failurePolicy} failedUnits=${String(
|
|
|
report.summary.failedUnitCount,
|
|
|
)} failedTestFiles=${String(report.summary.failedTestFileCount)} infraFailures=${String(
|
|
|
report.summary.infraFailureCount,
|
|
|
)}`,
|
|
|
);
|
|
|
if (report.summary.failedTestFileCount > 0) {
|
|
|
console.error("[test-parallel] failing tests");
|
|
|
const failedTestFiles = report.results
|
|
|
.flatMap((result) => (result.classification === "test-failure" ? result.failedTestFiles : []))
|
|
|
.filter((file) => typeof file === "string");
|
|
|
for (const file of new Set(failedTestFiles)) {
|
|
|
console.error(`- ${String(file)}`);
|
|
|
}
|
|
|
}
|
|
|
if (report.summary.infraFailureCount > 0) {
|
|
|
console.error("[test-parallel] infrastructure failures");
|
|
|
for (const result of report.results.filter(
|
|
|
(entry) => entry.classification === "infra-failure",
|
|
|
)) {
|
|
|
console.error(
|
|
|
`- ${formatRunLabel(result)} code=${String(result.exitCode)} signal=${result.signal ?? "none"} log=${result.logPath}${result.failureArtifactPath ? ` meta=${result.failureArtifactPath}` : ""}`,
|
|
|
);
|
|
|
}
|
|
|
}
|
|
|
console.error(`[test-parallel] summary artifact ${reportArtifactPath}`);
|
|
|
};
|
|
|
|
|
|
export function createExecutionArtifacts(env = process.env) {
|
|
|
let tempArtifactDir = null;
|
|
|
const ensureTempArtifactDir = () => {
|
|
|
if (tempArtifactDir === null) {
|
|
|
tempArtifactDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-test-parallel-"));
|
|
|
}
|
|
|
return tempArtifactDir;
|
|
|
};
|
|
|
const writeTempJsonArtifact = (name, value) => {
|
|
|
const filePath = path.join(ensureTempArtifactDir(), `${sanitizeArtifactName(name)}.json`);
|
|
|
fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`, "utf8");
|
|
|
return filePath;
|
|
|
};
|
|
|
const cleanupTempArtifacts = () => {
|
|
|
if (tempArtifactDir === null) {
|
|
|
return;
|
|
|
}
|
|
|
if (env.OPENCLAW_TEST_KEEP_TEMP_ARTIFACTS === "1") {
|
|
|
console.error(`[test-parallel] keeping temp artifacts at ${tempArtifactDir}`);
|
|
|
return;
|
|
|
}
|
|
|
fs.rmSync(tempArtifactDir, { recursive: true, force: true });
|
|
|
tempArtifactDir = null;
|
|
|
};
|
|
|
return { ensureTempArtifactDir, writeTempJsonArtifact, cleanupTempArtifacts };
|
|
|
}
|
|
|
|
|
|
export function createTempArtifactWriteStream(filePath) {
|
|
|
const fd = fs.openSync(filePath, "w");
|
|
|
return fs.createWriteStream(filePath, {
|
|
|
fd,
|
|
|
autoClose: true,
|
|
|
});
|
|
|
}
|
|
|
|
|
|
const ensureNodeOptionFlag = (nodeOptions, flagPrefix, nextValue) =>
|
|
|
nodeOptions.includes(flagPrefix) ? nodeOptions : `${nodeOptions} ${nextValue}`.trim();
|
|
|
|
|
|
const ensureNodeOptionFilePathFlag = (nodeOptions, flag, filePath) => {
|
|
|
const normalized = nodeOptions.trim();
|
|
|
const emptyAssignmentPattern = new RegExp(`(^|\\s)${flag}=(?=\\s|$)`, "u");
|
|
|
if (emptyAssignmentPattern.test(normalized)) {
|
|
|
return normalized.replace(emptyAssignmentPattern, `$1${flag}=${filePath}`);
|
|
|
}
|
|
|
const bareFlagPattern = new RegExp(`(^|\\s)${flag}(?=\\s|$)`, "u");
|
|
|
if (bareFlagPattern.test(normalized)) {
|
|
|
return normalized.replace(bareFlagPattern, `$1${flag}=${filePath}`);
|
|
|
}
|
|
|
return ensureNodeOptionFlag(normalized, `${flag}=`, `${flag}=${filePath}`);
|
|
|
};
|
|
|
|
|
|
const isNodeLikeProcess = (command) => /(?:^|\/)node(?:$|\.exe$)/iu.test(command);
|
|
|
|
|
|
const getShardLabel = (args) => {
|
|
|
const shardIndex = args.findIndex((arg) => arg === "--shard");
|
|
|
if (shardIndex < 0) {
|
|
|
return "";
|
|
|
}
|
|
|
return typeof args[shardIndex + 1] === "string" ? args[shardIndex + 1] : "";
|
|
|
};
|
|
|
|
|
|
const normalizeEnvFlag = (value) => value?.trim().toLowerCase();
|
|
|
|
|
|
const isEnvFlagEnabled = (value) => {
|
|
|
const normalized = normalizeEnvFlag(value);
|
|
|
return normalized === "1" || normalized === "true";
|
|
|
};
|
|
|
|
|
|
const isEnvFlagDisabled = (value) => {
|
|
|
const normalized = normalizeEnvFlag(value);
|
|
|
return normalized === "0" || normalized === "false";
|
|
|
};
|
|
|
|
|
|
const isWindowsEnv = (env, platform = process.platform) => {
|
|
|
if (platform === "win32") {
|
|
|
return true;
|
|
|
}
|
|
|
return normalizeEnvFlag(env.RUNNER_OS) === "windows";
|
|
|
};
|
|
|
|
|
|
const isFsModuleCacheEnabled = (env, platform = process.platform) => {
|
|
|
if (isWindowsEnv(env, platform)) {
|
|
|
return isEnvFlagEnabled(env.OPENCLAW_VITEST_FS_MODULE_CACHE);
|
|
|
}
|
|
|
return !isEnvFlagDisabled(env.OPENCLAW_VITEST_FS_MODULE_CACHE);
|
|
|
};
|
|
|
|
|
|
export const resolveVitestFsModuleCachePath = ({
|
|
|
cwd = process.cwd(),
|
|
|
env = process.env,
|
|
|
platform = process.platform,
|
|
|
unitId = "",
|
|
|
} = {}) => {
|
|
|
const explicitPath = env.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH?.trim();
|
|
|
if (!isFsModuleCacheEnabled(env, platform)) {
|
|
|
return undefined;
|
|
|
}
|
|
|
if (explicitPath) {
|
|
|
return explicitPath;
|
|
|
}
|
|
|
const pathImpl = isWindowsEnv(env, platform) ? path.win32 : path.posix;
|
|
|
return pathImpl.join(
|
|
|
cwd,
|
|
|
"node_modules",
|
|
|
".experimental-vitest-cache",
|
|
|
sanitizeArtifactName(unitId || "default"),
|
|
|
);
|
|
|
};
|
|
|
|
|
|
export function formatPlanOutput(plan) {
|
|
|
return [
|
|
|
`runtime=${plan.runtimeCapabilities.runtimeProfileName} mode=${plan.runtimeCapabilities.mode} intent=${plan.runtimeCapabilities.intentProfile} memoryBand=${plan.runtimeCapabilities.memoryBand} loadBand=${plan.runtimeCapabilities.loadBand} failurePolicy=${plan.failurePolicy} vitestMaxWorkers=${String(plan.executionBudget.vitestMaxWorkers ?? "default")} topLevelParallel=${plan.topLevelParallelEnabled ? String(plan.topLevelParallelLimit) : "off"}`,
|
|
|
...plan.selectedUnits.map(
|
|
|
(unit) =>
|
|
|
`${unit.id} filters=${String(countUnitEntryFilters(unit) ?? "all")} maxWorkers=${String(
|
|
|
unit.maxWorkers ?? "default",
|
|
|
)} surface=${unit.surface} isolate=${unit.isolate ? "yes" : "no"} pool=${unit.pool}`,
|
|
|
),
|
|
|
].join("\n");
|
|
|
}
|
|
|
|
|
|
export function formatExplanation(explanation) {
|
|
|
return [
|
|
|
`file=${explanation.file}`,
|
|
|
`runtime=${explanation.runtimeProfile} intent=${explanation.intentProfile} memoryBand=${explanation.memoryBand} loadBand=${explanation.loadBand}`,
|
|
|
`surface=${explanation.surface}`,
|
|
|
`isolate=${explanation.isolate ? "yes" : "no"}`,
|
|
|
`pool=${explanation.pool}`,
|
|
|
`maxWorkers=${String(explanation.maxWorkers ?? "default")}`,
|
|
|
`reasons=${explanation.reasons.join(",")}`,
|
|
|
`command=${explanation.args.join(" ")}`,
|
|
|
].join("\n");
|
|
|
}
|
|
|
|
|
|
const buildOrderedParallelSegments = (units) => {
|
|
|
const segments = [];
|
|
|
let deferredUnits = [];
|
|
|
for (const unit of units) {
|
|
|
if (unit.serialPhase) {
|
|
|
if (deferredUnits.length > 0) {
|
|
|
segments.push({ type: "deferred", units: deferredUnits });
|
|
|
deferredUnits = [];
|
|
|
}
|
|
|
const lastSegment = segments.at(-1);
|
|
|
if (lastSegment?.type === "serialPhase" && lastSegment.phase === unit.serialPhase) {
|
|
|
lastSegment.units.push(unit);
|
|
|
} else {
|
|
|
segments.push({ type: "serialPhase", phase: unit.serialPhase, units: [unit] });
|
|
|
}
|
|
|
continue;
|
|
|
}
|
|
|
deferredUnits.push(unit);
|
|
|
}
|
|
|
if (deferredUnits.length > 0) {
|
|
|
segments.push({ type: "deferred", units: deferredUnits });
|
|
|
}
|
|
|
return segments;
|
|
|
};
|
|
|
|
|
|
const prioritizeDeferredUnitsForPhase = (units, phase) => {
|
|
|
const preferredSurface =
|
|
|
phase === "extensions" || phase === "channels" ? phase : phase === "unit-fast" ? "unit" : null;
|
|
|
if (preferredSurface === null) {
|
|
|
return units;
|
|
|
}
|
|
|
const preferred = [];
|
|
|
const remaining = [];
|
|
|
for (const unit of units) {
|
|
|
if (unit.surface === preferredSurface) {
|
|
|
preferred.push(unit);
|
|
|
} else {
|
|
|
remaining.push(unit);
|
|
|
}
|
|
|
}
|
|
|
return preferred.length > 0 ? [...preferred, ...remaining] : units;
|
|
|
};
|
|
|
|
|
|
const partitionUnitsBySurface = (units, surface) => {
|
|
|
const matching = [];
|
|
|
const remaining = [];
|
|
|
for (const unit of units) {
|
|
|
if (unit.surface === surface) {
|
|
|
matching.push(unit);
|
|
|
} else {
|
|
|
remaining.push(unit);
|
|
|
}
|
|
|
}
|
|
|
return { matching, remaining };
|
|
|
};
|
|
|
|
|
|
export async function executePlan(plan, options = {}) {
|
|
|
const env = options.env ?? process.env;
|
|
|
const artifacts = options.artifacts ?? createExecutionArtifacts(env);
|
|
|
const spawnImpl = options.spawn ?? spawn;
|
|
|
const pnpmInvocation = resolvePnpmCommandInvocation({
|
|
|
npmExecPath: env.npm_execpath,
|
|
|
nodeExecPath: process.execPath,
|
|
|
platform: process.platform,
|
|
|
comSpec: env.ComSpec,
|
|
|
});
|
|
|
const children = new Set();
|
|
|
const windowsCiArgs = plan.runtimeCapabilities.isWindowsCi
|
|
|
? ["--dangerouslyIgnoreUnhandledErrors"]
|
|
|
: [];
|
|
|
const silentArgs = env.OPENCLAW_TEST_SHOW_PASSED_LOGS === "1" ? [] : ["--silent=passed-only"];
|
|
|
const rawMemoryTrace = env.OPENCLAW_TEST_MEMORY_TRACE?.trim().toLowerCase();
|
|
|
const memoryTraceEnabled =
|
|
|
process.platform !== "win32" &&
|
|
|
(rawMemoryTrace === "1" ||
|
|
|
rawMemoryTrace === "true" ||
|
|
|
(rawMemoryTrace !== "0" && rawMemoryTrace !== "false" && plan.runtimeCapabilities.isCI));
|
|
|
const parseEnvNumber = (name, fallback) => {
|
|
|
const parsed = Number.parseInt(env[name] ?? "", 10);
|
|
|
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
|
|
};
|
|
|
const memoryTracePollMs = Math.max(
|
|
|
250,
|
|
|
parseEnvNumber("OPENCLAW_TEST_MEMORY_TRACE_POLL_MS", 1000),
|
|
|
);
|
|
|
const memoryTraceTopCount = Math.max(
|
|
|
1,
|
|
|
parseEnvNumber("OPENCLAW_TEST_MEMORY_TRACE_TOP_COUNT", 6),
|
|
|
);
|
|
|
const requestedHeapSnapshotIntervalMs = Math.max(
|
|
|
0,
|
|
|
parseEnvNumber("OPENCLAW_TEST_HEAPSNAPSHOT_INTERVAL_MS", 0),
|
|
|
);
|
|
|
const heapSnapshotMinIntervalMs = 1000;
|
|
|
const heapSnapshotIntervalMs =
|
|
|
requestedHeapSnapshotIntervalMs > 0
|
|
|
? Math.max(heapSnapshotMinIntervalMs, requestedHeapSnapshotIntervalMs)
|
|
|
: 0;
|
|
|
const heapSnapshotEnabled =
|
|
|
process.platform !== "win32" && heapSnapshotIntervalMs >= heapSnapshotMinIntervalMs;
|
|
|
const heapSnapshotSignal = env.OPENCLAW_TEST_HEAPSNAPSHOT_SIGNAL?.trim() || "SIGUSR2";
|
|
|
const closeGraceMs = Math.max(100, parseEnvNumber("OPENCLAW_TEST_CLOSE_GRACE_MS", 2000));
|
|
|
const heapSnapshotBaseDir = heapSnapshotEnabled
|
|
|
? path.resolve(
|
|
|
env.OPENCLAW_TEST_HEAPSNAPSHOT_DIR?.trim() ||
|
|
|
path.join(os.tmpdir(), `openclaw-heapsnapshots-${Date.now()}`),
|
|
|
)
|
|
|
: null;
|
|
|
const maxOldSpaceSizeMb = (() => {
|
|
|
const raw = env.OPENCLAW_TEST_MAX_OLD_SPACE_SIZE_MB ?? "";
|
|
|
const parsed = Number.parseInt(raw, 10);
|
|
|
if (Number.isFinite(parsed) && parsed > 0) {
|
|
|
return parsed;
|
|
|
}
|
|
|
if (plan.runtimeCapabilities.isCI && !plan.runtimeCapabilities.isWindows) {
|
|
|
return DEFAULT_CI_MAX_OLD_SPACE_SIZE_MB;
|
|
|
}
|
|
|
return null;
|
|
|
})();
|
|
|
|
|
|
const shutdown = (signal) => {
|
|
|
for (const child of children) {
|
|
|
child.kill(signal);
|
|
|
}
|
|
|
};
|
|
|
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
|
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
|
process.on("exit", artifacts.cleanupTempArtifacts);
|
|
|
|
|
|
const shouldCollectAllFailures = plan.failurePolicy === "collect-all";
|
|
|
|
|
|
const runOnce = (unit, extraArgs = []) =>
|
|
|
new Promise((resolve) => {
|
|
|
const startedAt = Date.now();
|
|
|
const entryArgs = unit.args;
|
|
|
const explicitEntryFilters = getExplicitEntryFilters(entryArgs);
|
|
|
const args = unit.maxWorkers
|
|
|
? [
|
|
|
...entryArgs,
|
|
|
"--maxWorkers",
|
|
|
String(unit.maxWorkers),
|
|
|
...silentArgs,
|
|
|
...windowsCiArgs,
|
|
|
...extraArgs,
|
|
|
]
|
|
|
: [...entryArgs, ...silentArgs, ...windowsCiArgs, ...extraArgs];
|
|
|
const spawnArgs = [...pnpmInvocation.args, ...args];
|
|
|
const shardLabel = getShardLabel(extraArgs);
|
|
|
const artifactStem = [
|
|
|
sanitizeArtifactName(unit.id),
|
|
|
shardLabel ? `shard-${sanitizeArtifactName(shardLabel)}` : "",
|
|
|
String(startedAt),
|
|
|
]
|
|
|
.filter(Boolean)
|
|
|
.join("-");
|
|
|
const laneLogPath = path.join(artifacts.ensureTempArtifactDir(), `${artifactStem}.log`);
|
|
|
const laneLogStream = createTempArtifactWriteStream(laneLogPath);
|
|
|
laneLogStream.write(`[test-parallel] entry=${unit.id}\n`);
|
|
|
laneLogStream.write(`[test-parallel] cwd=${process.cwd()}\n`);
|
|
|
laneLogStream.write(
|
|
|
`[test-parallel] command=${[pnpmInvocation.command, ...spawnArgs].join(" ")}\n\n`,
|
|
|
);
|
|
|
console.log(
|
|
|
`[test-parallel] start ${unit.id} workers=${unit.maxWorkers ?? "default"} filters=${String(
|
|
|
countExplicitEntryFilters(entryArgs) ?? "all",
|
|
|
)}`,
|
|
|
);
|
|
|
const nodeOptions = env.NODE_OPTIONS ?? "";
|
|
|
const nextNodeOptions = WARNING_SUPPRESSION_FLAGS.reduce(
|
|
|
(acc, flag) => (acc.includes(flag) ? acc : `${acc} ${flag}`.trim()),
|
|
|
nodeOptions,
|
|
|
);
|
|
|
const heapSnapshotDir =
|
|
|
heapSnapshotBaseDir === null ? null : path.join(heapSnapshotBaseDir, unit.id);
|
|
|
let resolvedNodeOptions =
|
|
|
maxOldSpaceSizeMb && !nextNodeOptions.includes("--max-old-space-size=")
|
|
|
? `${nextNodeOptions} --max-old-space-size=${maxOldSpaceSizeMb}`.trim()
|
|
|
: nextNodeOptions;
|
|
|
const localStorageFilePath = path.join(
|
|
|
artifacts.ensureTempArtifactDir(),
|
|
|
`${artifactStem}.localstorage.json`,
|
|
|
);
|
|
|
resolvedNodeOptions = ensureNodeOptionFilePathFlag(
|
|
|
resolvedNodeOptions,
|
|
|
"--localstorage-file",
|
|
|
localStorageFilePath,
|
|
|
);
|
|
|
if (heapSnapshotEnabled && heapSnapshotDir) {
|
|
|
try {
|
|
|
fs.mkdirSync(heapSnapshotDir, { recursive: true });
|
|
|
} catch (err) {
|
|
|
console.error(
|
|
|
`[test-parallel] failed to create heap snapshot dir ${heapSnapshotDir}: ${String(err)}`,
|
|
|
);
|
|
|
resolve({
|
|
|
unitId: unit.id,
|
|
|
shardLabel,
|
|
|
classification: "infra-failure",
|
|
|
exitCode: 1,
|
|
|
signal: null,
|
|
|
elapsedMs: Date.now() - startedAt,
|
|
|
failedTestFiles: [...explicitEntryFilters],
|
|
|
explicitEntryFilters,
|
|
|
failureArtifactPath: null,
|
|
|
logPath: laneLogPath,
|
|
|
outputTail: "",
|
|
|
});
|
|
|
return;
|
|
|
}
|
|
|
resolvedNodeOptions = ensureNodeOptionFlag(
|
|
|
resolvedNodeOptions,
|
|
|
"--diagnostic-dir=",
|
|
|
`--diagnostic-dir=${heapSnapshotDir}`,
|
|
|
);
|
|
|
resolvedNodeOptions = ensureNodeOptionFlag(
|
|
|
resolvedNodeOptions,
|
|
|
"--heapsnapshot-signal=",
|
|
|
`--heapsnapshot-signal=${heapSnapshotSignal}`,
|
|
|
);
|
|
|
}
|
|
|
let output = "";
|
|
|
let fatalSeen = false;
|
|
|
let childError = null;
|
|
|
let child;
|
|
|
let pendingLine = "";
|
|
|
let memoryPollTimer = null;
|
|
|
let heapSnapshotTimer = null;
|
|
|
let closeFallbackTimer = null;
|
|
|
let failureArtifactPath = null;
|
|
|
let failureTail = "";
|
|
|
const memoryFileRecords = [];
|
|
|
let initialTreeSample = null;
|
|
|
let latestTreeSample = null;
|
|
|
let peakTreeSample = null;
|
|
|
let heapSnapshotSequence = 0;
|
|
|
let childExitState = null;
|
|
|
let settled = false;
|
|
|
const updatePeakTreeSample = (sample, reason) => {
|
|
|
if (!sample) {
|
|
|
return;
|
|
|
}
|
|
|
if (!peakTreeSample || sample.rssKb > peakTreeSample.rssKb) {
|
|
|
peakTreeSample = { ...sample, reason };
|
|
|
}
|
|
|
};
|
|
|
const triggerHeapSnapshot = (reason) => {
|
|
|
if (!heapSnapshotEnabled || !child?.pid || !heapSnapshotDir) {
|
|
|
return;
|
|
|
}
|
|
|
const records = getProcessTreeRecords(child.pid) ?? [];
|
|
|
const targetPids = records
|
|
|
.filter((record) => record.pid !== process.pid && isNodeLikeProcess(record.command))
|
|
|
.map((record) => record.pid);
|
|
|
if (targetPids.length === 0) {
|
|
|
return;
|
|
|
}
|
|
|
heapSnapshotSequence += 1;
|
|
|
let signaledCount = 0;
|
|
|
for (const pid of targetPids) {
|
|
|
try {
|
|
|
process.kill(pid, heapSnapshotSignal);
|
|
|
signaledCount += 1;
|
|
|
} catch {}
|
|
|
}
|
|
|
if (signaledCount > 0) {
|
|
|
console.log(
|
|
|
`[test-parallel][heap] ${unit.id} seq=${String(heapSnapshotSequence)} reason=${reason} signaled=${String(
|
|
|
signaledCount,
|
|
|
)}/${String(targetPids.length)} dir=${heapSnapshotDir}`,
|
|
|
);
|
|
|
}
|
|
|
};
|
|
|
const captureTreeSample = (reason) => {
|
|
|
if (!memoryTraceEnabled || !child?.pid) {
|
|
|
return null;
|
|
|
}
|
|
|
const sample = sampleProcessTreeRssKb(child.pid);
|
|
|
if (!sample) {
|
|
|
return null;
|
|
|
}
|
|
|
latestTreeSample = sample;
|
|
|
if (!initialTreeSample) {
|
|
|
initialTreeSample = sample;
|
|
|
}
|
|
|
updatePeakTreeSample(sample, reason);
|
|
|
return sample;
|
|
|
};
|
|
|
const logMemoryTraceForText = (text) => {
|
|
|
if (!memoryTraceEnabled) {
|
|
|
return;
|
|
|
}
|
|
|
const combined = `${pendingLine}${text}`;
|
|
|
const lines = combined.split(/\r?\n/u);
|
|
|
pendingLine = lines.pop() ?? "";
|
|
|
const completedFiles = parseCompletedTestFileLines(lines.join("\n"));
|
|
|
for (const completedFile of completedFiles) {
|
|
|
const sample = captureTreeSample(completedFile.file);
|
|
|
if (!sample) {
|
|
|
continue;
|
|
|
}
|
|
|
const previousRssKb =
|
|
|
memoryFileRecords.length > 0
|
|
|
? (memoryFileRecords.at(-1)?.rssKb ?? initialTreeSample?.rssKb ?? sample.rssKb)
|
|
|
: (initialTreeSample?.rssKb ?? sample.rssKb);
|
|
|
const deltaKb = sample.rssKb - previousRssKb;
|
|
|
const record = {
|
|
|
...completedFile,
|
|
|
rssKb: sample.rssKb,
|
|
|
processCount: sample.processCount,
|
|
|
deltaKb,
|
|
|
};
|
|
|
memoryFileRecords.push(record);
|
|
|
console.log(
|
|
|
`[test-parallel][mem] ${unit.id} file=${record.file} rss=${formatMemoryKb(
|
|
|
record.rssKb,
|
|
|
)} delta=${formatMemoryDeltaKb(record.deltaKb)} peak=${formatMemoryKb(
|
|
|
peakTreeSample?.rssKb ?? record.rssKb,
|
|
|
)} procs=${record.processCount}${record.durationMs ? ` duration=${formatElapsedMs(record.durationMs)}` : ""}`,
|
|
|
);
|
|
|
}
|
|
|
};
|
|
|
const logMemoryTraceSummary = () => {
|
|
|
if (!memoryTraceEnabled) {
|
|
|
return;
|
|
|
}
|
|
|
captureTreeSample("close");
|
|
|
const fallbackRecord =
|
|
|
memoryFileRecords.length === 0 &&
|
|
|
explicitEntryFilters.length === 1 &&
|
|
|
latestTreeSample &&
|
|
|
initialTreeSample
|
|
|
? [
|
|
|
{
|
|
|
file: explicitEntryFilters[0],
|
|
|
deltaKb: latestTreeSample.rssKb - initialTreeSample.rssKb,
|
|
|
},
|
|
|
]
|
|
|
: [];
|
|
|
const totalDeltaKb =
|
|
|
initialTreeSample && latestTreeSample
|
|
|
? latestTreeSample.rssKb - initialTreeSample.rssKb
|
|
|
: 0;
|
|
|
const topGrowthFiles = [...memoryFileRecords, ...fallbackRecord]
|
|
|
.filter((record) => record.deltaKb > 0 && typeof record.file === "string")
|
|
|
.toSorted((left, right) => right.deltaKb - left.deltaKb)
|
|
|
.slice(0, memoryTraceTopCount)
|
|
|
.map((record) => `${record.file}:${formatMemoryDeltaKb(record.deltaKb)}`);
|
|
|
console.log(
|
|
|
`[test-parallel][mem] summary ${unit.id} files=${memoryFileRecords.length} peak=${formatMemoryKb(
|
|
|
peakTreeSample?.rssKb ?? 0,
|
|
|
)} totalDelta=${formatMemoryDeltaKb(totalDeltaKb)} peakAt=${
|
|
|
peakTreeSample?.reason ?? "n/a"
|
|
|
} top=${topGrowthFiles.length > 0 ? topGrowthFiles.join(", ") : "none"}`,
|
|
|
);
|
|
|
};
|
|
|
const clearChildTimers = () => {
|
|
|
if (memoryPollTimer) {
|
|
|
clearInterval(memoryPollTimer);
|
|
|
memoryPollTimer = null;
|
|
|
}
|
|
|
if (heapSnapshotTimer) {
|
|
|
clearInterval(heapSnapshotTimer);
|
|
|
heapSnapshotTimer = null;
|
|
|
}
|
|
|
if (closeFallbackTimer) {
|
|
|
clearTimeout(closeFallbackTimer);
|
|
|
closeFallbackTimer = null;
|
|
|
}
|
|
|
};
|
|
|
const finalizeRun = (code, signal, source = "close") => {
|
|
|
if (settled) {
|
|
|
return;
|
|
|
}
|
|
|
settled = true;
|
|
|
clearChildTimers();
|
|
|
children.delete(child);
|
|
|
const resolvedCode = resolveTestRunExitCode({
|
|
|
code,
|
|
|
signal,
|
|
|
output,
|
|
|
fatalSeen,
|
|
|
childError,
|
|
|
});
|
|
|
const elapsedMs = Date.now() - startedAt;
|
|
|
logMemoryTraceSummary();
|
|
|
if (resolvedCode !== 0) {
|
|
|
failureTail = formatCapturedOutputTail(output);
|
|
|
failureArtifactPath = artifacts.writeTempJsonArtifact(`${artifactStem}-failure`, {
|
|
|
entry: unit.id,
|
|
|
command: [pnpmInvocation.command, ...spawnArgs],
|
|
|
elapsedMs,
|
|
|
error: childError ? String(childError) : null,
|
|
|
exitCode: resolvedCode,
|
|
|
fatalSeen,
|
|
|
logPath: laneLogPath,
|
|
|
outputTail: failureTail,
|
|
|
signal: signal ?? null,
|
|
|
});
|
|
|
if (failureTail) {
|
|
|
console.error(`[test-parallel] failure tail ${unit.id}\n${failureTail}`);
|
|
|
}
|
|
|
console.error(
|
|
|
`[test-parallel] failure artifacts ${unit.id} log=${laneLogPath} meta=${failureArtifactPath}`,
|
|
|
);
|
|
|
}
|
|
|
if (source !== "close") {
|
|
|
laneLogStream.write(
|
|
|
`\n[test-parallel] finalize source=${source} after child exit without close\n`,
|
|
|
);
|
|
|
}
|
|
|
laneLogStream.write(
|
|
|
`\n[test-parallel] done ${unit.id} code=${String(resolvedCode)} signal=${
|
|
|
signal ?? "none"
|
|
|
} elapsed=${formatElapsedMs(elapsedMs)}\n`,
|
|
|
);
|
|
|
laneLogStream.end();
|
|
|
console.log(
|
|
|
`[test-parallel] done ${unit.id} code=${String(resolvedCode)} elapsed=${formatElapsedMs(elapsedMs)}`,
|
|
|
);
|
|
|
const failedTestFiles = extractFailedTestFiles(output);
|
|
|
const classification = classifyRunResult({
|
|
|
resolvedCode,
|
|
|
signal,
|
|
|
fatalSeen,
|
|
|
childError,
|
|
|
failedTestFiles,
|
|
|
});
|
|
|
resolve({
|
|
|
unitId: unit.id,
|
|
|
shardLabel,
|
|
|
classification,
|
|
|
exitCode: resolvedCode,
|
|
|
signal: signal ?? null,
|
|
|
elapsedMs,
|
|
|
failedTestFiles,
|
|
|
explicitEntryFilters,
|
|
|
failureArtifactPath,
|
|
|
logPath: laneLogPath,
|
|
|
outputTail: failureTail,
|
|
|
});
|
|
|
};
|
|
|
try {
|
|
|
const childEnv = {
|
|
|
...env,
|
|
|
...unit.env,
|
|
|
VITEST_GROUP: unit.id,
|
|
|
NODE_OPTIONS: resolvedNodeOptions,
|
|
|
};
|
|
|
const vitestFsModuleCachePath = resolveVitestFsModuleCachePath({
|
|
|
env: childEnv,
|
|
|
platform: process.platform,
|
|
|
unitId: unit.id,
|
|
|
});
|
|
|
if (vitestFsModuleCachePath) {
|
|
|
childEnv.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH = vitestFsModuleCachePath;
|
|
|
laneLogStream.write(`[test-parallel] fsModuleCachePath=${vitestFsModuleCachePath}\n`);
|
|
|
}
|
|
|
child = spawnImpl(pnpmInvocation.command, spawnArgs, {
|
|
|
stdio: ["inherit", "pipe", "pipe"],
|
|
|
env: childEnv,
|
|
|
shell: false,
|
|
|
});
|
|
|
captureTreeSample("spawn");
|
|
|
if (memoryTraceEnabled) {
|
|
|
memoryPollTimer = setInterval(() => {
|
|
|
captureTreeSample("poll");
|
|
|
}, memoryTracePollMs);
|
|
|
}
|
|
|
if (heapSnapshotEnabled) {
|
|
|
heapSnapshotTimer = setInterval(() => {
|
|
|
triggerHeapSnapshot("interval");
|
|
|
}, heapSnapshotIntervalMs);
|
|
|
}
|
|
|
} catch (err) {
|
|
|
laneLogStream.end();
|
|
|
console.error(`[test-parallel] spawn failed: ${String(err)}`);
|
|
|
resolve({
|
|
|
unitId: unit.id,
|
|
|
shardLabel: getShardLabel(extraArgs),
|
|
|
classification: "infra-failure",
|
|
|
exitCode: 1,
|
|
|
signal: null,
|
|
|
elapsedMs: Date.now() - startedAt,
|
|
|
failedTestFiles: [...explicitEntryFilters],
|
|
|
explicitEntryFilters,
|
|
|
failureArtifactPath: null,
|
|
|
logPath: laneLogPath,
|
|
|
outputTail: String(err),
|
|
|
});
|
|
|
return;
|
|
|
}
|
|
|
children.add(child);
|
|
|
child.stdout?.on("data", (chunk) => {
|
|
|
const text = chunk.toString();
|
|
|
fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`);
|
|
|
output = appendCapturedOutput(output, text);
|
|
|
laneLogStream.write(text);
|
|
|
logMemoryTraceForText(text);
|
|
|
process.stdout.write(chunk);
|
|
|
});
|
|
|
child.stderr?.on("data", (chunk) => {
|
|
|
const text = chunk.toString();
|
|
|
fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`);
|
|
|
output = appendCapturedOutput(output, text);
|
|
|
laneLogStream.write(text);
|
|
|
logMemoryTraceForText(text);
|
|
|
process.stderr.write(chunk);
|
|
|
});
|
|
|
child.on("error", (err) => {
|
|
|
childError = err;
|
|
|
laneLogStream.write(`\n[test-parallel] child error: ${String(err)}\n`);
|
|
|
console.error(`[test-parallel] child error: ${String(err)}`);
|
|
|
});
|
|
|
child.on("exit", (code, signal) => {
|
|
|
childExitState = { code, signal };
|
|
|
if (settled || closeFallbackTimer) {
|
|
|
return;
|
|
|
}
|
|
|
closeFallbackTimer = setTimeout(() => {
|
|
|
child.stdout?.destroy();
|
|
|
child.stderr?.destroy();
|
|
|
finalizeRun(code, signal, "exit-timeout");
|
|
|
}, closeGraceMs);
|
|
|
});
|
|
|
child.on("close", (code, signal) => {
|
|
|
finalizeRun(childExitState?.code ?? code, childExitState?.signal ?? signal, "close");
|
|
|
});
|
|
|
});
|
|
|
|
|
|
const runUnit = async (unit, extraArgs = []) => {
|
|
|
const results = [];
|
|
|
if (unit.fixedShardIndex !== undefined) {
|
|
|
if (plan.shardIndexOverride !== null && plan.shardIndexOverride !== unit.fixedShardIndex) {
|
|
|
return results;
|
|
|
}
|
|
|
results.push(await runOnce(unit, extraArgs));
|
|
|
return results;
|
|
|
}
|
|
|
const explicitFilterCount = countUnitEntryFilters(unit);
|
|
|
const topLevelAssignedShard = plan.topLevelSingleShardAssignments.get(unit);
|
|
|
if (topLevelAssignedShard !== undefined) {
|
|
|
if (plan.shardIndexOverride !== null && plan.shardIndexOverride !== topLevelAssignedShard) {
|
|
|
return results;
|
|
|
}
|
|
|
results.push(await runOnce(unit, extraArgs));
|
|
|
return results;
|
|
|
}
|
|
|
const effectiveShardCount =
|
|
|
explicitFilterCount === null
|
|
|
? plan.shardCount
|
|
|
: Math.min(plan.shardCount, Math.max(1, explicitFilterCount - 1));
|
|
|
if (effectiveShardCount <= 1) {
|
|
|
if (plan.shardIndexOverride !== null && plan.shardIndexOverride > effectiveShardCount) {
|
|
|
return results;
|
|
|
}
|
|
|
results.push(await runOnce(unit, extraArgs));
|
|
|
return results;
|
|
|
}
|
|
|
if (plan.shardIndexOverride !== null) {
|
|
|
if (plan.shardIndexOverride > effectiveShardCount) {
|
|
|
return results;
|
|
|
}
|
|
|
results.push(
|
|
|
await runOnce(unit, [
|
|
|
"--shard",
|
|
|
`${plan.shardIndexOverride}/${effectiveShardCount}`,
|
|
|
...extraArgs,
|
|
|
]),
|
|
|
);
|
|
|
return results;
|
|
|
}
|
|
|
for (let shardIndex = 1; shardIndex <= effectiveShardCount; shardIndex += 1) {
|
|
|
results.push(
|
|
|
// eslint-disable-next-line no-await-in-loop
|
|
|
await runOnce(unit, ["--shard", `${shardIndex}/${effectiveShardCount}`, ...extraArgs]),
|
|
|
);
|
|
|
if (!shouldCollectAllFailures && results.at(-1)?.exitCode !== 0) {
|
|
|
return results;
|
|
|
}
|
|
|
}
|
|
|
return results;
|
|
|
};
|
|
|
|
|
|
const runUnitsWithLimit = async (units, extraArgs = [], concurrency = 1) => {
|
|
|
const results = [];
|
|
|
if (units.length === 0) {
|
|
|
return results;
|
|
|
}
|
|
|
const normalizedConcurrency = Math.max(1, Math.floor(concurrency));
|
|
|
if (normalizedConcurrency <= 1) {
|
|
|
for (const unit of units) {
|
|
|
results.push(
|
|
|
// eslint-disable-next-line no-await-in-loop
|
|
|
...(await runUnit(unit, extraArgs)),
|
|
|
);
|
|
|
if (!shouldCollectAllFailures && results.some((result) => result.exitCode !== 0)) {
|
|
|
return results;
|
|
|
}
|
|
|
}
|
|
|
return results;
|
|
|
}
|
|
|
let nextIndex = 0;
|
|
|
let stopScheduling = false;
|
|
|
const worker = async () => {
|
|
|
while (!stopScheduling) {
|
|
|
const unitIndex = nextIndex;
|
|
|
nextIndex += 1;
|
|
|
if (unitIndex >= units.length) {
|
|
|
return;
|
|
|
}
|
|
|
const unitResults = await runUnit(units[unitIndex], extraArgs);
|
|
|
results.push(...unitResults);
|
|
|
if (!shouldCollectAllFailures && unitResults.some((result) => result.exitCode !== 0)) {
|
|
|
stopScheduling = true;
|
|
|
}
|
|
|
}
|
|
|
};
|
|
|
const workerCount = Math.min(normalizedConcurrency, units.length);
|
|
|
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
|
|
return results;
|
|
|
};
|
|
|
|
|
|
const runUnits = async (units, extraArgs = []) => {
|
|
|
if (plan.topLevelParallelEnabled) {
|
|
|
return runUnitsWithLimit(units, extraArgs, plan.topLevelParallelLimit);
|
|
|
}
|
|
|
return runUnitsWithLimit(units, extraArgs);
|
|
|
};
|
|
|
|
|
|
if (plan.passthroughMetadataOnly) {
|
|
|
const report = buildFinalRunReport([
|
|
|
await runOnce(
|
|
|
{
|
|
|
id: "vitest-meta",
|
|
|
args: ["vitest", "run"],
|
|
|
maxWorkers: null,
|
|
|
},
|
|
|
plan.passthroughOptionArgs,
|
|
|
),
|
|
|
]);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
|
|
|
if (plan.targetedUnits.length > 0) {
|
|
|
if (plan.passthroughRequiresSingleRun && plan.targetedUnits.length > 1) {
|
|
|
console.error(
|
|
|
"[test-parallel] The provided Vitest args require a single run, but the selected test filters span multiple wrapper configs. Run one target/config at a time.",
|
|
|
);
|
|
|
return {
|
|
|
exitCode: 2,
|
|
|
results: [],
|
|
|
summary: {
|
|
|
failedRunCount: 0,
|
|
|
failedUnitCount: 0,
|
|
|
failedTestFileCount: 0,
|
|
|
infraFailureCount: 0,
|
|
|
},
|
|
|
};
|
|
|
}
|
|
|
const results = [];
|
|
|
results.push(...(await runUnits(plan.parallelUnits, plan.passthroughOptionArgs)));
|
|
|
if (!shouldCollectAllFailures && results.some((result) => result.exitCode !== 0)) {
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
for (const unit of plan.serialUnits) {
|
|
|
results.push(
|
|
|
// eslint-disable-next-line no-await-in-loop
|
|
|
...(await runUnit(unit, plan.passthroughOptionArgs)),
|
|
|
);
|
|
|
if (!shouldCollectAllFailures && results.some((result) => result.exitCode !== 0)) {
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
}
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
|
|
|
if (plan.passthroughRequiresSingleRun && plan.passthroughOptionArgs.length > 0) {
|
|
|
console.error(
|
|
|
"[test-parallel] The provided Vitest args require a single run. Use the dedicated npm script for that workflow (for example `pnpm test:coverage`) or target a single test file/filter.",
|
|
|
);
|
|
|
return {
|
|
|
exitCode: 2,
|
|
|
results: [],
|
|
|
summary: {
|
|
|
failedRunCount: 0,
|
|
|
failedUnitCount: 0,
|
|
|
failedTestFileCount: 0,
|
|
|
infraFailureCount: 0,
|
|
|
},
|
|
|
};
|
|
|
}
|
|
|
|
|
|
const results = [];
|
|
|
if (plan.serialPrefixUnits.length > 0) {
|
|
|
const orderedSegments = buildOrderedParallelSegments(plan.parallelUnits);
|
|
|
let pendingDeferredSegment = null;
|
|
|
let carriedDeferredPromise = null;
|
|
|
let carriedDeferredSurface = null;
|
|
|
for (const segment of orderedSegments) {
|
|
|
if (segment.type === "deferred") {
|
|
|
pendingDeferredSegment = segment;
|
|
|
continue;
|
|
|
}
|
|
|
// Preserve phase ordering, but let batches inside the same shared phase use
|
|
|
// the normal top-level concurrency budget.
|
|
|
let deferredPromise = null;
|
|
|
let deferredCarryPromise = carriedDeferredPromise;
|
|
|
let deferredCarrySurface = carriedDeferredSurface;
|
|
|
if (
|
|
|
segment.phase === "unit-fast" &&
|
|
|
pendingDeferredSegment !== null &&
|
|
|
plan.topLevelParallelEnabled
|
|
|
) {
|
|
|
const availableSlots = Math.max(0, plan.topLevelParallelLimit - segment.units.length);
|
|
|
if (availableSlots > 0) {
|
|
|
const prePhaseDeferred = pendingDeferredSegment.units;
|
|
|
if (prePhaseDeferred.length > 0) {
|
|
|
deferredCarryPromise = runUnitsWithLimit(
|
|
|
prePhaseDeferred,
|
|
|
plan.passthroughOptionArgs,
|
|
|
availableSlots,
|
|
|
);
|
|
|
deferredCarrySurface = prePhaseDeferred.some((unit) => unit.surface === "channels")
|
|
|
? "channels"
|
|
|
: null;
|
|
|
pendingDeferredSegment = null;
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
if (pendingDeferredSegment !== null) {
|
|
|
const prioritizedDeferred = prioritizeDeferredUnitsForPhase(
|
|
|
pendingDeferredSegment.units,
|
|
|
segment.phase,
|
|
|
);
|
|
|
if (segment.phase === "extensions") {
|
|
|
const { matching: channelDeferred, remaining: otherDeferred } = partitionUnitsBySurface(
|
|
|
prioritizedDeferred,
|
|
|
"channels",
|
|
|
);
|
|
|
deferredPromise =
|
|
|
otherDeferred.length > 0
|
|
|
? runUnitsWithLimit(
|
|
|
otherDeferred,
|
|
|
plan.passthroughOptionArgs,
|
|
|
plan.deferredRunConcurrency ?? 1,
|
|
|
)
|
|
|
: null;
|
|
|
deferredCarryPromise =
|
|
|
channelDeferred.length > 0
|
|
|
? runUnitsWithLimit(
|
|
|
channelDeferred,
|
|
|
plan.passthroughOptionArgs,
|
|
|
plan.deferredRunConcurrency ?? 1,
|
|
|
)
|
|
|
: carriedDeferredPromise;
|
|
|
deferredCarrySurface = channelDeferred.length > 0 ? "channels" : carriedDeferredSurface;
|
|
|
} else {
|
|
|
deferredPromise = runUnitsWithLimit(
|
|
|
prioritizedDeferred,
|
|
|
plan.passthroughOptionArgs,
|
|
|
plan.deferredRunConcurrency ?? 1,
|
|
|
);
|
|
|
}
|
|
|
}
|
|
|
pendingDeferredSegment = null;
|
|
|
// eslint-disable-next-line no-await-in-loop
|
|
|
const serialPhaseResults = await runUnits(segment.units, plan.passthroughOptionArgs);
|
|
|
results.push(...serialPhaseResults);
|
|
|
if (!shouldCollectAllFailures && serialPhaseResults.some((result) => result.exitCode !== 0)) {
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
if (deferredCarryPromise !== null && deferredCarrySurface === segment.phase) {
|
|
|
const carriedDeferredResults =
|
|
|
// eslint-disable-next-line no-await-in-loop
|
|
|
await deferredCarryPromise;
|
|
|
results.push(...carriedDeferredResults);
|
|
|
if (
|
|
|
!shouldCollectAllFailures &&
|
|
|
carriedDeferredResults.some((result) => result.exitCode !== 0)
|
|
|
) {
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
deferredCarryPromise = null;
|
|
|
deferredCarrySurface = null;
|
|
|
}
|
|
|
if (deferredPromise !== null) {
|
|
|
const deferredResults =
|
|
|
// eslint-disable-next-line no-await-in-loop
|
|
|
await deferredPromise;
|
|
|
results.push(...deferredResults);
|
|
|
if (!shouldCollectAllFailures && deferredResults.some((result) => result.exitCode !== 0)) {
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
}
|
|
|
carriedDeferredPromise = deferredCarryPromise;
|
|
|
carriedDeferredSurface = deferredCarrySurface;
|
|
|
}
|
|
|
if (pendingDeferredSegment !== null) {
|
|
|
const deferredParallelResults = await runUnitsWithLimit(
|
|
|
pendingDeferredSegment.units,
|
|
|
plan.passthroughOptionArgs,
|
|
|
plan.deferredRunConcurrency ?? 1,
|
|
|
);
|
|
|
results.push(...deferredParallelResults);
|
|
|
if (
|
|
|
!shouldCollectAllFailures &&
|
|
|
deferredParallelResults.some((result) => result.exitCode !== 0)
|
|
|
) {
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
}
|
|
|
if (carriedDeferredPromise !== null) {
|
|
|
const carriedDeferredResults = await carriedDeferredPromise;
|
|
|
results.push(...carriedDeferredResults);
|
|
|
if (
|
|
|
!shouldCollectAllFailures &&
|
|
|
carriedDeferredResults.some((result) => result.exitCode !== 0)
|
|
|
) {
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
}
|
|
|
} else {
|
|
|
const parallelResults = await runUnits(plan.parallelUnits, plan.passthroughOptionArgs);
|
|
|
results.push(...parallelResults);
|
|
|
if (!shouldCollectAllFailures && parallelResults.some((result) => result.exitCode !== 0)) {
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
for (const unit of plan.serialUnits) {
|
|
|
results.push(
|
|
|
// eslint-disable-next-line no-await-in-loop
|
|
|
...(await runUnit(unit, plan.passthroughOptionArgs)),
|
|
|
);
|
|
|
if (!shouldCollectAllFailures && results.some((result) => result.exitCode !== 0)) {
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|
|
|
}
|
|
|
const report = buildFinalRunReport(results);
|
|
|
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
|
|
|
printFinalRunSummary(plan, report, reportArtifactPath);
|
|
|
return report;
|
|
|
}
|