perf: parallelize local check gate

This commit is contained in:
Peter Steinberger
2026-04-20 13:55:48 +01:00
parent a1bd02fdfd
commit 46ae3d314a
14 changed files with 246 additions and 72 deletions

View File

@@ -140,6 +140,10 @@
- Do not add `tsc --noEmit`, `typecheck`, or `check:types` lanes for repo type checking. Use `tsgo` graphs. `tsc` is allowed only when emitting declaration/package-boundary compatibility artifacts that `tsgo` does not replace.
- Boundary rule: core must not know extension implementation details. Extensions hook into core through manifests, registries, capabilities, and public `openclaw/plugin-sdk/*` contracts. If you find core production code naming a specific extension, or a core test that is really testing extension-owned behavior, call it out and prefer moving coverage/logic to the owning extension or a generic contract test.
- Lint/format: `pnpm check`
- `pnpm lint`: type-aware lint shards for core, extensions, and scripts, run in parallel after shared boundary artifacts are prepared once.
- `pnpm lint:core`, `pnpm lint:extensions`, `pnpm lint:scripts`: focused lint shards.
- `pnpm lint:all`: legacy single-pass repo-wide oxlint, useful when comparing shard behavior.
- `pnpm lint:apps`: app lint surface such as Swift; keep app lint separate from repo TypeScript lint.
- Local agent/dev shells default to host-aware `OPENCLAW_LOCAL_CHECK=1` behavior for `pnpm tsgo` and `pnpm lint`; set `OPENCLAW_LOCAL_CHECK_MODE=throttled` to force the lower-memory profile, `OPENCLAW_LOCAL_CHECK_MODE=full` to keep lock-only behavior, or `OPENCLAW_LOCAL_CHECK=0` in CI/shared runs.
- Format check: `pnpm format:check` (oxfmt --check)
- Format fix: `pnpm format` or `pnpm format:fix` (oxfmt --write)
@@ -148,7 +152,7 @@
- A local dev gate is the fast default loop, usually `pnpm check` plus any scoped test you actually need.
- A landing gate is the broader bar before pushing `main`, usually `pnpm check`, `pnpm test`, and `pnpm build` when the touched surface can affect build output, packaging, lazy-loading/module boundaries, or published surfaces.
- A CI gate is whatever the relevant workflow enforces for that lane (for example `check`, `check-additional`, `build-smoke`, or release validation).
- Local dev gate: prefer `pnpm check` for the normal edit loop. It keeps the repo-architecture policy guards out of the default local loop.
- Local dev gate: prefer `pnpm check` for the normal edit loop. It runs typecheck and lint first, then parallelizes independent policy guards. It keeps the repo-architecture policy guards out of the default local loop.
- Timed local gate: use `pnpm check:timed` to see per-stage cost. Add `:architecture` only when investigating the CI architecture gate locally.
- CI architecture gate: `check-additional` enforces architecture and boundary policy guards that are intentionally kept out of the default local loop.
- Formatting gate: the pre-commit hook runs targeted formatting on staged source files before `pnpm check`. If you want a repo-wide formatting-only preflight locally, run `pnpm format:check` explicitly.

View File

@@ -57,7 +57,7 @@ On pushes, the `checks` matrix adds the push-only `compat-node22` lane. On pull
## Local Equivalents
```bash
pnpm check # fast local gate: project-reference tsgo + lint + fast guards
pnpm check # fast local gate: project-reference tsgo + sharded lint + parallel fast guards
pnpm check:timed # same gate with per-stage timings
pnpm build:strict-smoke
pnpm check:architecture

View File

@@ -1236,7 +1236,7 @@
"canon:check:json": "node scripts/canon.mjs check --json",
"canon:enforce": "node scripts/canon.mjs enforce --json",
"canvas:a2ui:bundle": "node scripts/bundle-a2ui.mjs",
"check": "pnpm check:no-conflict-markers && pnpm tool-display:check && pnpm check:host-env-policy:swift && pnpm tsgo:all && pnpm lint && pnpm lint:webhook:no-low-level-body-read && pnpm lint:auth:no-pairing-store-group && pnpm lint:auth:pairing-account-scope && pnpm check:import-cycles",
"check": "node scripts/check.mjs",
"check:architecture": "pnpm check:import-cycles && pnpm check:madge-import-cycles",
"check:base-config-schema": "node --import tsx scripts/generate-base-config-schema.ts --check",
"check:bundled-channel-config-metadata": "node --import tsx scripts/generate-bundled-channel-config-metadata.ts --check",
@@ -1300,13 +1300,16 @@
"ios:version:check": "node --import tsx scripts/ios-sync-versioning.ts --check",
"ios:version:pin": "node --import tsx scripts/ios-pin-version.ts",
"ios:version:sync": "node --import tsx scripts/ios-sync-versioning.ts --write",
"lint": "node scripts/run-oxlint.mjs",
"lint": "node scripts/run-oxlint-shards.mjs",
"lint:agent:ingress-owner": "node scripts/check-ingress-agent-owner-context.mjs",
"lint:all": "pnpm lint && pnpm lint:swift",
"lint:all": "node scripts/run-oxlint.mjs",
"lint:apps": "pnpm lint:swift",
"lint:auth:no-pairing-store-group": "node scripts/check-no-pairing-store-group-auth.mjs",
"lint:auth:pairing-account-scope": "node scripts/check-pairing-account-scope.mjs",
"lint:core": "node scripts/run-oxlint.mjs --tsconfig tsconfig.oxlint.core.json src ui packages",
"lint:docs": "pnpm dlx markdownlint-cli2",
"lint:docs:fix": "pnpm dlx markdownlint-cli2 --fix",
"lint:extensions": "node scripts/run-oxlint.mjs --tsconfig tsconfig.oxlint.extensions.json extensions",
"lint:extensions:bundled": "node scripts/run-bundled-extension-oxlint.mjs",
"lint:extensions:channels": "node scripts/run-extension-channel-oxlint.mjs",
"lint:extensions:no-plugin-sdk-internal": "node scripts/check-extension-plugin-sdk-boundary.mjs --mode=plugin-sdk-internal",
@@ -1319,6 +1322,7 @@
"lint:plugins:no-monolithic-plugin-sdk-entry-imports": "node --import tsx scripts/check-no-monolithic-plugin-sdk-entry-imports.ts",
"lint:plugins:no-register-http-handler": "node scripts/check-no-register-http-handler.mjs",
"lint:plugins:plugin-sdk-subpaths-exported": "node scripts/check-plugin-sdk-subpath-exports.mjs",
"lint:scripts": "node scripts/run-oxlint.mjs --tsconfig tsconfig.oxlint.scripts.json scripts",
"lint:swift": "swiftlint lint --config .swiftlint.yml && (cd apps/ios && swiftlint lint --config .swiftlint.yml)",
"lint:tmp:channel-agnostic-boundaries": "node scripts/check-channel-agnostic-boundaries.mjs",
"lint:tmp:dynamic-import-warts": "node scripts/check-dynamic-import-warts.mjs",

View File

@@ -1,57 +1,3 @@
import { spawnSync } from "node:child_process";
import { performance } from "node:perf_hooks";
import { main } from "./check.mjs";
const includeArchitecture = process.argv.includes("--include-architecture");
const stages = [
{ name: "conflict markers", args: ["check:no-conflict-markers"] },
{ name: "tool display", args: ["tool-display:check"] },
{ name: "host env policy", args: ["check:host-env-policy:swift"] },
{ name: "typecheck", args: ["tsgo:all"] },
{ name: "lint", args: ["lint"] },
{ name: "webhook body guard", args: ["lint:webhook:no-low-level-body-read"] },
{ name: "pairing store guard", args: ["lint:auth:no-pairing-store-group"] },
{ name: "pairing account guard", args: ["lint:auth:pairing-account-scope"] },
{ name: "runtime import cycles", args: ["check:import-cycles"] },
];
if (includeArchitecture) {
stages.push({ name: "architecture import cycles", args: ["check:madge-import-cycles"] });
}
const timings = [];
let exitCode = 0;
for (const { name, args } of stages) {
const startedAt = performance.now();
console.error(`\n[check:timed] ${name}`);
const result = spawnSync("pnpm", args, {
stdio: "inherit",
shell: process.platform === "win32",
});
const durationMs = performance.now() - startedAt;
timings.push({ name, durationMs, status: result.status ?? 1 });
if (result.error) {
throw result.error;
}
if (result.status !== 0) {
exitCode = result.status ?? 1;
break;
}
}
console.error("\n[check:timed] summary");
for (const timing of timings) {
const status = timing.status === 0 ? "ok" : `failed:${timing.status}`;
console.error(`${formatMs(timing.durationMs).padStart(8)} ${status.padEnd(9)} ${timing.name}`);
}
process.exitCode = exitCode;
function formatMs(durationMs) {
if (durationMs < 1000) {
return `${Math.round(durationMs)}ms`;
}
return `${(durationMs / 1000).toFixed(2)}s`;
}
await main([...process.argv.slice(2), "--timed"]);

125
scripts/check.mjs Normal file
View File

@@ -0,0 +1,125 @@
import { spawn } from "node:child_process";
import { performance } from "node:perf_hooks";
export async function main(argv = process.argv.slice(2)) {
const timed = argv.includes("--timed");
const includeArchitecture = argv.includes("--include-architecture");
const tailChecks = [
{ name: "webhook body guard", args: ["lint:webhook:no-low-level-body-read"] },
{ name: "pairing store guard", args: ["lint:auth:no-pairing-store-group"] },
{ name: "pairing account guard", args: ["lint:auth:pairing-account-scope"] },
includeArchitecture
? { name: "architecture import cycles", args: ["check:architecture"] }
: { name: "runtime import cycles", args: ["check:import-cycles"] },
];
const stages = [
{
name: "preflight guards",
parallel: false,
commands: [
{ name: "conflict markers", args: ["check:no-conflict-markers"] },
{ name: "tool display", args: ["tool-display:check"] },
{ name: "host env policy", args: ["check:host-env-policy:swift"] },
],
},
{
name: "typecheck",
parallel: false,
commands: [{ name: "typecheck", args: ["tsgo:all"] }],
},
{
name: "lint",
parallel: false,
commands: [{ name: "lint", args: ["lint"] }],
},
{
name: "policy guards",
parallel: true,
commands: tailChecks,
},
];
const timings = [];
let exitCode = 0;
for (const stage of stages) {
console.error(`\n[check] ${stage.name}`);
const results = stage.parallel
? await Promise.all(stage.commands.map((command) => runCommand(command)))
: await runSerial(stage.commands);
timings.push(...results);
const failed = results.find((result) => result.status !== 0);
if (failed) {
exitCode = failed.status;
break;
}
}
if (timed || exitCode !== 0) {
printSummary(timings);
}
process.exitCode = exitCode;
}
async function runSerial(commands) {
const results = [];
for (const command of commands) {
const result = await runCommand(command);
results.push(result);
if (result.status !== 0) {
break;
}
}
return results;
}
async function runCommand(command) {
const startedAt = performance.now();
const child = spawn("pnpm", command.args, {
stdio: "inherit",
shell: process.platform === "win32",
});
return await new Promise((resolve) => {
child.once("error", (error) => {
console.error(error);
resolve({
name: command.name,
durationMs: performance.now() - startedAt,
status: 1,
});
});
child.once("close", (status) => {
resolve({
name: command.name,
durationMs: performance.now() - startedAt,
status: status ?? 1,
});
});
});
}
function printSummary(timings) {
console.error("\n[check] summary");
for (const timing of timings) {
const status = timing.status === 0 ? "ok" : `failed:${timing.status}`;
console.error(
`${formatMs(timing.durationMs).padStart(8)} ${status.padEnd(9)} ${timing.name}`,
);
}
}
function formatMs(durationMs) {
if (durationMs < 1000) {
return `${Math.round(durationMs)}ms`;
}
return `${(durationMs / 1000).toFixed(2)}s`;
}
if (import.meta.main) {
await main();
}

View File

@@ -0,0 +1,62 @@
import { spawn, spawnSync } from "node:child_process";
import path from "node:path";
const extraArgs = process.argv.slice(2);
const runner = path.resolve("scripts", "run-oxlint.mjs");
const prepareResult = spawnSync(
process.execPath,
[path.resolve("scripts", "prepare-extension-package-boundary-artifacts.mjs")],
{
stdio: "inherit",
env: process.env,
},
);
if (prepareResult.error) {
throw prepareResult.error;
}
if ((prepareResult.status ?? 1) !== 0) {
process.exit(prepareResult.status ?? 1);
}
const shards = [
{
name: "core",
args: ["--tsconfig", "tsconfig.oxlint.core.json", "src", "ui", "packages"],
},
{
name: "extensions",
args: ["--tsconfig", "tsconfig.oxlint.extensions.json", "extensions"],
},
{
name: "scripts",
args: ["--tsconfig", "tsconfig.oxlint.scripts.json", "scripts"],
},
];
const results = await Promise.all(shards.map((shard) => runShard(shard)));
process.exitCode = results.find((status) => status !== 0) ?? 0;
async function runShard(shard) {
console.error(`[oxlint:${shard.name}] starting`);
const child = spawn(process.execPath, [runner, ...shard.args, ...extraArgs], {
stdio: "inherit",
env: {
...process.env,
OPENCLAW_OXLINT_SKIP_LOCK: "1",
OPENCLAW_OXLINT_SKIP_PREPARE: "1",
},
});
return await new Promise((resolve) => {
child.once("error", (error) => {
console.error(error);
resolve(1);
});
child.once("close", (status) => {
console.error(`[oxlint:${shard.name}] finished`);
resolve(status ?? 1);
});
});
}

View File

@@ -54,19 +54,25 @@ function prepareExtensionPackageBoundaryArtifacts(env) {
export function main(argv = process.argv.slice(2), runtimeEnv = process.env) {
const { args: finalArgs, env } = applyLocalOxlintPolicy(argv, runtimeEnv);
const releaseLock = shouldAcquireLocalHeavyCheckLockForOxlint(finalArgs, {
cwd: process.cwd(),
env,
})
? acquireLocalHeavyCheckLockSync({
cwd: process.cwd(),
env,
toolName: "oxlint",
})
: () => {};
const releaseLock =
env.OPENCLAW_OXLINT_SKIP_LOCK === "1"
? () => {}
: shouldAcquireLocalHeavyCheckLockForOxlint(finalArgs, {
cwd: process.cwd(),
env,
})
? acquireLocalHeavyCheckLockSync({
cwd: process.cwd(),
env,
toolName: "oxlint",
})
: () => {};
try {
if (shouldPrepareExtensionPackageBoundaryArtifacts(finalArgs)) {
if (
env.OPENCLAW_OXLINT_SKIP_PREPARE !== "1" &&
shouldPrepareExtensionPackageBoundaryArtifacts(finalArgs)
) {
prepareExtensionPackageBoundaryArtifacts(env);
}

View File

@@ -1,5 +1,8 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"tsBuildInfoFile": ".artifacts/tsgo-cache/core.tsbuildinfo"
},
"include": ["src/**/*", "ui/**/*", "packages/**/*"],
"exclude": ["node_modules", "dist", "**/dist/**", "**/*.test.ts", "**/*.test.tsx", "test/**"]
}

View File

@@ -1,5 +1,8 @@
{
"extends": "./tsconfig.test.json",
"compilerOptions": {
"tsBuildInfoFile": ".artifacts/tsgo-cache/core-test.tsbuildinfo"
},
"include": [
"src/**/*.d.ts",
"src/**/*.test.ts",

View File

@@ -1,5 +1,8 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"tsBuildInfoFile": ".artifacts/tsgo-cache/extensions.tsbuildinfo"
},
"include": ["src/**/*.d.ts", "ui/src/**/*.d.ts", "extensions/**/*"],
"exclude": ["node_modules", "dist", "**/dist/**", "**/*.test.ts", "**/*.test.tsx", "test/**"]
}

View File

@@ -1,5 +1,8 @@
{
"extends": "./tsconfig.test.json",
"compilerOptions": {
"tsBuildInfoFile": ".artifacts/tsgo-cache/extensions-test.tsbuildinfo"
},
"include": [
"src/**/*.d.ts",
"ui/**/*.d.ts",

View File

@@ -0,0 +1,5 @@
{
"extends": "./tsconfig.json",
"include": ["src/**/*", "ui/**/*", "packages/**/*"],
"exclude": ["node_modules", "dist", "dist-runtime"]
}

View File

@@ -0,0 +1,5 @@
{
"extends": "./tsconfig.json",
"include": ["src/**/*", "ui/src/**/*", "packages/**/*.d.ts", "extensions/**/*"],
"exclude": ["node_modules", "dist", "dist-runtime"]
}

View File

@@ -0,0 +1,5 @@
{
"extends": "./tsconfig.json",
"include": ["src/**/*.d.ts", "packages/**/*.d.ts", "scripts/**/*"],
"exclude": ["node_modules", "dist", "dist-runtime"]
}