mirror of
https://fastgit.cc/github.com/openclaw/openclaw
synced 2026-04-20 21:02:10 +08:00
perf: parallelize local check gate
This commit is contained in:
@@ -140,6 +140,10 @@
|
||||
- Do not add `tsc --noEmit`, `typecheck`, or `check:types` lanes for repo type checking. Use `tsgo` graphs. `tsc` is allowed only when emitting declaration/package-boundary compatibility artifacts that `tsgo` does not replace.
|
||||
- Boundary rule: core must not know extension implementation details. Extensions hook into core through manifests, registries, capabilities, and public `openclaw/plugin-sdk/*` contracts. If you find core production code naming a specific extension, or a core test that is really testing extension-owned behavior, call it out and prefer moving coverage/logic to the owning extension or a generic contract test.
|
||||
- Lint/format: `pnpm check`
|
||||
- `pnpm lint`: type-aware lint shards for core, extensions, and scripts, run in parallel after shared boundary artifacts are prepared once.
|
||||
- `pnpm lint:core`, `pnpm lint:extensions`, `pnpm lint:scripts`: focused lint shards.
|
||||
- `pnpm lint:all`: legacy single-pass repo-wide oxlint, useful when comparing shard behavior.
|
||||
- `pnpm lint:apps`: app lint surface such as Swift; keep app lint separate from repo TypeScript lint.
|
||||
- Local agent/dev shells default to host-aware `OPENCLAW_LOCAL_CHECK=1` behavior for `pnpm tsgo` and `pnpm lint`; set `OPENCLAW_LOCAL_CHECK_MODE=throttled` to force the lower-memory profile, `OPENCLAW_LOCAL_CHECK_MODE=full` to keep lock-only behavior, or `OPENCLAW_LOCAL_CHECK=0` in CI/shared runs.
|
||||
- Format check: `pnpm format:check` (oxfmt --check)
|
||||
- Format fix: `pnpm format` or `pnpm format:fix` (oxfmt --write)
|
||||
@@ -148,7 +152,7 @@
|
||||
- A local dev gate is the fast default loop, usually `pnpm check` plus any scoped test you actually need.
|
||||
- A landing gate is the broader bar before pushing `main`, usually `pnpm check`, `pnpm test`, and `pnpm build` when the touched surface can affect build output, packaging, lazy-loading/module boundaries, or published surfaces.
|
||||
- A CI gate is whatever the relevant workflow enforces for that lane (for example `check`, `check-additional`, `build-smoke`, or release validation).
|
||||
- Local dev gate: prefer `pnpm check` for the normal edit loop. It keeps the repo-architecture policy guards out of the default local loop.
|
||||
- Local dev gate: prefer `pnpm check` for the normal edit loop. It runs typecheck and lint first, then parallelizes independent policy guards. It keeps the repo-architecture policy guards out of the default local loop.
|
||||
- Timed local gate: use `pnpm check:timed` to see per-stage cost. Add `:architecture` only when investigating the CI architecture gate locally.
|
||||
- CI architecture gate: `check-additional` enforces architecture and boundary policy guards that are intentionally kept out of the default local loop.
|
||||
- Formatting gate: the pre-commit hook runs targeted formatting on staged source files before `pnpm check`. If you want a repo-wide formatting-only preflight locally, run `pnpm format:check` explicitly.
|
||||
|
||||
@@ -57,7 +57,7 @@ On pushes, the `checks` matrix adds the push-only `compat-node22` lane. On pull
|
||||
## Local Equivalents
|
||||
|
||||
```bash
|
||||
pnpm check # fast local gate: project-reference tsgo + lint + fast guards
|
||||
pnpm check # fast local gate: project-reference tsgo + sharded lint + parallel fast guards
|
||||
pnpm check:timed # same gate with per-stage timings
|
||||
pnpm build:strict-smoke
|
||||
pnpm check:architecture
|
||||
|
||||
10
package.json
10
package.json
@@ -1236,7 +1236,7 @@
|
||||
"canon:check:json": "node scripts/canon.mjs check --json",
|
||||
"canon:enforce": "node scripts/canon.mjs enforce --json",
|
||||
"canvas:a2ui:bundle": "node scripts/bundle-a2ui.mjs",
|
||||
"check": "pnpm check:no-conflict-markers && pnpm tool-display:check && pnpm check:host-env-policy:swift && pnpm tsgo:all && pnpm lint && pnpm lint:webhook:no-low-level-body-read && pnpm lint:auth:no-pairing-store-group && pnpm lint:auth:pairing-account-scope && pnpm check:import-cycles",
|
||||
"check": "node scripts/check.mjs",
|
||||
"check:architecture": "pnpm check:import-cycles && pnpm check:madge-import-cycles",
|
||||
"check:base-config-schema": "node --import tsx scripts/generate-base-config-schema.ts --check",
|
||||
"check:bundled-channel-config-metadata": "node --import tsx scripts/generate-bundled-channel-config-metadata.ts --check",
|
||||
@@ -1300,13 +1300,16 @@
|
||||
"ios:version:check": "node --import tsx scripts/ios-sync-versioning.ts --check",
|
||||
"ios:version:pin": "node --import tsx scripts/ios-pin-version.ts",
|
||||
"ios:version:sync": "node --import tsx scripts/ios-sync-versioning.ts --write",
|
||||
"lint": "node scripts/run-oxlint.mjs",
|
||||
"lint": "node scripts/run-oxlint-shards.mjs",
|
||||
"lint:agent:ingress-owner": "node scripts/check-ingress-agent-owner-context.mjs",
|
||||
"lint:all": "pnpm lint && pnpm lint:swift",
|
||||
"lint:all": "node scripts/run-oxlint.mjs",
|
||||
"lint:apps": "pnpm lint:swift",
|
||||
"lint:auth:no-pairing-store-group": "node scripts/check-no-pairing-store-group-auth.mjs",
|
||||
"lint:auth:pairing-account-scope": "node scripts/check-pairing-account-scope.mjs",
|
||||
"lint:core": "node scripts/run-oxlint.mjs --tsconfig tsconfig.oxlint.core.json src ui packages",
|
||||
"lint:docs": "pnpm dlx markdownlint-cli2",
|
||||
"lint:docs:fix": "pnpm dlx markdownlint-cli2 --fix",
|
||||
"lint:extensions": "node scripts/run-oxlint.mjs --tsconfig tsconfig.oxlint.extensions.json extensions",
|
||||
"lint:extensions:bundled": "node scripts/run-bundled-extension-oxlint.mjs",
|
||||
"lint:extensions:channels": "node scripts/run-extension-channel-oxlint.mjs",
|
||||
"lint:extensions:no-plugin-sdk-internal": "node scripts/check-extension-plugin-sdk-boundary.mjs --mode=plugin-sdk-internal",
|
||||
@@ -1319,6 +1322,7 @@
|
||||
"lint:plugins:no-monolithic-plugin-sdk-entry-imports": "node --import tsx scripts/check-no-monolithic-plugin-sdk-entry-imports.ts",
|
||||
"lint:plugins:no-register-http-handler": "node scripts/check-no-register-http-handler.mjs",
|
||||
"lint:plugins:plugin-sdk-subpaths-exported": "node scripts/check-plugin-sdk-subpath-exports.mjs",
|
||||
"lint:scripts": "node scripts/run-oxlint.mjs --tsconfig tsconfig.oxlint.scripts.json scripts",
|
||||
"lint:swift": "swiftlint lint --config .swiftlint.yml && (cd apps/ios && swiftlint lint --config .swiftlint.yml)",
|
||||
"lint:tmp:channel-agnostic-boundaries": "node scripts/check-channel-agnostic-boundaries.mjs",
|
||||
"lint:tmp:dynamic-import-warts": "node scripts/check-dynamic-import-warts.mjs",
|
||||
|
||||
@@ -1,57 +1,3 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { main } from "./check.mjs";
|
||||
|
||||
const includeArchitecture = process.argv.includes("--include-architecture");
|
||||
|
||||
const stages = [
|
||||
{ name: "conflict markers", args: ["check:no-conflict-markers"] },
|
||||
{ name: "tool display", args: ["tool-display:check"] },
|
||||
{ name: "host env policy", args: ["check:host-env-policy:swift"] },
|
||||
{ name: "typecheck", args: ["tsgo:all"] },
|
||||
{ name: "lint", args: ["lint"] },
|
||||
{ name: "webhook body guard", args: ["lint:webhook:no-low-level-body-read"] },
|
||||
{ name: "pairing store guard", args: ["lint:auth:no-pairing-store-group"] },
|
||||
{ name: "pairing account guard", args: ["lint:auth:pairing-account-scope"] },
|
||||
{ name: "runtime import cycles", args: ["check:import-cycles"] },
|
||||
];
|
||||
|
||||
if (includeArchitecture) {
|
||||
stages.push({ name: "architecture import cycles", args: ["check:madge-import-cycles"] });
|
||||
}
|
||||
|
||||
const timings = [];
|
||||
let exitCode = 0;
|
||||
|
||||
for (const { name, args } of stages) {
|
||||
const startedAt = performance.now();
|
||||
console.error(`\n[check:timed] ${name}`);
|
||||
const result = spawnSync("pnpm", args, {
|
||||
stdio: "inherit",
|
||||
shell: process.platform === "win32",
|
||||
});
|
||||
const durationMs = performance.now() - startedAt;
|
||||
timings.push({ name, durationMs, status: result.status ?? 1 });
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
if (result.status !== 0) {
|
||||
exitCode = result.status ?? 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
console.error("\n[check:timed] summary");
|
||||
for (const timing of timings) {
|
||||
const status = timing.status === 0 ? "ok" : `failed:${timing.status}`;
|
||||
console.error(`${formatMs(timing.durationMs).padStart(8)} ${status.padEnd(9)} ${timing.name}`);
|
||||
}
|
||||
|
||||
process.exitCode = exitCode;
|
||||
|
||||
function formatMs(durationMs) {
|
||||
if (durationMs < 1000) {
|
||||
return `${Math.round(durationMs)}ms`;
|
||||
}
|
||||
return `${(durationMs / 1000).toFixed(2)}s`;
|
||||
}
|
||||
await main([...process.argv.slice(2), "--timed"]);
|
||||
|
||||
125
scripts/check.mjs
Normal file
125
scripts/check.mjs
Normal file
@@ -0,0 +1,125 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { performance } from "node:perf_hooks";
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const timed = argv.includes("--timed");
|
||||
const includeArchitecture = argv.includes("--include-architecture");
|
||||
|
||||
const tailChecks = [
|
||||
{ name: "webhook body guard", args: ["lint:webhook:no-low-level-body-read"] },
|
||||
{ name: "pairing store guard", args: ["lint:auth:no-pairing-store-group"] },
|
||||
{ name: "pairing account guard", args: ["lint:auth:pairing-account-scope"] },
|
||||
includeArchitecture
|
||||
? { name: "architecture import cycles", args: ["check:architecture"] }
|
||||
: { name: "runtime import cycles", args: ["check:import-cycles"] },
|
||||
];
|
||||
|
||||
const stages = [
|
||||
{
|
||||
name: "preflight guards",
|
||||
parallel: false,
|
||||
commands: [
|
||||
{ name: "conflict markers", args: ["check:no-conflict-markers"] },
|
||||
{ name: "tool display", args: ["tool-display:check"] },
|
||||
{ name: "host env policy", args: ["check:host-env-policy:swift"] },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "typecheck",
|
||||
parallel: false,
|
||||
commands: [{ name: "typecheck", args: ["tsgo:all"] }],
|
||||
},
|
||||
{
|
||||
name: "lint",
|
||||
parallel: false,
|
||||
commands: [{ name: "lint", args: ["lint"] }],
|
||||
},
|
||||
{
|
||||
name: "policy guards",
|
||||
parallel: true,
|
||||
commands: tailChecks,
|
||||
},
|
||||
];
|
||||
|
||||
const timings = [];
|
||||
let exitCode = 0;
|
||||
|
||||
for (const stage of stages) {
|
||||
console.error(`\n[check] ${stage.name}`);
|
||||
const results = stage.parallel
|
||||
? await Promise.all(stage.commands.map((command) => runCommand(command)))
|
||||
: await runSerial(stage.commands);
|
||||
|
||||
timings.push(...results);
|
||||
const failed = results.find((result) => result.status !== 0);
|
||||
if (failed) {
|
||||
exitCode = failed.status;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (timed || exitCode !== 0) {
|
||||
printSummary(timings);
|
||||
}
|
||||
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
|
||||
async function runSerial(commands) {
|
||||
const results = [];
|
||||
for (const command of commands) {
|
||||
const result = await runCommand(command);
|
||||
results.push(result);
|
||||
if (result.status !== 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async function runCommand(command) {
|
||||
const startedAt = performance.now();
|
||||
const child = spawn("pnpm", command.args, {
|
||||
stdio: "inherit",
|
||||
shell: process.platform === "win32",
|
||||
});
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
child.once("error", (error) => {
|
||||
console.error(error);
|
||||
resolve({
|
||||
name: command.name,
|
||||
durationMs: performance.now() - startedAt,
|
||||
status: 1,
|
||||
});
|
||||
});
|
||||
child.once("close", (status) => {
|
||||
resolve({
|
||||
name: command.name,
|
||||
durationMs: performance.now() - startedAt,
|
||||
status: status ?? 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function printSummary(timings) {
|
||||
console.error("\n[check] summary");
|
||||
for (const timing of timings) {
|
||||
const status = timing.status === 0 ? "ok" : `failed:${timing.status}`;
|
||||
console.error(
|
||||
`${formatMs(timing.durationMs).padStart(8)} ${status.padEnd(9)} ${timing.name}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function formatMs(durationMs) {
|
||||
if (durationMs < 1000) {
|
||||
return `${Math.round(durationMs)}ms`;
|
||||
}
|
||||
return `${(durationMs / 1000).toFixed(2)}s`;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
await main();
|
||||
}
|
||||
62
scripts/run-oxlint-shards.mjs
Normal file
62
scripts/run-oxlint-shards.mjs
Normal file
@@ -0,0 +1,62 @@
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import path from "node:path";
|
||||
|
||||
const extraArgs = process.argv.slice(2);
|
||||
const runner = path.resolve("scripts", "run-oxlint.mjs");
|
||||
|
||||
const prepareResult = spawnSync(
|
||||
process.execPath,
|
||||
[path.resolve("scripts", "prepare-extension-package-boundary-artifacts.mjs")],
|
||||
{
|
||||
stdio: "inherit",
|
||||
env: process.env,
|
||||
},
|
||||
);
|
||||
|
||||
if (prepareResult.error) {
|
||||
throw prepareResult.error;
|
||||
}
|
||||
if ((prepareResult.status ?? 1) !== 0) {
|
||||
process.exit(prepareResult.status ?? 1);
|
||||
}
|
||||
|
||||
const shards = [
|
||||
{
|
||||
name: "core",
|
||||
args: ["--tsconfig", "tsconfig.oxlint.core.json", "src", "ui", "packages"],
|
||||
},
|
||||
{
|
||||
name: "extensions",
|
||||
args: ["--tsconfig", "tsconfig.oxlint.extensions.json", "extensions"],
|
||||
},
|
||||
{
|
||||
name: "scripts",
|
||||
args: ["--tsconfig", "tsconfig.oxlint.scripts.json", "scripts"],
|
||||
},
|
||||
];
|
||||
|
||||
const results = await Promise.all(shards.map((shard) => runShard(shard)));
|
||||
process.exitCode = results.find((status) => status !== 0) ?? 0;
|
||||
|
||||
async function runShard(shard) {
|
||||
console.error(`[oxlint:${shard.name}] starting`);
|
||||
const child = spawn(process.execPath, [runner, ...shard.args, ...extraArgs], {
|
||||
stdio: "inherit",
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_OXLINT_SKIP_LOCK: "1",
|
||||
OPENCLAW_OXLINT_SKIP_PREPARE: "1",
|
||||
},
|
||||
});
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
child.once("error", (error) => {
|
||||
console.error(error);
|
||||
resolve(1);
|
||||
});
|
||||
child.once("close", (status) => {
|
||||
console.error(`[oxlint:${shard.name}] finished`);
|
||||
resolve(status ?? 1);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -54,19 +54,25 @@ function prepareExtensionPackageBoundaryArtifacts(env) {
|
||||
|
||||
export function main(argv = process.argv.slice(2), runtimeEnv = process.env) {
|
||||
const { args: finalArgs, env } = applyLocalOxlintPolicy(argv, runtimeEnv);
|
||||
const releaseLock = shouldAcquireLocalHeavyCheckLockForOxlint(finalArgs, {
|
||||
cwd: process.cwd(),
|
||||
env,
|
||||
})
|
||||
? acquireLocalHeavyCheckLockSync({
|
||||
cwd: process.cwd(),
|
||||
env,
|
||||
toolName: "oxlint",
|
||||
})
|
||||
: () => {};
|
||||
const releaseLock =
|
||||
env.OPENCLAW_OXLINT_SKIP_LOCK === "1"
|
||||
? () => {}
|
||||
: shouldAcquireLocalHeavyCheckLockForOxlint(finalArgs, {
|
||||
cwd: process.cwd(),
|
||||
env,
|
||||
})
|
||||
? acquireLocalHeavyCheckLockSync({
|
||||
cwd: process.cwd(),
|
||||
env,
|
||||
toolName: "oxlint",
|
||||
})
|
||||
: () => {};
|
||||
|
||||
try {
|
||||
if (shouldPrepareExtensionPackageBoundaryArtifacts(finalArgs)) {
|
||||
if (
|
||||
env.OPENCLAW_OXLINT_SKIP_PREPARE !== "1" &&
|
||||
shouldPrepareExtensionPackageBoundaryArtifacts(finalArgs)
|
||||
) {
|
||||
prepareExtensionPackageBoundaryArtifacts(env);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"tsBuildInfoFile": ".artifacts/tsgo-cache/core.tsbuildinfo"
|
||||
},
|
||||
"include": ["src/**/*", "ui/**/*", "packages/**/*"],
|
||||
"exclude": ["node_modules", "dist", "**/dist/**", "**/*.test.ts", "**/*.test.tsx", "test/**"]
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
{
|
||||
"extends": "./tsconfig.test.json",
|
||||
"compilerOptions": {
|
||||
"tsBuildInfoFile": ".artifacts/tsgo-cache/core-test.tsbuildinfo"
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.d.ts",
|
||||
"src/**/*.test.ts",
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"tsBuildInfoFile": ".artifacts/tsgo-cache/extensions.tsbuildinfo"
|
||||
},
|
||||
"include": ["src/**/*.d.ts", "ui/src/**/*.d.ts", "extensions/**/*"],
|
||||
"exclude": ["node_modules", "dist", "**/dist/**", "**/*.test.ts", "**/*.test.tsx", "test/**"]
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
{
|
||||
"extends": "./tsconfig.test.json",
|
||||
"compilerOptions": {
|
||||
"tsBuildInfoFile": ".artifacts/tsgo-cache/extensions-test.tsbuildinfo"
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.d.ts",
|
||||
"ui/**/*.d.ts",
|
||||
|
||||
5
tsconfig.oxlint.core.json
Normal file
5
tsconfig.oxlint.core.json
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"include": ["src/**/*", "ui/**/*", "packages/**/*"],
|
||||
"exclude": ["node_modules", "dist", "dist-runtime"]
|
||||
}
|
||||
5
tsconfig.oxlint.extensions.json
Normal file
5
tsconfig.oxlint.extensions.json
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"include": ["src/**/*", "ui/src/**/*", "packages/**/*.d.ts", "extensions/**/*"],
|
||||
"exclude": ["node_modules", "dist", "dist-runtime"]
|
||||
}
|
||||
5
tsconfig.oxlint.scripts.json
Normal file
5
tsconfig.oxlint.scripts.json
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"include": ["src/**/*.d.ts", "packages/**/*.d.ts", "scripts/**/*"],
|
||||
"exclude": ["node_modules", "dist", "dist-runtime"]
|
||||
}
|
||||
Reference in New Issue
Block a user