From c727388f937ffc9128ba4e4d2ae37d4c72487bc1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 15 Apr 2026 12:04:31 +0100 Subject: [PATCH] fix(plugins): localize bundled runtime deps to extensions (#67099) * fix(plugins): localize bundled runtime deps to extensions * fix(plugins): move staged runtime deps out of root * fix(packaging): harden prepack and runtime dep staging * fix(packaging): preserve optional runtime dep staging * Update CHANGELOG.md * fix(packaging): harden runtime staging filesystem writes * fix(docker): ship preinstall warning in bootstrap layers * fix(packaging): exclude staged plugin node_modules from npm pack --- CHANGELOG.md | 1 + Dockerfile | 2 +- extensions/feishu/src/client.ts | 22 +- extensions/google/package.json | 3 + extensions/line/package.json | 3 + extensions/openshell/package.json | 3 + extensions/qqbot/src/api.ts | 24 +- extensions/qqbot/src/slash-commands.ts | 27 +- package.json | 23 +- pnpm-lock.yaml | 72 +-- scripts/docker/cleanup-smoke/Dockerfile | 2 +- scripts/e2e/Dockerfile | 2 +- .../bundled-plugin-root-runtime-mirrors.mjs | 16 + scripts/openclaw-prepack.ts | 23 +- .../preinstall-package-manager-warning.mjs | 64 +++ scripts/release-check.ts | 6 +- scripts/root-dependency-ownership-audit.mjs | 305 +++++++++++ scripts/stage-bundled-plugin-runtime-deps.mjs | 505 ++++++++++++++---- .../package-manifest.contract.test.ts | 20 +- ...in-sdk-package-contract-guardrails.test.ts | 27 - src/plugins/sdk-alias.test.ts | 9 +- src/plugins/sdk-alias.ts | 7 + .../stage-bundled-plugin-runtime-deps.test.ts | 2 +- test/openclaw-prepack.test.ts | 15 +- test/release-check.test.ts | 30 +- ...preinstall-package-manager-warning.test.ts | 72 +++ .../root-dependency-ownership-audit.test.ts | 82 +++ .../stage-bundled-plugin-runtime-deps.test.ts | 144 ++++- tsdown.config.ts | 101 +++- 29 files changed, 1335 insertions(+), 277 deletions(-) create mode 100644 scripts/preinstall-package-manager-warning.mjs create mode 100644 scripts/root-dependency-ownership-audit.mjs create mode 100644 test/scripts/preinstall-package-manager-warning.test.ts create mode 100644 test/scripts/root-dependency-ownership-audit.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 65b7b72d1f5..2b360b5f1c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Docs: https://docs.openclaw.ai - Agents/local models: add experimental `agents.defaults.experimental.localModelLean: true` to drop heavyweight default tools like `browser`, `cron`, and `message`, reducing prompt size for weaker local-model setups without changing the normal path. Thanks @ImLukeF. - QA/Matrix: split Matrix live QA into a source-linked `qa-matrix` runner and keep repo-private `qa-*` surfaces out of packaged and published builds. (#66723) Thanks @gumadeiras. - Control UI/Overview: add a Model Auth status card showing OAuth token health and provider rate-limit pressure at a glance, with attention callouts when OAuth tokens are expiring or expired. Backed by a new `models.authStatus` gateway method that strips credentials and caches for 60s. (#66211) Thanks @omarshahine. +- Packaging/plugins: localize bundled plugin runtime deps to their owning extensions, trim the published docs payload, and tighten install/package-manager guardrails so published builds stay leaner and core stops carrying extension-owned runtime baggage. Thanks @vincentkoc. - GitHub Copilot/memory search: add a GitHub Copilot embedding provider for memory search, and expose a dedicated Copilot embedding host helper so plugins can reuse the transport while honoring remote overrides, token refresh, and safer payload validation. (#61718) Thanks @feiskyer and @vincentkoc. ### Fixes diff --git a/Dockerfile b/Dockerfile index d02eebde044..bd5f9d02175 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,7 +65,7 @@ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc ./ COPY openclaw.mjs ./ COPY ui/package.json ./ui/package.json COPY patches ./patches -COPY scripts/postinstall-bundled-plugins.mjs scripts/npm-runner.mjs scripts/windows-cmd-helpers.mjs ./scripts/ +COPY scripts/postinstall-bundled-plugins.mjs scripts/preinstall-package-manager-warning.mjs scripts/npm-runner.mjs scripts/windows-cmd-helpers.mjs ./scripts/ COPY --from=ext-deps /out/ ./${OPENCLAW_BUNDLED_PLUGIN_DIR}/ diff --git a/extensions/feishu/src/client.ts b/extensions/feishu/src/client.ts index 02ac7fb16b6..4c1ab41da22 100644 --- a/extensions/feishu/src/client.ts +++ b/extensions/feishu/src/client.ts @@ -5,7 +5,27 @@ import { resolveAmbientNodeProxyAgent } from "openclaw/plugin-sdk/extension-shar import type { FeishuConfig, FeishuDomain, ResolvedFeishuAccount } from "./types.js"; const require = createRequire(import.meta.url); -const { version: pluginVersion } = require("../package.json") as { version: string }; +const PACKAGE_JSON_CANDIDATES = [ + "../package.json", + "./package.json", + "../../package.json", +] as const; + +function readPluginVersion(): string { + for (const candidate of PACKAGE_JSON_CANDIDATES) { + try { + const version = (require(candidate) as { version?: unknown }).version; + if (typeof version === "string" && version.trim().length > 0) { + return version; + } + } catch { + // Ignore missing candidate paths across source and bundled layouts. + } + } + return "unknown"; +} + +const pluginVersion = readPluginVersion(); export { pluginVersion }; diff --git a/extensions/google/package.json b/extensions/google/package.json index 6620ce06c70..c6cd8c035a5 100644 --- a/extensions/google/package.json +++ b/extensions/google/package.json @@ -11,6 +11,9 @@ "@openclaw/plugin-sdk": "workspace:*" }, "openclaw": { + "bundle": { + "stageRuntimeDependencies": true + }, "extensions": [ "./index.ts" ] diff --git a/extensions/line/package.json b/extensions/line/package.json index eb7350a07f0..fbbc4935847 100644 --- a/extensions/line/package.json +++ b/extensions/line/package.json @@ -4,6 +4,9 @@ "private": true, "description": "OpenClaw LINE channel plugin", "type": "module", + "dependencies": { + "@line/bot-sdk": "^11.0.0" + }, "devDependencies": { "@openclaw/plugin-sdk": "workspace:*", "openclaw": "workspace:*" diff --git a/extensions/openshell/package.json b/extensions/openshell/package.json index df17a45d663..ae67e904636 100644 --- a/extensions/openshell/package.json +++ b/extensions/openshell/package.json @@ -4,6 +4,9 @@ "private": true, "description": "OpenClaw OpenShell sandbox backend", "type": "module", + "dependencies": { + "openshell": "0.1.0" + }, "devDependencies": { "@openclaw/plugin-sdk": "workspace:*" }, diff --git a/extensions/qqbot/src/api.ts b/extensions/qqbot/src/api.ts index 9ab281655b2..511ff6ca637 100644 --- a/extensions/qqbot/src/api.ts +++ b/extensions/qqbot/src/api.ts @@ -11,12 +11,26 @@ const TOKEN_URL = "https://bots.qq.com/app/getAppAccessToken"; // Plugin User-Agent format: QQBotPlugin/{version} (Node/{nodeVersion}; {os}) const _require = createRequire(import.meta.url); -let _pluginVersion = "unknown"; -try { - _pluginVersion = _require("../package.json").version ?? "unknown"; -} catch { - /* fallback */ +const PACKAGE_JSON_CANDIDATES = [ + "../package.json", + "./package.json", + "../../package.json", +] as const; + +function readPluginVersion(): string { + for (const candidate of PACKAGE_JSON_CANDIDATES) { + try { + const version = (_require(candidate) as { version?: unknown }).version; + if (typeof version === "string" && version.trim().length > 0) { + return version; + } + } catch { + // Ignore missing candidate paths across source and bundled layouts. + } + } + return "unknown"; } +const _pluginVersion = readPluginVersion(); export const PLUGIN_USER_AGENT = `QQBotPlugin/${_pluginVersion} (Node/${process.versions.node}; ${os.platform()})`; // ========================================================================= diff --git a/extensions/qqbot/src/slash-commands.ts b/extensions/qqbot/src/slash-commands.ts index c02809328b4..ead86c8ef09 100644 --- a/extensions/qqbot/src/slash-commands.ts +++ b/extensions/qqbot/src/slash-commands.ts @@ -16,15 +16,28 @@ import type { QQBotAccountConfig } from "./types.js"; import { debugLog } from "./utils/debug-log.js"; import { getHomeDir, getQQBotDataDir, isWindows } from "./utils/platform.js"; const require = createRequire(import.meta.url); +const PACKAGE_JSON_CANDIDATES = [ + "../package.json", + "./package.json", + "../../package.json", +] as const; + +function readPluginVersion(): string { + for (const candidate of PACKAGE_JSON_CANDIDATES) { + try { + const version = (require(candidate) as { version?: unknown }).version; + if (typeof version === "string" && version.trim().length > 0) { + return version; + } + } catch { + // Ignore missing candidate paths across source and bundled layouts. + } + } + return "unknown"; +} // Read the package version from package.json. -let PLUGIN_VERSION = "unknown"; -try { - const pkg = require("../package.json"); - PLUGIN_VERSION = pkg.version ?? "unknown"; -} catch { - // fallback -} +const PLUGIN_VERSION = readPluginVersion(); const QQBOT_PLUGIN_GITHUB_URL = "https://github.com/openclaw/openclaw/tree/main/extensions/qqbot"; const QQBOT_UPGRADE_GUIDE_URL = "https://q.qq.com/qqbot/openclaw/upgrade.html"; diff --git a/package.json b/package.json index 3d576460cad..c1881327cd0 100644 --- a/package.json +++ b/package.json @@ -29,17 +29,17 @@ "dist/", "!dist/**/*.map", "!dist/plugin-sdk/.tsbuildinfo", + "!dist/extensions/*/node_modules/**", "!dist/extensions/qa-channel/**", "dist/extensions/qa-channel/runtime-api.js", "!dist/extensions/qa-lab/**", "dist/extensions/qa-lab/runtime-api.js", "!dist/extensions/qa-matrix/**", - "docs/", - "!docs/.generated/**", - "!docs/.i18n/zh-CN.tm.jsonl", + "docs/reference/templates/", "qa/scenarios/", "skills/", "scripts/npm-runner.mjs", + "scripts/preinstall-package-manager-warning.mjs", "scripts/postinstall-bundled-plugins.mjs", "scripts/windows-cmd-helpers.mjs" ], @@ -1159,6 +1159,7 @@ "deadcode:report:ci:ts-unused": "mkdir -p .artifacts/deadcode && pnpm deadcode:ts-unused > .artifacts/deadcode/ts-unused-exports.txt 2>&1 || true", "deadcode:ts-prune": "pnpm dlx ts-prune src extensions scripts", "deadcode:ts-unused": "pnpm dlx ts-unused-exports tsconfig.json --ignoreTestFiles --exitWithCount", + "deps:root-ownership": "node scripts/root-dependency-ownership-audit.mjs", "dev": "node scripts/run-node.mjs", "docs:bin": "node scripts/build-docs-list.mjs", "docs:check-i18n-glossary": "node scripts/check-docs-i18n-glossary.mjs", @@ -1234,6 +1235,7 @@ "plugin-sdk:usage": "node --import tsx scripts/analyze-plugin-sdk-usage.ts", "plugins:sync": "node --import tsx scripts/sync-plugin-versions.ts", "postinstall": "node scripts/postinstall-bundled-plugins.mjs", + "preinstall": "node scripts/preinstall-package-manager-warning.mjs", "prepack": "node --import tsx scripts/openclaw-prepack.ts", "prepare": "command -v git >/dev/null 2>&1 && git rev-parse --is-inside-work-tree >/dev/null 2>&1 && git config core.hooksPath git-hooks || exit 0", "prepush:ci": "bash scripts/prepush-ci.sh", @@ -1374,19 +1376,13 @@ "dependencies": { "@agentclientprotocol/sdk": "0.18.2", "@anthropic-ai/vertex-sdk": "^0.15.0", - "@aws-sdk/client-bedrock": "3.1028.0", "@aws-sdk/client-bedrock-runtime": "3.1028.0", "@aws-sdk/credential-provider-node": "3.972.30", - "@aws/bedrock-token-generator": "^1.1.0", - "@buape/carbon": "0.15.0", "@clack/prompts": "^1.2.0", - "@google/genai": "^1.49.0", "@grammyjs/runner": "^2.0.3", "@grammyjs/transformer-throttler": "^1.2.1", "@homebridge/ciao": "^1.3.6", "@lancedb/lancedb": "^0.27.2", - "@larksuiteoapi/node-sdk": "^1.60.0", - "@line/bot-sdk": "^11.0.0", "@lydell/node-pty": "1.2.0-beta.12", "@mariozechner/pi-agent-core": "0.66.1", "@mariozechner/pi-ai": "0.66.1", @@ -1396,8 +1392,6 @@ "@modelcontextprotocol/sdk": "1.29.0", "@mozilla/readability": "^0.6.0", "@sinclair/typebox": "0.34.49", - "@slack/bolt": "^4.7.0", - "@slack/web-api": "^7.15.0", "@whiskeysockets/baileys": "7.0.0-rc.9", "ajv": "^8.18.0", "chalk": "^5.6.2", @@ -1405,14 +1399,12 @@ "cli-highlight": "^2.1.11", "commander": "^14.0.3", "croner": "^10.0.1", - "discord-api-types": "^0.38.45", "dotenv": "^17.4.1", "express": "^5.2.1", "file-type": "22.0.1", "gaxios": "7.1.4", "google-auth-library": "^10.6.2", "grammy": "^1.42.0", - "hono": "4.12.12", "https-proxy-agent": "^9.0.0", "ipaddr.js": "^2.3.0", "jimp": "^1.6.1", @@ -1427,7 +1419,6 @@ "node-edge-tts": "^1.2.10", "nostr-tools": "^2.23.3", "openai": "^6.34.0", - "opusscript": "^0.1.1", "osc-progress": "^0.3.0", "pdfjs-dist": "^5.6.205", "playwright-core": "1.59.1", @@ -1479,11 +1470,9 @@ } }, "optionalDependencies": { - "@discordjs/opus": "^0.10.0", "@matrix-org/matrix-sdk-crypto-nodejs": "^0.4.0", "fake-indexeddb": "^6.2.5", - "music-metadata": "^11.12.3", - "openshell": "0.1.0" + "music-metadata": "^11.12.3" }, "overrides": { "axios": "1.15.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3dceb2db276..95154049fc9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,27 +43,15 @@ importers: '@anthropic-ai/vertex-sdk': specifier: ^0.15.0 version: 0.15.0(zod@4.3.6) - '@aws-sdk/client-bedrock': - specifier: 3.1028.0 - version: 3.1028.0 '@aws-sdk/client-bedrock-runtime': specifier: 3.1028.0 version: 3.1028.0 '@aws-sdk/credential-provider-node': specifier: 3.972.30 version: 3.972.30 - '@aws/bedrock-token-generator': - specifier: ^1.1.0 - version: 1.1.0 - '@buape/carbon': - specifier: 0.15.0 - version: 0.15.0(@discordjs/opus@0.10.0)(@emnapi/core@1.9.2)(@emnapi/runtime@1.9.2)(hono@4.12.12)(opusscript@0.1.1) '@clack/prompts': specifier: ^1.2.0 version: 1.2.0 - '@google/genai': - specifier: ^1.49.0 - version: 1.49.0(@modelcontextprotocol/sdk@1.29.0(zod@4.3.6)) '@grammyjs/runner': specifier: ^2.0.3 version: 2.0.3(grammy@1.42.0) @@ -76,12 +64,6 @@ importers: '@lancedb/lancedb': specifier: ^0.27.2 version: 0.27.2(apache-arrow@18.1.0) - '@larksuiteoapi/node-sdk': - specifier: ^1.60.0 - version: 1.60.0 - '@line/bot-sdk': - specifier: ^11.0.0 - version: 11.0.0 '@lydell/node-pty': specifier: 1.2.0-beta.12 version: 1.2.0-beta.12 @@ -112,12 +94,6 @@ importers: '@sinclair/typebox': specifier: 0.34.49 version: 0.34.49 - '@slack/bolt': - specifier: ^4.7.0 - version: 4.7.0(@types/express@5.0.6) - '@slack/web-api': - specifier: ^7.15.0 - version: 7.15.0 '@whiskeysockets/baileys': specifier: 7.0.0-rc.9 version: 7.0.0-rc.9(patch_hash=23ec8efe1484afa57c51b96955ba331d1467521a8e676a18c2690da7e70a6201)(audio-decode@2.2.3)(jimp@1.6.1)(sharp@0.34.5) @@ -139,9 +115,6 @@ importers: croner: specifier: ^10.0.1 version: 10.0.1 - discord-api-types: - specifier: ^0.38.45 - version: 0.38.45 dotenv: specifier: ^17.4.1 version: 17.4.1 @@ -160,9 +133,6 @@ importers: grammy: specifier: ^1.42.0 version: 1.42.0 - hono: - specifier: 4.12.12 - version: 4.12.12 https-proxy-agent: specifier: ^9.0.0 version: 9.0.0 @@ -208,9 +178,6 @@ importers: openai: specifier: ^6.34.0 version: 6.34.0(ws@8.20.0)(zod@4.3.6) - opusscript: - specifier: ^0.1.1 - version: 0.1.1 osc-progress: specifier: ^0.3.0 version: 0.3.0 @@ -327,9 +294,6 @@ importers: specifier: ^4.1.4 version: 4.1.4(@opentelemetry/api@1.9.1)(@types/node@25.6.0)(@vitest/browser-playwright@4.1.4)(@vitest/coverage-v8@4.1.4)(jsdom@29.0.2(@noble/hashes@2.0.1))(vite@8.0.8(@types/node@25.6.0)(esbuild@0.27.7)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.3)) optionalDependencies: - '@discordjs/opus': - specifier: ^0.10.0 - version: 0.10.0 '@matrix-org/matrix-sdk-crypto-nodejs': specifier: ^0.4.0 version: 0.4.0 @@ -339,9 +303,6 @@ importers: music-metadata: specifier: ^11.12.3 version: 11.12.3 - openshell: - specifier: 0.1.0 - version: 0.1.0 extensions/acpx: dependencies: @@ -694,6 +655,10 @@ importers: version: link:../../packages/plugin-sdk extensions/line: + dependencies: + '@line/bot-sdk': + specifier: ^11.0.0 + version: 11.0.0 devDependencies: '@openclaw/plugin-sdk': specifier: workspace:* @@ -961,6 +926,10 @@ importers: version: link:../../packages/plugin-sdk extensions/openshell: + dependencies: + openshell: + specifier: 0.1.0 + version: 0.1.0 devDependencies: '@openclaw/plugin-sdk': specifier: workspace:* @@ -10983,8 +10952,7 @@ snapshots: dependencies: tslib: 2.8.1 - '@telegraf/types@7.1.0': - optional: true + '@telegraf/types@7.1.0': {} '@thi.ng/bitstream@2.4.45': dependencies: @@ -11732,21 +11700,18 @@ snapshots: dependencies: base-x: 5.0.1 - buffer-alloc-unsafe@1.1.0: - optional: true + buffer-alloc-unsafe@1.1.0: {} buffer-alloc@1.2.0: dependencies: buffer-alloc-unsafe: 1.1.0 buffer-fill: 1.0.0 - optional: true buffer-crc32@0.2.13: {} buffer-equal-constant-time@1.0.1: {} - buffer-fill@1.0.0: - optional: true + buffer-fill@1.0.0: {} buffer-from@1.1.2: {} @@ -12135,8 +12100,7 @@ snapshots: domelementtype: 2.3.0 domhandler: 5.0.3 - dotenv@16.6.1: - optional: true + dotenv@16.6.1: {} dotenv@17.4.1: {} @@ -13444,8 +13408,7 @@ snapshots: dependencies: '@wasm-audio-decoders/common': 9.0.7 - mri@1.2.0: - optional: true + mri@1.2.0: {} mrmime@2.0.1: {} @@ -13677,7 +13640,6 @@ snapshots: transitivePeerDependencies: - encoding - supports-color - optional: true opus-decoder@0.7.11: dependencies: @@ -13792,8 +13754,7 @@ snapshots: dependencies: p-finally: 1.0.0 - p-timeout@4.1.0: - optional: true + p-timeout@4.1.0: {} p-timeout@7.0.1: {} @@ -14378,14 +14339,12 @@ snapshots: safe-compare@1.1.4: dependencies: buffer-alloc: 1.2.0 - optional: true safe-stable-stringify@2.5.0: {} safer-buffer@2.1.2: {} - sandwich-stream@2.0.2: - optional: true + sandwich-stream@2.0.2: {} sass-lookup@6.1.1: dependencies: @@ -14781,7 +14740,6 @@ snapshots: transitivePeerDependencies: - encoding - supports-color - optional: true text-decoder@1.2.7: dependencies: diff --git a/scripts/docker/cleanup-smoke/Dockerfile b/scripts/docker/cleanup-smoke/Dockerfile index 329752b58b3..9284c4a56b5 100644 --- a/scripts/docker/cleanup-smoke/Dockerfile +++ b/scripts/docker/cleanup-smoke/Dockerfile @@ -20,7 +20,7 @@ COPY ui/package.json ./ui/package.json COPY packages ./packages COPY extensions ./extensions COPY patches ./patches -COPY scripts/postinstall-bundled-plugins.mjs scripts/npm-runner.mjs scripts/windows-cmd-helpers.mjs ./scripts/ +COPY scripts/postinstall-bundled-plugins.mjs scripts/preinstall-package-manager-warning.mjs scripts/npm-runner.mjs scripts/windows-cmd-helpers.mjs ./scripts/ RUN --mount=type=cache,id=openclaw-pnpm-store,target=/root/.local/share/pnpm/store,sharing=locked \ corepack enable \ && if ! pnpm install --frozen-lockfile >/tmp/openclaw-cleanup-pnpm-install.log 2>&1; then \ diff --git a/scripts/e2e/Dockerfile b/scripts/e2e/Dockerfile index ff33a06b08a..cb3e6a68b45 100644 --- a/scripts/e2e/Dockerfile +++ b/scripts/e2e/Dockerfile @@ -22,7 +22,7 @@ COPY --chown=appuser:appuser package.json pnpm-lock.yaml pnpm-workspace.yaml .np COPY --chown=appuser:appuser ui/package.json ./ui/package.json COPY --chown=appuser:appuser extensions ./extensions COPY --chown=appuser:appuser patches ./patches -COPY --chown=appuser:appuser scripts/postinstall-bundled-plugins.mjs scripts/npm-runner.mjs scripts/windows-cmd-helpers.mjs ./scripts/ +COPY --chown=appuser:appuser scripts/postinstall-bundled-plugins.mjs scripts/preinstall-package-manager-warning.mjs scripts/npm-runner.mjs scripts/windows-cmd-helpers.mjs ./scripts/ RUN --mount=type=cache,id=openclaw-pnpm-store,target=/home/appuser/.local/share/pnpm/store,sharing=locked \ pnpm install --frozen-lockfile diff --git a/scripts/lib/bundled-plugin-root-runtime-mirrors.mjs b/scripts/lib/bundled-plugin-root-runtime-mirrors.mjs index a105e8b1b42..a227a020aad 100644 --- a/scripts/lib/bundled-plugin-root-runtime-mirrors.mjs +++ b/scripts/lib/bundled-plugin-root-runtime-mirrors.mjs @@ -2,6 +2,10 @@ import fs from "node:fs"; import path from "node:path"; const JS_EXTENSIONS = new Set([".cjs", ".js", ".mjs"]); +const CURATED_ROOT_RUNTIME_MIRRORS = new Set([ + "@matrix-org/matrix-sdk-crypto-nodejs", + "@matrix-org/matrix-sdk-crypto-wasm", +]); export function collectRuntimeDependencySpecs(packageJson = {}) { return new Map( @@ -152,6 +156,18 @@ export function collectRootDistBundledRuntimeMirrors(params) { const bundledSpecs = params.bundledRuntimeDependencySpecs; const mirrors = new Map(); + for (const dependencyName of CURATED_ROOT_RUNTIME_MIRRORS) { + const bundledSpec = bundledSpecs.get(dependencyName); + if (!bundledSpec) { + continue; + } + mirrors.set(dependencyName, { + importers: new Set([""]), + pluginIds: bundledSpec.pluginIds, + spec: bundledSpec.spec, + }); + } + for (const filePath of walkJavaScriptFiles(distDir)) { const source = fs.readFileSync(filePath, "utf8"); const relativePath = path.relative(distDir, filePath).replaceAll(path.sep, "/"); diff --git a/scripts/openclaw-prepack.ts b/scripts/openclaw-prepack.ts index 508294fb69d..c1f3d564989 100644 --- a/scripts/openclaw-prepack.ts +++ b/scripts/openclaw-prepack.ts @@ -5,8 +5,6 @@ import { existsSync, readdirSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { formatErrorMessage } from "../src/infra/errors.ts"; import { writePackageDistInventory } from "../src/infra/package-dist-inventory.ts"; - -const skipPrepackPreparedEnv = "OPENCLAW_PREPACK_PREPARED"; const requiredPreparedPathGroups = [ ["dist/index.js", "dist/index.mjs"], ["dist/control-ui/index.html"], @@ -22,14 +20,6 @@ function normalizeFiles(files: Iterable): Set { return new Set(Array.from(files, (file) => file.replace(/\\/g, "/"))); } -export function shouldSkipPrepack(env = process.env): boolean { - const raw = env[skipPrepackPreparedEnv]; - if (!raw) { - return false; - } - return !/^(0|false)$/i.test(raw); -} - export function collectPreparedPrepackErrors( files: Iterable, assetPaths: Iterable, @@ -83,9 +73,7 @@ function ensurePreparedArtifacts(): void { const preparedFiles = collectPreparedFilePaths(); const errors = collectPreparedPrepackErrors(preparedFiles.files, preparedFiles.assets); if (errors.length === 0) { - console.error( - `prepack: using prepared artifacts from ${skipPrepackPreparedEnv}; skipping rebuild.`, - ); + console.error("prepack: using existing prepared artifacts."); return; } for (const error of errors) { @@ -97,7 +85,7 @@ function ensurePreparedArtifacts(): void { } console.error( - `prepack: ${skipPrepackPreparedEnv}=1 requires an existing build and Control UI bundle. Run \`pnpm build && pnpm ui:build\` first or unset ${skipPrepackPreparedEnv}.`, + "prepack: requires an existing build and Control UI bundle. Run `pnpm build && pnpm ui:build` before packing or publishing.", ); process.exit(1); } @@ -123,14 +111,9 @@ async function writeDistInventory(): Promise { async function main(): Promise { const pnpmCommand = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; - if (shouldSkipPrepack()) { - ensurePreparedArtifacts(); - await writeDistInventory(); - runBuildSmoke(); - return; - } run(pnpmCommand, ["build"]); run(pnpmCommand, ["ui:build"]); + ensurePreparedArtifacts(); await writeDistInventory(); runBuildSmoke(); } diff --git a/scripts/preinstall-package-manager-warning.mjs b/scripts/preinstall-package-manager-warning.mjs new file mode 100644 index 00000000000..c34044bcf4d --- /dev/null +++ b/scripts/preinstall-package-manager-warning.mjs @@ -0,0 +1,64 @@ +import { pathToFileURL } from "node:url"; + +const allowedLifecyclePackageManagers = new Set(["pnpm", "npm", "yarn", "bun"]); + +function normalizeEnvValue(value) { + return typeof value === "string" ? value.trim() : ""; +} + +function normalizeLifecyclePackageManagerName(value) { + const normalized = normalizeEnvValue(value).toLowerCase(); + if (!/^[a-z0-9][a-z0-9._-]*$/u.test(normalized)) { + return null; + } + return allowedLifecyclePackageManagers.has(normalized) ? normalized : null; +} + +export function detectLifecyclePackageManager(env = process.env) { + const userAgent = normalizeEnvValue(env.npm_config_user_agent); + const userAgentMatch = /^([A-Za-z0-9._-]+)\//u.exec(userAgent); + if (userAgentMatch) { + return normalizeLifecyclePackageManagerName(userAgentMatch[1]); + } + + const execPath = normalizeEnvValue(env.npm_execpath).toLowerCase(); + if (execPath.includes("pnpm")) { + return "pnpm"; + } + if (execPath.includes("npm")) { + return "npm"; + } + if (execPath.includes("yarn")) { + return "yarn"; + } + if (execPath.includes("bun")) { + return "bun"; + } + + return null; +} + +export function createPackageManagerWarningMessage(packageManager) { + if (!packageManager || packageManager === "pnpm") { + return null; + } + + return [ + `[openclaw] warning: detected ${packageManager} for install lifecycle.`, + "[openclaw] this repo works best with pnpm; npm-compatible installs are slower and much larger here.", + "[openclaw] prefer: corepack pnpm install", + ].join("\n"); +} + +export function warnIfNonPnpmLifecycle(env = process.env, warn = console.warn) { + const message = createPackageManagerWarningMessage(detectLifecyclePackageManager(env)); + if (!message) { + return false; + } + warn(message); + return true; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + warnIfNonPnpmLifecycle(); +} diff --git a/scripts/release-check.ts b/scripts/release-check.ts index 0b9e63aef22..35db18baa91 100755 --- a/scripts/release-check.ts +++ b/scripts/release-check.ts @@ -46,6 +46,7 @@ const requiredPathGroups = [ ...WORKSPACE_TEMPLATE_PACK_PATHS, ...listRequiredQaScenarioPackPaths(), "scripts/npm-runner.mjs", + "scripts/preinstall-package-manager-warning.mjs", "scripts/postinstall-bundled-plugins.mjs", "dist/plugin-sdk/compat.js", "dist/plugin-sdk/root-alias.cjs", @@ -260,13 +261,10 @@ export function collectMissingPackPaths(paths: Iterable): string[] { } export function collectForbiddenPackPaths(paths: Iterable): string[] { - const isAllowedBundledPluginNodeModulesPath = (path: string) => - /^dist\/extensions\/[^/]+\/node_modules\//.test(path); return [...paths] .filter( (path) => - forbiddenPrefixes.some((prefix) => path.startsWith(prefix)) || - (/node_modules\//.test(path) && !isAllowedBundledPluginNodeModulesPath(path)), + forbiddenPrefixes.some((prefix) => path.startsWith(prefix)) || /node_modules\//.test(path), ) .toSorted((left, right) => left.localeCompare(right)); } diff --git a/scripts/root-dependency-ownership-audit.mjs b/scripts/root-dependency-ownership-audit.mjs new file mode 100644 index 00000000000..ad72ea61d24 --- /dev/null +++ b/scripts/root-dependency-ownership-audit.mjs @@ -0,0 +1,305 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + collectBundledPluginRuntimeDependencySpecs, + collectRootDistBundledRuntimeMirrors, + packageNameFromSpecifier, +} from "./lib/bundled-plugin-root-runtime-mirrors.mjs"; + +const DEFAULT_SCAN_ROOTS = ["src", "extensions", "packages", "ui", "scripts", "test"]; +const SCANNED_EXTENSIONS = new Set([".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx"]); +const IMPORT_PATTERNS = [ + /\bfrom\s*["']([^"']+)["']/g, + /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g, + /\brequire\s*\(\s*["']([^"']+)["']\s*\)/g, + /\b(?:require|[_$A-Za-z][\w$]*require[\w$]*)\.resolve\s*\(\s*["']([^"']+)["']\s*\)/gi, +]; + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function isScannableSourceFile(fileName) { + return SCANNED_EXTENSIONS.has(path.extname(fileName)); +} + +function shouldSkipDir(dirName) { + return dirName === "dist" || dirName === "node_modules" || dirName === ".git"; +} + +function walkFiles(rootDir) { + if (!fs.existsSync(rootDir)) { + return []; + } + const files = []; + const queue = [rootDir]; + while (queue.length > 0) { + const current = queue.shift(); + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const fullPath = path.join(current, entry.name); + if (entry.isDirectory()) { + if (shouldSkipDir(entry.name)) { + continue; + } + queue.push(fullPath); + continue; + } + if (entry.isFile() && isScannableSourceFile(entry.name)) { + files.push(fullPath); + } + } + } + return files.toSorted((left, right) => left.localeCompare(right)); +} + +function normalizeRelativePath(filePath, repoRoot) { + return path.relative(repoRoot, filePath).replaceAll(path.sep, "/"); +} + +function sectionFor(relativePath) { + const [section = "other"] = relativePath.split("/"); + return section; +} + +export function collectModuleSpecifiers(source) { + const specifiers = new Set(); + for (const pattern of IMPORT_PATTERNS) { + for (const match of source.matchAll(pattern)) { + if (match[1]) { + specifiers.add(match[1]); + } + } + } + return specifiers; +} + +function collectExtensionDependencyDeclarations(repoRoot) { + const declarations = new Map(); + const extensionsRoot = path.join(repoRoot, "extensions"); + if (!fs.existsSync(extensionsRoot)) { + return declarations; + } + + for (const entry of fs.readdirSync(extensionsRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const packageJsonPath = path.join(extensionsRoot, entry.name, "package.json"); + if (!fs.existsSync(packageJsonPath)) { + continue; + } + const packageJson = readJson(packageJsonPath); + for (const section of [ + "dependencies", + "optionalDependencies", + "devDependencies", + "peerDependencies", + ]) { + for (const depName of Object.keys(packageJson[section] ?? {})) { + const existing = declarations.get(depName) ?? []; + existing.push(`${entry.name}:${section}`); + declarations.set(depName, existing); + } + } + } + + for (const values of declarations.values()) { + values.sort((left, right) => left.localeCompare(right)); + } + + return declarations; +} + +function sectionSetContainsCore(sectionSet) { + return sectionSet.has("src") || sectionSet.has("packages") || sectionSet.has("ui"); +} + +function sectionSetIsSubsetOf(sectionSet, allowed) { + for (const value of sectionSet) { + if (!allowed.has(value)) { + return false; + } + } + return sectionSet.size > 0; +} + +export function classifyRootDependencyOwnership(record) { + const sections = new Set(record.sections); + + if (record.rootMirrorImporters.length > 0) { + return { + category: "extension_only_root_mirror", + recommendation: + "blocked by packaged host graph: remove root mirror only after bundled runtime resolution stops importing it from root dist", + }; + } + + if (sections.size === 0) { + return { + category: "unreferenced", + recommendation: "investigate removal; no direct source imports found in scanned files", + }; + } + + if (sectionSetIsSubsetOf(sections, new Set(["scripts", "test"]))) { + return { + category: "script_or_test_only", + recommendation: "consider moving from dependencies to devDependencies", + }; + } + + if (sectionSetContainsCore(sections)) { + if (sections.has("extensions")) { + return { + category: "shared_core_and_extension", + recommendation: + "keep at root until shared code is split or extension/core boundary changes", + }; + } + return { + category: "core_runtime", + recommendation: "keep at root", + }; + } + + if (sectionSetIsSubsetOf(sections, new Set(["extensions", "test"]))) { + return { + category: "extension_only_localizable", + recommendation: + "candidate to remove from root package.json and rely on owning extension manifests", + }; + } + + return { + category: "mixed_noncore", + recommendation: "inspect manually; usage spans non-core surfaces", + }; +} + +export function collectRootDependencyOwnershipAudit(params = {}) { + const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); + const rootPackageJson = readJson(path.join(repoRoot, "package.json")); + const rootDependencies = { + ...rootPackageJson.dependencies, + ...rootPackageJson.optionalDependencies, + }; + const records = new Map( + Object.keys(rootDependencies).map((depName) => [ + depName, + { + depName, + sections: new Set(), + files: new Set(), + declaredInExtensions: [], + rootMirrorImporters: [], + spec: rootDependencies[depName], + }, + ]), + ); + + const scanRoots = params.scanRoots ?? DEFAULT_SCAN_ROOTS; + for (const scanRoot of scanRoots) { + for (const filePath of walkFiles(path.join(repoRoot, scanRoot))) { + const relativePath = normalizeRelativePath(filePath, repoRoot); + const source = fs.readFileSync(filePath, "utf8"); + for (const specifier of collectModuleSpecifiers(source)) { + const depName = packageNameFromSpecifier(specifier); + if (!depName || !records.has(depName)) { + continue; + } + const record = records.get(depName); + record.sections.add(sectionFor(relativePath)); + record.files.add(relativePath); + } + } + } + + const extensionDeclarations = collectExtensionDependencyDeclarations(repoRoot); + for (const [depName, declarations] of extensionDeclarations) { + const record = records.get(depName); + if (record) { + record.declaredInExtensions = declarations; + } + } + + const distDir = path.join(repoRoot, "dist"); + if (fs.existsSync(distDir)) { + const bundledSpecs = collectBundledPluginRuntimeDependencySpecs( + path.join(repoRoot, "extensions"), + ); + const rootMirrors = collectRootDistBundledRuntimeMirrors({ + bundledRuntimeDependencySpecs: bundledSpecs, + distDir, + }); + for (const [depName, mirror] of rootMirrors) { + const record = records.get(depName); + if (!record) { + continue; + } + record.rootMirrorImporters = [...mirror.importers].toSorted((left, right) => + left.localeCompare(right), + ); + } + } + + return [...records.values()] + .map((record) => { + const classification = classifyRootDependencyOwnership({ + ...record, + sections: [...record.sections].toSorted((left, right) => left.localeCompare(right)), + }); + return { + depName: record.depName, + spec: record.spec, + sections: [...record.sections].toSorted((left, right) => left.localeCompare(right)), + fileCount: record.files.size, + sampleFiles: [...record.files].slice(0, 5), + declaredInExtensions: record.declaredInExtensions, + rootMirrorImporters: record.rootMirrorImporters, + category: classification.category, + recommendation: classification.recommendation, + }; + }) + .toSorted((left, right) => left.depName.localeCompare(right.depName)); +} + +function printTextReport(records) { + const grouped = new Map(); + for (const record of records) { + const existing = grouped.get(record.category) ?? []; + existing.push(record); + grouped.set(record.category, existing); + } + + for (const category of [...grouped.keys()].toSorted((left, right) => left.localeCompare(right))) { + console.log(`\n## ${category}`); + for (const record of grouped.get(category)) { + const details = [`sections=${record.sections.join(",") || "-"}`, `files=${record.fileCount}`]; + if (record.declaredInExtensions.length > 0) { + details.push(`extensions=${record.declaredInExtensions.join(",")}`); + } + if (record.rootMirrorImporters.length > 0) { + details.push(`rootDist=${record.rootMirrorImporters.join(",")}`); + } + console.log(`- ${record.depName}@${record.spec} :: ${details.join(" | ")}`); + console.log(` ${record.recommendation}`); + } + } +} + +function main(argv = process.argv.slice(2)) { + const asJson = argv.includes("--json"); + const records = collectRootDependencyOwnershipAudit(); + if (asJson) { + console.log(JSON.stringify(records, null, 2)); + return; + } + printTextReport(records); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(); +} diff --git a/scripts/stage-bundled-plugin-runtime-deps.mjs b/scripts/stage-bundled-plugin-runtime-deps.mjs index f0c7a1e9237..1431bce8fc6 100644 --- a/scripts/stage-bundled-plugin-runtime-deps.mjs +++ b/scripts/stage-bundled-plugin-runtime-deps.mjs @@ -1,7 +1,6 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import semverSatisfies from "semver/functions/satisfies.js"; @@ -35,18 +34,67 @@ function sanitizeTempPrefixSegment(value) { return normalized.length > 0 ? normalized : "plugin"; } -function replaceDir(targetPath, sourcePath) { - removePathIfExists(targetPath); +function makePluginOwnedTempDir(pluginDir, label) { + return makeTempDir(pluginDir, `.openclaw-runtime-deps-${label}-`); +} + +function assertPathIsNotSymlink(targetPath, label) { try { - fs.renameSync(sourcePath, targetPath); - return; - } catch (error) { - if (error?.code !== "EXDEV") { - throw error; + if (fs.lstatSync(targetPath).isSymbolicLink()) { + throw new Error(`refusing to ${label} via symlinked path: ${targetPath}`); } + } catch (error) { + if (error?.code === "ENOENT") { + return; + } + throw error; + } +} + +function replaceDirAtomically(targetPath, sourcePath) { + assertPathIsNotSymlink(targetPath, "replace runtime deps"); + const targetParentDir = path.dirname(targetPath); + fs.mkdirSync(targetParentDir, { recursive: true }); + const backupPath = makeTempDir( + targetParentDir, + `.openclaw-runtime-deps-backup-${sanitizeTempPrefixSegment(path.basename(targetPath))}-`, + ); + removePathIfExists(backupPath); + + let movedExistingTarget = false; + try { + if (fs.existsSync(targetPath)) { + fs.renameSync(targetPath, backupPath); + movedExistingTarget = true; + } + fs.renameSync(sourcePath, targetPath); + removePathIfExists(backupPath); + } catch (error) { + if (movedExistingTarget && !fs.existsSync(targetPath) && fs.existsSync(backupPath)) { + fs.renameSync(backupPath, targetPath); + } + throw error; + } +} + +function writeJsonAtomically(targetPath, value) { + assertPathIsNotSymlink(targetPath, "write runtime deps stamp"); + const targetParentDir = path.dirname(targetPath); + fs.mkdirSync(targetParentDir, { recursive: true }); + const tempDir = makeTempDir( + targetParentDir, + `.openclaw-runtime-deps-stamp-${sanitizeTempPrefixSegment(path.basename(targetPath))}-`, + ); + const tempPath = path.join(tempDir, path.basename(targetPath)); + try { + fs.writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + }); + fs.renameSync(tempPath, targetPath); + } finally { + removePathIfExists(tempDir); } - fs.cpSync(sourcePath, targetPath, { recursive: true, force: true }); - removePathIfExists(sourcePath); } function dependencyPathSegments(depName) { @@ -80,19 +128,6 @@ function dependencyNodeModulesPath(nodeModulesDir, depName) { return segments ? path.join(nodeModulesDir, ...segments) : null; } -function readInstalledDependencyVersion(nodeModulesDir, depName) { - const depRoot = dependencyNodeModulesPath(nodeModulesDir, depName); - if (depRoot === null) { - return null; - } - const packageJsonPath = path.join(depRoot, "package.json"); - if (!fs.existsSync(packageJsonPath)) { - return null; - } - const version = readJson(packageJsonPath).version; - return typeof version === "string" ? version : null; -} - function dependencyVersionSatisfied(spec, installedVersion) { return semverSatisfies(installedVersion, spec, { includePrerelease: false }); } @@ -147,7 +182,8 @@ const defaultStagedRuntimeDepPruneRules = new Map([ ["@jimp/plugin-quantize", { paths: ["src/__image_snapshots__"] }], ["@jimp/plugin-threshold", { paths: ["src/__image_snapshots__"] }], ]); -const runtimeDepsStagingVersion = 3; +const runtimeDepsStagingVersion = 5; +const exactVersionSpecRe = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u; function resolveRuntimeDepPruneConfig(params = {}) { return { @@ -175,7 +211,10 @@ function resolveInstalledDependencyRoot(params) { for (const depRoot of candidates) { const installedVersion = readInstalledDependencyVersionFromRoot(depRoot); - if (installedVersion !== null && dependencyVersionSatisfied(params.spec, installedVersion)) { + if (installedVersion === null) { + continue; + } + if (params.enforceSpec === false || dependencyVersionSatisfied(params.spec, installedVersion)) { return depRoot; } } @@ -183,14 +222,18 @@ function resolveInstalledDependencyRoot(params) { return null; } -function collectInstalledRuntimeDependencyRoots(rootNodeModulesDir, dependencySpecs) { +function collectInstalledRuntimeDependencyRoots( + rootNodeModulesDir, + dependencySpecs, + directDependencyPackageRoot = null, +) { const packageCache = new Map(); const directRoots = []; const allRoots = []; const queue = Object.entries(dependencySpecs).map(([depName, spec]) => ({ depName, spec, - parentPackageRoot: null, + parentPackageRoot: directDependencyPackageRoot, direct: true, })); const seen = new Set(); @@ -200,6 +243,7 @@ function collectInstalledRuntimeDependencyRoots(rootNodeModulesDir, dependencySp const depRoot = resolveInstalledDependencyRoot({ depName: current.depName, spec: current.spec, + enforceSpec: current.direct, parentPackageRoot: current.parentPackageRoot, rootNodeModulesDir, }); @@ -328,10 +372,23 @@ function selectRuntimeDependencyRootsToCopy(resolution) { return rootsToCopy; } -function resolveInstalledDirectDependencyNames(rootNodeModulesDir, dependencySpecs) { +function resolveInstalledDirectDependencyNames( + rootNodeModulesDir, + dependencySpecs, + directDependencyPackageRoot = null, +) { const directDependencyNames = []; for (const [depName, spec] of Object.entries(dependencySpecs)) { - const installedVersion = readInstalledDependencyVersion(rootNodeModulesDir, depName); + const depRoot = resolveInstalledDependencyRoot({ + depName, + spec, + parentPackageRoot: directDependencyPackageRoot, + rootNodeModulesDir, + }); + if (depRoot === null) { + return null; + } + const installedVersion = readInstalledDependencyVersionFromRoot(depRoot); if (installedVersion === null || !dependencyVersionSatisfied(spec, installedVersion)) { return null; } @@ -390,6 +447,7 @@ function resolveInstalledRuntimeClosureFingerprint(params) { const resolution = collectInstalledRuntimeDependencyRoots( params.rootNodeModulesDir, dependencySpecs, + params.directDependencyPackageRoot, ); if (resolution === null) { return null; @@ -486,6 +544,32 @@ function listBundledPluginRuntimeDirs(repoRoot) { .filter((pluginDir) => fs.existsSync(path.join(pluginDir, "package.json"))); } +function resolveInstalledWorkspacePluginRoot(repoRoot, pluginId) { + const currentPluginRoot = path.join(repoRoot, "extensions", pluginId); + if (fs.existsSync(path.join(currentPluginRoot, "node_modules"))) { + return currentPluginRoot; + } + + const nodeModulesDir = path.join(repoRoot, "node_modules"); + if (!fs.existsSync(nodeModulesDir)) { + return currentPluginRoot; + } + + let installedWorkspaceRoot; + try { + installedWorkspaceRoot = path.dirname(fs.realpathSync(nodeModulesDir)); + } catch { + return currentPluginRoot; + } + + const installedPluginRoot = path.join(installedWorkspaceRoot, "extensions", pluginId); + if (fs.existsSync(path.join(installedPluginRoot, "package.json"))) { + return installedPluginRoot; + } + + return currentPluginRoot; +} + function hasRuntimeDeps(packageJson) { return ( Object.keys(packageJson.dependencies ?? {}).length > 0 || @@ -524,6 +608,168 @@ function sanitizeBundledManifestForRuntimeInstall(pluginDir) { return packageJson; } +function isSafeRuntimeDependencySpec(spec) { + if (typeof spec !== "string") { + return false; + } + const normalized = spec.trim(); + if (normalized.length === 0) { + return false; + } + const lower = normalized.toLowerCase(); + if ( + lower.startsWith("file:") || + lower.startsWith("link:") || + lower.startsWith("workspace:") || + lower.startsWith("git:") || + lower.startsWith("git+") || + lower.startsWith("ssh:") || + lower.startsWith("http:") || + lower.startsWith("https:") + ) { + return false; + } + if (normalized.includes("://")) { + return false; + } + if ( + normalized.startsWith("/") || + normalized.startsWith("\\") || + normalized.startsWith("../") || + normalized.startsWith("..\\") || + normalized.includes("/../") || + normalized.includes("\\..\\") + ) { + return false; + } + return true; +} + +function assertSafeRuntimeDependencySpec(depName, spec) { + if (!isSafeRuntimeDependencySpec(spec)) { + throw new Error(`disallowed runtime dependency spec for ${depName}: ${spec}`); + } +} + +function resolveInstalledPinnedDependencyVersion(params) { + const depRoot = resolveInstalledDependencyRoot({ + depName: params.depName, + enforceSpec: true, + parentPackageRoot: params.parentPackageRoot, + rootNodeModulesDir: params.rootNodeModulesDir, + spec: params.spec, + }); + if (depRoot === null) { + return null; + } + return readInstalledDependencyVersionFromRoot(depRoot); +} + +function resolvePinnedRuntimeDependencyVersion(params) { + assertSafeRuntimeDependencySpec(params.depName, params.spec); + if (exactVersionSpecRe.test(params.spec)) { + return params.spec; + } + const installedVersion = resolveInstalledPinnedDependencyVersion(params); + if (typeof installedVersion === "string" && exactVersionSpecRe.test(installedVersion)) { + return installedVersion; + } + throw new Error( + `runtime dependency ${params.depName} must resolve to an exact installed version, got: ${params.spec}`, + ); +} + +function collectRuntimeDependencyGroups(packageJson) { + const readRuntimeGroup = (group) => + Object.fromEntries( + Object.entries(group ?? {}).filter( + (entry) => typeof entry[0] === "string" && typeof entry[1] === "string", + ), + ); + return { + dependencies: readRuntimeGroup(packageJson.dependencies), + optionalDependencies: readRuntimeGroup(packageJson.optionalDependencies), + }; +} + +function resolvePinnedRuntimeDependencyGroup(group, params = {}) { + return Object.fromEntries( + Object.entries(group).map(([name, version]) => { + const pinnedVersion = resolvePinnedRuntimeDependencyVersion({ + depName: name, + parentPackageRoot: params.directDependencyPackageRoot ?? null, + rootNodeModulesDir: params.rootNodeModulesDir ?? path.join(process.cwd(), "node_modules"), + spec: version, + }); + return [name, pinnedVersion]; + }), + ); +} + +function resolvePinnedRuntimeDependencyGroups(packageJson, params = {}) { + const runtimeGroups = collectRuntimeDependencyGroups(packageJson); + return { + dependencies: resolvePinnedRuntimeDependencyGroup(runtimeGroups.dependencies, params), + optionalDependencies: resolvePinnedRuntimeDependencyGroup( + runtimeGroups.optionalDependencies, + params, + ), + }; +} + +export function collectRuntimeDependencyInstallManifest(packageJson, params = {}) { + const pinnedGroups = resolvePinnedRuntimeDependencyGroups(packageJson, params); + return createRuntimeInstallManifest(params.pluginId ?? "runtime-deps", pinnedGroups); +} + +export function collectRuntimeDependencyInstallSpecs(packageJson, params = {}) { + const manifest = collectRuntimeDependencyInstallManifest(packageJson, params); + const buildSpecs = (group) => + Object.entries(group ?? {}).map(([name, version]) => `${name}@${String(version)}`); + return { + dependencies: buildSpecs(manifest.dependencies), + optionalDependencies: buildSpecs(manifest.optionalDependencies), + }; +} + +function createRuntimeInstallManifest(pluginId, pinnedGroups) { + const manifest = { + name: `openclaw-runtime-deps-${sanitizeTempPrefixSegment(pluginId)}`, + private: true, + version: "0.0.0", + }; + if (Object.keys(pinnedGroups.dependencies).length > 0) { + manifest.dependencies = pinnedGroups.dependencies; + } + if (Object.keys(pinnedGroups.optionalDependencies).length > 0) { + manifest.optionalDependencies = pinnedGroups.optionalDependencies; + } + return manifest; +} + +function runNpmInstall(params) { + const npmEnv = { + ...(params.npmRunner.env ?? process.env), + CI: "1", + npm_config_loglevel: "error", + npm_config_yes: "true", + }; + const result = spawnSync(params.npmRunner.command, params.npmRunner.args, { + cwd: params.cwd, + encoding: "utf8", + env: npmEnv, + shell: params.npmRunner.shell, + stdio: ["ignore", "pipe", "pipe"], + timeout: params.timeoutMs ?? 5 * 60 * 1000, + windowsVerbatimArguments: params.npmRunner.windowsVerbatimArguments, + }); + if (result.status === 0) { + return; + } + const output = [result.stderr, result.stdout].filter(Boolean).join("\n").trim(); + throw new Error(output || "npm install failed"); +} + function resolveRuntimeDepsStampPath(pluginDir) { return path.join(pluginDir, ".openclaw-runtime-deps-stamp.json"); } @@ -561,7 +807,14 @@ function readRuntimeDepsStamp(stampPath) { } function stageInstalledRootRuntimeDeps(params) { - const { fingerprint, packageJson, pluginDir, pruneConfig, repoRoot } = params; + const { + directDependencyPackageRoot = null, + fingerprint, + packageJson, + pluginDir, + pruneConfig, + repoRoot, + } = params; const dependencySpecs = { ...packageJson.dependencies, ...packageJson.optionalDependencies, @@ -574,11 +827,16 @@ function stageInstalledRootRuntimeDeps(params) { const directDependencyNames = resolveInstalledDirectDependencyNames( rootNodeModulesDir, dependencySpecs, + directDependencyPackageRoot, ); if (directDependencyNames === null) { return false; } - const resolution = collectInstalledRuntimeDependencyRoots(rootNodeModulesDir, dependencySpecs); + const resolution = collectInstalledRuntimeDependencyRoots( + rootNodeModulesDir, + dependencySpecs, + directDependencyPackageRoot, + ); if (resolution === null) { return false; } @@ -588,10 +846,7 @@ function stageInstalledRootRuntimeDeps(params) { const nodeModulesDir = path.join(pluginDir, "node_modules"); const stampPath = resolveRuntimeDepsStampPath(pluginDir); const stagedNodeModulesDir = path.join( - makeTempDir( - os.tmpdir(), - `openclaw-runtime-deps-${sanitizeTempPrefixSegment(path.basename(pluginDir))}-`, - ), + makePluginOwnedTempDir(pluginDir, "stage"), "node_modules", ); @@ -620,8 +875,8 @@ function stageInstalledRootRuntimeDeps(params) { } pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig); - replaceDir(nodeModulesDir, stagedNodeModulesDir); - writeJson(stampPath, { + replaceDirAtomically(nodeModulesDir, stagedNodeModulesDir); + writeJsonAtomically(stampPath, { fingerprint, generatedAt: new Date().toISOString(), }); @@ -631,66 +886,6 @@ function stageInstalledRootRuntimeDeps(params) { } } -function installPluginRuntimeDeps(params) { - const { fingerprint, packageJson, pluginDir, pluginId, pruneConfig, repoRoot } = params; - if ( - repoRoot && - stageInstalledRootRuntimeDeps({ fingerprint, packageJson, pluginDir, pruneConfig, repoRoot }) - ) { - return; - } - const nodeModulesDir = path.join(pluginDir, "node_modules"); - const stampPath = resolveRuntimeDepsStampPath(pluginDir); - const tempInstallDir = makeTempDir( - os.tmpdir(), - `openclaw-runtime-deps-${sanitizeTempPrefixSegment(pluginId)}-`, - ); - const npmRunner = resolveNpmRunner({ - npmArgs: [ - "install", - "--omit=dev", - "--silent", - "--ignore-scripts", - "--legacy-peer-deps", - "--package-lock=false", - ], - }); - try { - writeJson(path.join(tempInstallDir, "package.json"), packageJson); - const result = spawnSync(npmRunner.command, npmRunner.args, { - cwd: tempInstallDir, - encoding: "utf8", - env: npmRunner.env, - stdio: "pipe", - shell: npmRunner.shell, - windowsVerbatimArguments: npmRunner.windowsVerbatimArguments, - }); - if (result.status !== 0) { - const output = [result.stderr, result.stdout].filter(Boolean).join("\n").trim(); - throw new Error( - `failed to stage bundled runtime deps for ${pluginId}: ${output || "npm install failed"}`, - ); - } - - const stagedNodeModulesDir = path.join(tempInstallDir, "node_modules"); - if (!fs.existsSync(stagedNodeModulesDir)) { - throw new Error( - `failed to stage bundled runtime deps for ${pluginId}: npm install produced no node_modules directory`, - ); - } - - pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig); - - replaceDir(nodeModulesDir, stagedNodeModulesDir); - writeJson(stampPath, { - fingerprint, - generatedAt: new Date().toISOString(), - }); - } finally { - removePathIfExists(tempInstallDir); - } -} - function installPluginRuntimeDepsWithRetries(params) { const { attempts = 3 } = params; let lastError; @@ -708,6 +903,86 @@ function installPluginRuntimeDepsWithRetries(params) { throw lastError; } +function createRootRuntimeStagingError(params) { + const runtimeDependencyNames = [ + ...Object.keys(params.packageJson.dependencies ?? {}), + ...Object.keys(params.packageJson.optionalDependencies ?? {}), + ].toSorted((left, right) => left.localeCompare(right)); + const dependencyLabel = + runtimeDependencyNames.length > 0 ? runtimeDependencyNames.join(", ") : ""; + const causeMessage = + params.cause instanceof Error && typeof params.cause.message === "string" + ? ` Cause: ${params.cause.message}` + : ""; + return new Error( + `failed to stage bundled runtime deps for ${params.pluginId}: ` + + `runtime dependency closure must resolve from the installed root workspace graph. ` + + `Could not materialize: ${dependencyLabel}. ` + + "Run `pnpm install` and rebuild from a trusted workspace checkout, or provide a hardened fallback installer." + + causeMessage, + ); +} + +function installPluginRuntimeDeps(params) { + const { + directDependencyPackageRoot = null, + fingerprint, + packageJson, + pluginDir, + pluginId, + pruneConfig, + repoRoot, + } = params; + const nodeModulesDir = path.join(pluginDir, "node_modules"); + const stampPath = resolveRuntimeDepsStampPath(pluginDir); + const tempInstallDir = makePluginOwnedTempDir(pluginDir, "install"); + const pinnedGroups = resolvePinnedRuntimeDependencyGroups(packageJson, { + directDependencyPackageRoot, + rootNodeModulesDir: path.join(repoRoot, "node_modules"), + }); + const requiredDependencyCount = Object.keys(pinnedGroups.dependencies).length; + try { + writeJson( + path.join(tempInstallDir, "package.json"), + createRuntimeInstallManifest(pluginId, pinnedGroups), + ); + if (requiredDependencyCount > 0 || Object.keys(pinnedGroups.optionalDependencies).length > 0) { + runNpmInstall({ + cwd: tempInstallDir, + npmRunner: resolveNpmRunner({ + npmArgs: [ + "install", + "--omit=dev", + "--ignore-scripts", + "--legacy-peer-deps", + "--package-lock=false", + "--silent", + ], + }), + }); + } + const stagedNodeModulesDir = path.join(tempInstallDir, "node_modules"); + if (requiredDependencyCount > 0 && !fs.existsSync(stagedNodeModulesDir)) { + throw new Error( + `failed to stage bundled runtime deps for ${pluginId}: explicit npm install produced no node_modules directory`, + ); + } + if (fs.existsSync(stagedNodeModulesDir)) { + pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig); + replaceDirAtomically(nodeModulesDir, stagedNodeModulesDir); + } else { + assertPathIsNotSymlink(nodeModulesDir, "remove runtime deps"); + removePathIfExists(nodeModulesDir); + } + writeJsonAtomically(stampPath, { + fingerprint, + generatedAt: new Date().toISOString(), + }); + } finally { + removePathIfExists(tempInstallDir); + } +} + export function stageBundledPluginRuntimeDeps(params = {}) { const repoRoot = params.cwd ?? params.repoRoot ?? process.cwd(); const installPluginRuntimeDepsImpl = @@ -716,6 +991,10 @@ export function stageBundledPluginRuntimeDeps(params = {}) { const pruneConfig = resolveRuntimeDepPruneConfig(params); for (const pluginDir of listBundledPluginRuntimeDirs(repoRoot)) { const pluginId = path.basename(pluginDir); + const sourcePluginRoot = resolveInstalledWorkspacePluginRoot(repoRoot, pluginId); + const directDependencyPackageRoot = fs.existsSync(path.join(sourcePluginRoot, "package.json")) + ? sourcePluginRoot + : null; const packageJson = sanitizeBundledManifestForRuntimeInstall(pluginDir); const nodeModulesDir = path.join(pluginDir, "node_modules"); const stampPath = resolveRuntimeDepsStampPath(pluginDir); @@ -725,6 +1004,7 @@ export function stageBundledPluginRuntimeDeps(params = {}) { continue; } const rootInstalledRuntimeFingerprint = resolveInstalledRuntimeClosureFingerprint({ + directDependencyPackageRoot, packageJson, rootNodeModulesDir: path.join(repoRoot, "node_modules"), }); @@ -736,18 +1016,35 @@ export function stageBundledPluginRuntimeDeps(params = {}) { if (fs.existsSync(nodeModulesDir) && stamp?.fingerprint === fingerprint) { continue; } - installPluginRuntimeDepsWithRetries({ - attempts: installAttempts, - install: installPluginRuntimeDepsImpl, - installParams: { + if ( + stageInstalledRootRuntimeDeps({ + directDependencyPackageRoot, fingerprint, packageJson, pluginDir, - pluginId, pruneConfig, repoRoot, - }, - }); + }) + ) { + continue; + } + try { + installPluginRuntimeDepsWithRetries({ + attempts: installAttempts, + install: installPluginRuntimeDepsImpl, + installParams: { + directDependencyPackageRoot, + fingerprint, + packageJson, + pluginDir, + pluginId, + pruneConfig, + repoRoot, + }, + }); + } catch (error) { + throw createRootRuntimeStagingError({ packageJson, pluginId, cause: error }); + } } } diff --git a/src/plugins/contracts/package-manifest.contract.test.ts b/src/plugins/contracts/package-manifest.contract.test.ts index 56f79c456f4..8e7fb294042 100644 --- a/src/plugins/contracts/package-manifest.contract.test.ts +++ b/src/plugins/contracts/package-manifest.contract.test.ts @@ -6,19 +6,16 @@ const packageManifestContractTests: PackageManifestContractParams[] = [ { pluginId: "bluebubbles", minHostVersionBaseline: "2026.3.22" }, { pluginId: "discord", - mirroredRootRuntimeDeps: [ - "@buape/carbon", - "@discordjs/opus", - "https-proxy-agent", - "opusscript", - ], + pluginLocalRuntimeDeps: ["@buape/carbon", "@discordjs/opus", "discord-api-types", "opusscript"], + mirroredRootRuntimeDeps: ["https-proxy-agent"], minHostVersionBaseline: "2026.3.22", }, { pluginId: "feishu", - mirroredRootRuntimeDeps: ["@larksuiteoapi/node-sdk"], + pluginLocalRuntimeDeps: ["@larksuiteoapi/node-sdk"], minHostVersionBaseline: "2026.3.22", }, + { pluginId: "google", pluginLocalRuntimeDeps: ["@google/genai"] }, { pluginId: "googlechat", mirroredRootRuntimeDeps: ["google-auth-library"], @@ -26,6 +23,11 @@ const packageManifestContractTests: PackageManifestContractParams[] = [ }, { pluginId: "irc", minHostVersionBaseline: "2026.3.22" }, { pluginId: "line", minHostVersionBaseline: "2026.3.22" }, + { pluginId: "amazon-bedrock", pluginLocalRuntimeDeps: ["@aws-sdk/client-bedrock"] }, + { + pluginId: "amazon-bedrock-mantle", + pluginLocalRuntimeDeps: ["@aws/bedrock-token-generator"], + }, { pluginId: "matrix", minHostVersionBaseline: "2026.3.22" }, { pluginId: "mattermost", minHostVersionBaseline: "2026.3.22" }, { @@ -36,9 +38,11 @@ const packageManifestContractTests: PackageManifestContractParams[] = [ { pluginId: "msteams", minHostVersionBaseline: "2026.3.22" }, { pluginId: "nextcloud-talk", minHostVersionBaseline: "2026.3.22" }, { pluginId: "nostr", minHostVersionBaseline: "2026.3.22" }, + { pluginId: "openshell", pluginLocalRuntimeDeps: ["openshell"] }, { pluginId: "slack", - mirroredRootRuntimeDeps: ["@slack/bolt", "@slack/web-api", "https-proxy-agent"], + pluginLocalRuntimeDeps: ["@slack/bolt", "@slack/web-api"], + mirroredRootRuntimeDeps: ["https-proxy-agent"], }, { pluginId: "synology-chat", minHostVersionBaseline: "2026.3.22" }, { diff --git a/src/plugins/contracts/plugin-sdk-package-contract-guardrails.test.ts b/src/plugins/contracts/plugin-sdk-package-contract-guardrails.test.ts index 8793ebd21ce..a9e0712d6d5 100644 --- a/src/plugins/contracts/plugin-sdk-package-contract-guardrails.test.ts +++ b/src/plugins/contracts/plugin-sdk-package-contract-guardrails.test.ts @@ -73,18 +73,6 @@ function readMatrixPackageJson(): { }; } -function readAmazonBedrockPackageJson(): { - dependencies?: Record; - optionalDependencies?: Record; -} { - return JSON.parse( - readFileSync(resolve(REPO_ROOT, "extensions/amazon-bedrock/package.json"), "utf8"), - ) as { - dependencies?: Record; - optionalDependencies?: Record; - }; -} - function collectRuntimeDependencySpecs(packageJson: { dependencies?: Record; optionalDependencies?: Record; @@ -318,16 +306,6 @@ describe("plugin-sdk package contract guardrails", () => { } }); - it("mirrors Bedrock runtime deps needed by the bundled host graph", () => { - const rootRuntimeDeps = collectRuntimeDependencySpecs(readRootPackageJson()); - const bedrockPackageJson = readAmazonBedrockPackageJson(); - const bedrockRuntimeDeps = collectRuntimeDependencySpecs(bedrockPackageJson); - - for (const dep of ["@aws-sdk/client-bedrock"]) { - expect(rootRuntimeDeps.get(dep)).toBe(bedrockRuntimeDeps.get(dep)); - } - }); - it("resolves matrix crypto WASM from the root runtime surface", () => { const rootRequire = createRootPackageRequire(); // Normalize filesystem separators so the package assertion stays portable. @@ -346,14 +324,9 @@ describe("plugin-sdk package contract guardrails", () => { const archivePath = packOpenClawToTempDir(packDir); const packedPackageJson = await readPackedRootPackageJson(archivePath); const matrixPackageJson = readMatrixPackageJson(); - const bedrockPackageJson = readAmazonBedrockPackageJson(); - expect(packedPackageJson.dependencies?.["@matrix-org/matrix-sdk-crypto-wasm"]).toBe( matrixPackageJson.dependencies?.["@matrix-org/matrix-sdk-crypto-wasm"], ); - expect(packedPackageJson.dependencies?.["@aws-sdk/client-bedrock"]).toBe( - bedrockPackageJson.dependencies?.["@aws-sdk/client-bedrock"], - ); expect(packedPackageJson.dependencies?.["@openclaw/plugin-package-contract"]).toBeUndefined(); }); diff --git a/src/plugins/sdk-alias.test.ts b/src/plugins/sdk-alias.test.ts index c3b91ff1ce6..cba0071797c 100644 --- a/src/plugins/sdk-alias.test.ts +++ b/src/plugins/sdk-alias.test.ts @@ -793,16 +793,21 @@ describe("plugin sdk alias helpers", () => { } }); - it("allows plugin loader dist shortcuts on non-Windows hosts", () => { + it("keeps bundled plugin dist modules on the aliased Jiti path", () => { expect( resolvePluginLoaderJitiTryNative(`/repo/${bundledDistPluginFile("browser", "index.js")}`, { preferBuiltDist: true, }), - ).toBe(true); + ).toBe(false); expect( resolvePluginLoaderJitiTryNative(`/repo/${bundledDistPluginFile("browser", "helper.ts")}`, { preferBuiltDist: true, }), + ).toBe(false); + expect( + resolvePluginLoaderJitiTryNative("/repo/dist/plugins/runtime/index.js", { + preferBuiltDist: true, + }), ).toBe(true); }); diff --git a/src/plugins/sdk-alias.ts b/src/plugins/sdk-alias.ts index 6c8d6ca3429..20eefa05661 100644 --- a/src/plugins/sdk-alias.ts +++ b/src/plugins/sdk-alias.ts @@ -472,6 +472,10 @@ function supportsNativeJitiRuntime(): boolean { return typeof versions.bun !== "string" && process.platform !== "win32"; } +function isBundledPluginDistModulePath(modulePath: string): boolean { + return modulePath.replace(/\\/g, "/").includes("/dist/extensions/"); +} + export function shouldPreferNativeJiti(modulePath: string): boolean { if (!supportsNativeJitiRuntime()) { return false; @@ -493,6 +497,9 @@ export function resolvePluginLoaderJitiTryNative( preferBuiltDist?: boolean; }, ): boolean { + if (isBundledPluginDistModulePath(modulePath)) { + return false; + } return ( shouldPreferNativeJiti(modulePath) || (supportsNativeJitiRuntime() && diff --git a/src/plugins/stage-bundled-plugin-runtime-deps.test.ts b/src/plugins/stage-bundled-plugin-runtime-deps.test.ts index f25e287fbb1..830576b52d8 100644 --- a/src/plugins/stage-bundled-plugin-runtime-deps.test.ts +++ b/src/plugins/stage-bundled-plugin-runtime-deps.test.ts @@ -252,7 +252,7 @@ describe("stageBundledPluginRuntimeDeps", () => { }); }); - it("strips non-runtime dependency sections before temp npm staging", async () => { + it("strips non-runtime dependency sections before fallback runtime staging", async () => { const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-manifest-"); writeRepoFile( repoRoot, diff --git a/test/openclaw-prepack.test.ts b/test/openclaw-prepack.test.ts index 0db2dd75a7c..a47b36ea5e9 100644 --- a/test/openclaw-prepack.test.ts +++ b/test/openclaw-prepack.test.ts @@ -1,18 +1,5 @@ import { describe, expect, it } from "vitest"; -import { collectPreparedPrepackErrors, shouldSkipPrepack } from "../scripts/openclaw-prepack.ts"; - -describe("shouldSkipPrepack", () => { - it("treats unset and explicit false values as disabled", () => { - expect(shouldSkipPrepack({})).toBe(false); - expect(shouldSkipPrepack({ OPENCLAW_PREPACK_PREPARED: "0" })).toBe(false); - expect(shouldSkipPrepack({ OPENCLAW_PREPACK_PREPARED: "false" })).toBe(false); - }); - - it("treats non-false values as enabled", () => { - expect(shouldSkipPrepack({ OPENCLAW_PREPACK_PREPARED: "1" })).toBe(true); - expect(shouldSkipPrepack({ OPENCLAW_PREPACK_PREPARED: "true" })).toBe(true); - }); -}); +import { collectPreparedPrepackErrors } from "../scripts/openclaw-prepack.ts"; describe("collectPreparedPrepackErrors", () => { it("accepts prepared release artifacts", () => { diff --git a/test/release-check.test.ts b/test/release-check.test.ts index 27d839fcbbc..b3f7213b3b9 100644 --- a/test/release-check.test.ts +++ b/test/release-check.test.ts @@ -122,6 +122,14 @@ describe("bundled plugin root runtime mirrors", () => { function makeBundledSpecs() { return new Map([ ["@larksuiteoapi/node-sdk", { conflicts: [], pluginIds: ["feishu"], spec: "^1.60.0" }], + [ + "@matrix-org/matrix-sdk-crypto-nodejs", + { conflicts: [], pluginIds: ["matrix"], spec: "^0.4.0" }, + ], + [ + "@matrix-org/matrix-sdk-crypto-wasm", + { conflicts: [], pluginIds: ["matrix"], spec: "18.0.0" }, + ], ]); } @@ -156,8 +164,18 @@ describe("bundled plugin root runtime mirrors", () => { distDir, }); - expect([...mirrors.keys()]).toEqual(["@larksuiteoapi/node-sdk"]); + expect([...mirrors.keys()].toSorted((left, right) => left.localeCompare(right))).toEqual([ + "@larksuiteoapi/node-sdk", + "@matrix-org/matrix-sdk-crypto-nodejs", + "@matrix-org/matrix-sdk-crypto-wasm", + ]); expect([...mirrors.get("@larksuiteoapi/node-sdk")!.importers]).toEqual(["probe-Cz2PiFtC.js"]); + expect([...mirrors.get("@matrix-org/matrix-sdk-crypto-nodejs")!.importers]).toEqual([ + "", + ]); + expect([...mirrors.get("@matrix-org/matrix-sdk-crypto-wasm")!.importers]).toEqual([ + "", + ]); } finally { rmSync(tempRoot, { recursive: true, force: true }); } @@ -247,7 +265,7 @@ describe("bundled plugin root runtime mirrors", () => { }); describe("collectForbiddenPackPaths", () => { - it("allows bundled plugin runtime deps under dist/extensions but still blocks other node_modules", () => { + it("blocks all packaged node_modules payloads", () => { expect( collectForbiddenPackPaths([ "dist/index.js", @@ -255,7 +273,11 @@ describe("collectForbiddenPackPaths", () => { bundledPluginFile("tlon", "node_modules/.bin/tlon"), "node_modules/.bin/openclaw", ]), - ).toEqual([bundledPluginFile("tlon", "node_modules/.bin/tlon"), "node_modules/.bin/openclaw"]); + ).toEqual([ + bundledDistPluginFile("discord", "node_modules/@buape/carbon/index.js"), + bundledPluginFile("tlon", "node_modules/.bin/tlon"), + "node_modules/.bin/openclaw", + ]); }); it("blocks generated docs artifacts from npm pack output", () => { @@ -296,6 +318,7 @@ describe("collectMissingPackPaths", () => { "dist/control-ui/index.html", "qa/scenarios/index.md", "scripts/npm-runner.mjs", + "scripts/preinstall-package-manager-warning.mjs", "scripts/postinstall-bundled-plugins.mjs", bundledDistPluginFile("diffs", "assets/viewer-runtime.js"), bundledDistPluginFile("matrix", "helper-api.js"), @@ -327,6 +350,7 @@ describe("collectMissingPackPaths", () => { ...requiredPluginSdkPackPaths, ...WORKSPACE_TEMPLATE_PACK_PATHS, "scripts/npm-runner.mjs", + "scripts/preinstall-package-manager-warning.mjs", "scripts/postinstall-bundled-plugins.mjs", "dist/plugin-sdk/root-alias.cjs", "dist/build-info.json", diff --git a/test/scripts/preinstall-package-manager-warning.test.ts b/test/scripts/preinstall-package-manager-warning.test.ts new file mode 100644 index 00000000000..afadea22b27 --- /dev/null +++ b/test/scripts/preinstall-package-manager-warning.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createPackageManagerWarningMessage, + detectLifecyclePackageManager, + warnIfNonPnpmLifecycle, +} from "../../scripts/preinstall-package-manager-warning.mjs"; + +describe("detectLifecyclePackageManager", () => { + it("prefers npm_config_user_agent when present", () => { + expect( + detectLifecyclePackageManager({ + npm_config_user_agent: "npm/11.4.1 node/v22.20.0 darwin arm64", + }), + ).toBe("npm"); + }); + + it("falls back to npm_execpath when user agent is missing", () => { + expect( + detectLifecyclePackageManager({ + npm_execpath: "/Users/test/.cache/node/corepack/v1/pnpm/10.32.1/bin/pnpm.cjs", + }), + ).toBe("pnpm"); + }); + + it("ignores untrusted user-agent tokens with control characters", () => { + expect( + detectLifecyclePackageManager({ + npm_config_user_agent: "\u001bnpm/11.4.1 node/v22.20.0 darwin arm64", + npm_execpath: "/Users/test/.cache/node/corepack/v1/pnpm/10.32.1/bin/pnpm.cjs", + }), + ).toBe("pnpm"); + }); +}); + +describe("createPackageManagerWarningMessage", () => { + it("returns null for pnpm", () => { + expect(createPackageManagerWarningMessage("pnpm")).toBeNull(); + }); + + it("warns for npm installs", () => { + expect(createPackageManagerWarningMessage("npm")).toContain("prefer: corepack pnpm install"); + }); +}); + +describe("warnIfNonPnpmLifecycle", () => { + it("warns once for npm lifecycle runs", () => { + const warn = vi.fn(); + expect( + warnIfNonPnpmLifecycle( + { + npm_config_user_agent: "npm/11.4.1 node/v22.20.0 darwin arm64", + }, + warn, + ), + ).toBe(true); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[0]).toContain("detected npm"); + }); + + it("stays quiet for pnpm", () => { + const warn = vi.fn(); + expect( + warnIfNonPnpmLifecycle( + { + npm_config_user_agent: "pnpm/10.32.1 npm/? node/v22.20.0 darwin arm64", + }, + warn, + ), + ).toBe(false); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/test/scripts/root-dependency-ownership-audit.test.ts b/test/scripts/root-dependency-ownership-audit.test.ts new file mode 100644 index 00000000000..cb5ad499504 --- /dev/null +++ b/test/scripts/root-dependency-ownership-audit.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; +import { + classifyRootDependencyOwnership, + collectModuleSpecifiers, +} from "../../scripts/root-dependency-ownership-audit.mjs"; + +describe("collectModuleSpecifiers", () => { + it("captures require.resolve package lookups used by runtime shims and bundled plugins", () => { + expect([ + ...collectModuleSpecifiers(` + const require = createRequire(import.meta.url); + const runtimeRequire = createRequire(runtimePackagePath); + require.resolve("gaxios"); + runtimeRequire.resolve("openshell/package.json"); + `), + ]).toEqual(["gaxios", "openshell/package.json"]); + }); +}); + +describe("classifyRootDependencyOwnership", () => { + it("treats root-dist bundled runtime mirrors as blocked extension deps", () => { + expect( + classifyRootDependencyOwnership({ + sections: ["extensions"], + rootMirrorImporters: ["discovery-DZDwKJdJ.js"], + }), + ).toEqual({ + category: "extension_only_root_mirror", + recommendation: + "blocked by packaged host graph: remove root mirror only after bundled runtime resolution stops importing it from root dist", + }); + }); + + it("treats scripts and tests as dev-only candidates", () => { + expect( + classifyRootDependencyOwnership({ + sections: ["scripts", "test"], + rootMirrorImporters: [], + }), + ).toEqual({ + category: "script_or_test_only", + recommendation: "consider moving from dependencies to devDependencies", + }); + }); + + it("treats extension-only deps as localizable when no root mirror exists", () => { + expect( + classifyRootDependencyOwnership({ + sections: ["extensions", "test"], + rootMirrorImporters: [], + }), + ).toEqual({ + category: "extension_only_localizable", + recommendation: + "candidate to remove from root package.json and rely on owning extension manifests", + }); + }); + + it("treats src-owned deps as core runtime", () => { + expect( + classifyRootDependencyOwnership({ + sections: ["src"], + rootMirrorImporters: [], + }), + ).toEqual({ + category: "core_runtime", + recommendation: "keep at root", + }); + }); + + it("treats unreferenced deps as removal candidates", () => { + expect( + classifyRootDependencyOwnership({ + sections: [], + rootMirrorImporters: [], + }), + ).toEqual({ + category: "unreferenced", + recommendation: "investigate removal; no direct source imports found in scanned files", + }); + }); +}); diff --git a/test/scripts/stage-bundled-plugin-runtime-deps.test.ts b/test/scripts/stage-bundled-plugin-runtime-deps.test.ts index 4a365b31e0c..ae6d5b6b2bc 100644 --- a/test/scripts/stage-bundled-plugin-runtime-deps.test.ts +++ b/test/scripts/stage-bundled-plugin-runtime-deps.test.ts @@ -1,7 +1,11 @@ import fs from "node:fs"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { stageBundledPluginRuntimeDeps } from "../../scripts/stage-bundled-plugin-runtime-deps.mjs"; +import { + collectRuntimeDependencyInstallManifest, + collectRuntimeDependencyInstallSpecs, + stageBundledPluginRuntimeDeps, +} from "../../scripts/stage-bundled-plugin-runtime-deps.mjs"; import { createScriptTestHarness } from "./test-helpers.js"; const { createTempDir } = createScriptTestHarness(); @@ -23,6 +27,90 @@ describe("stageBundledPluginRuntimeDeps", () => { return { pluginDir, repoRoot }; } + it("pins fallback install specs to exact installed versions", () => { + const { repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { + direct: "^1.0.0", + }, + optionalDependencies: { + optional: "~2.0.0", + }, + }, + }); + const rootNodeModulesDir = path.join(repoRoot, "node_modules"); + fs.mkdirSync(path.join(rootNodeModulesDir, "direct"), { recursive: true }); + fs.mkdirSync(path.join(rootNodeModulesDir, "optional"), { recursive: true }); + fs.writeFileSync( + path.join(rootNodeModulesDir, "direct", "package.json"), + '{ "name": "direct", "version": "1.2.3" }\n', + "utf8", + ); + fs.writeFileSync( + path.join(rootNodeModulesDir, "optional", "package.json"), + '{ "name": "optional", "version": "2.0.4" }\n', + "utf8", + ); + + expect( + collectRuntimeDependencyInstallSpecs( + { + dependencies: { direct: "^1.0.0" }, + optionalDependencies: { optional: "~2.0.0" }, + }, + { rootNodeModulesDir }, + ), + ).toEqual({ + dependencies: ["direct@1.2.3"], + optionalDependencies: ["optional@2.0.4"], + }); + }); + + it("rejects unsafe runtime dependency specs for fallback installs", () => { + expect(() => + collectRuntimeDependencyInstallSpecs( + { + dependencies: { direct: "file:/etc/passwd" }, + }, + { rootNodeModulesDir: "/tmp/node_modules" }, + ), + ).toThrow(/disallowed runtime dependency spec for direct: file:\/etc\/passwd/u); + }); + + it("writes required and optional fallback deps into one manifest", () => { + const rootNodeModulesDir = createTempDir("openclaw-runtime-deps-manifest-"); + fs.mkdirSync(path.join(rootNodeModulesDir, "direct"), { recursive: true }); + fs.mkdirSync(path.join(rootNodeModulesDir, "optional"), { recursive: true }); + fs.writeFileSync( + path.join(rootNodeModulesDir, "direct", "package.json"), + '{ "name": "direct", "version": "1.2.3" }\n', + "utf8", + ); + fs.writeFileSync( + path.join(rootNodeModulesDir, "optional", "package.json"), + '{ "name": "optional", "version": "2.0.4" }\n', + "utf8", + ); + + expect( + collectRuntimeDependencyInstallManifest( + { + dependencies: { direct: "^1.0.0" }, + optionalDependencies: { optional: "~2.0.0" }, + }, + { pluginId: "fixture-plugin", rootNodeModulesDir }, + ), + ).toEqual({ + name: "openclaw-runtime-deps-fixture-plugin", + private: true, + version: "0.0.0", + dependencies: { direct: "1.2.3" }, + optionalDependencies: { optional: "2.0.4" }, + }); + }); + it("skips restaging when runtime deps stamp matches the sanitized manifest", () => { const { pluginDir, repoRoot } = createBundledPluginFixture({ packageJson: { @@ -194,6 +282,60 @@ describe("stageBundledPluginRuntimeDeps", () => { ).toBe("module.exports = 'second';\n"); }); + it("refuses to replace a symlinked plugin node_modules directory", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { direct: "1.0.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + const directDir = path.join(repoRoot, "node_modules", "direct"); + const outsideDir = path.join(repoRoot, "outside-node-modules"); + const nodeModulesDir = path.join(pluginDir, "node_modules"); + fs.mkdirSync(directDir, { recursive: true }); + fs.mkdirSync(outsideDir, { recursive: true }); + fs.writeFileSync( + path.join(directDir, "package.json"), + '{ "name": "direct", "version": "1.0.0" }\n', + "utf8", + ); + fs.writeFileSync(path.join(directDir, "index.js"), "module.exports = 'direct';\n", "utf8"); + fs.symlinkSync(outsideDir, nodeModulesDir); + + expect(() => stageBundledPluginRuntimeDeps({ cwd: repoRoot })).toThrow( + /refusing to replace runtime deps via symlinked path/u, + ); + }); + + it("refuses to write a runtime deps stamp through a symlink", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { direct: "1.0.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + const directDir = path.join(repoRoot, "node_modules", "direct"); + const outsideStamp = path.join(repoRoot, "outside-stamp.json"); + const stampPath = path.join(pluginDir, ".openclaw-runtime-deps-stamp.json"); + fs.mkdirSync(directDir, { recursive: true }); + fs.writeFileSync( + path.join(directDir, "package.json"), + '{ "name": "direct", "version": "1.0.0" }\n', + "utf8", + ); + fs.writeFileSync(path.join(directDir, "index.js"), "module.exports = 'direct';\n", "utf8"); + fs.writeFileSync(outsideStamp, '{"outside":true}\n', "utf8"); + fs.symlinkSync(outsideStamp, stampPath); + + expect(() => stageBundledPluginRuntimeDeps({ cwd: repoRoot })).toThrow( + /refusing to write runtime deps stamp via symlinked path/u, + ); + }); + it("stages runtime deps from the root node_modules when already installed", () => { const { pluginDir, repoRoot } = createBundledPluginFixture({ packageJson: { diff --git a/tsdown.config.ts b/tsdown.config.ts index 0c5ef3c3497..fe8336674b6 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { defineConfig, type UserConfig } from "tsdown"; import { - listBundledPluginBuildEntries, + collectBundledPluginBuildEntries, listBundledPluginRuntimeDependencies, } from "./scripts/lib/bundled-plugin-build-entries.mjs"; import { buildPluginSdkEntrySources } from "./scripts/lib/plugin-sdk-entries.mjs"; @@ -90,7 +90,7 @@ function nodeBuildConfig(config: UserConfig): UserConfig { }; } -const bundledPluginBuildEntries = listBundledPluginBuildEntries(); +const bundledPluginBuildEntries = collectBundledPluginBuildEntries(); const bundledPluginRuntimeDependencies = listBundledPluginRuntimeDependencies(); function buildBundledHookEntries(): Record { @@ -135,6 +135,70 @@ function shouldNeverBundleDependency(id: string): boolean { }); } +function shouldStageBundledPluginRuntimeDependencies(packageJson: unknown): boolean { + return ( + typeof packageJson === "object" && + packageJson !== null && + (packageJson as { openclaw?: { bundle?: { stageRuntimeDependencies?: boolean } } }).openclaw + ?.bundle?.stageRuntimeDependencies === true + ); +} + +function listBundledPluginEntrySources( + entries: Array<{ + id: string; + packageJson: unknown; + sourceEntries: string[]; + }>, +): Record { + return Object.fromEntries( + entries.flatMap(({ id, sourceEntries }) => + sourceEntries.map((entry) => { + const normalizedEntry = entry.replace(/^\.\//u, ""); + const entryKey = bundledPluginFile(id, normalizedEntry.replace(/\.[^.]+$/u, "")); + return [ + entryKey, + normalizedEntry ? `extensions/${id}/${normalizedEntry}` : `extensions/${id}`, + ]; + }), + ), + ); +} + +function normalizeBundledPluginOutEntry(entry: string): string { + return entry.replace(/^\.\//u, "").replace(/\.[^.]+$/u, ""); +} + +function isPluginSdkSelfReference(id: string): boolean { + return ( + id === "openclaw/plugin-sdk" || + id.startsWith("openclaw/plugin-sdk/") || + id === "@openclaw/plugin-sdk" || + id.startsWith("@openclaw/plugin-sdk/") + ); +} + +function buildBundledPluginNeverBundlePredicate(packageJson: { + dependencies?: Record; + optionalDependencies?: Record; +}) { + const runtimeDependencies = shouldStageBundledPluginRuntimeDependencies(packageJson) + ? [ + ...Object.keys(packageJson.dependencies ?? {}), + ...Object.keys(packageJson.optionalDependencies ?? {}), + ].toSorted((left, right) => left.localeCompare(right)) + : []; + + return (id: string): boolean => { + if (isPluginSdkSelfReference(id)) { + return true; + } + return runtimeDependencies.some((dependency) => { + return id === dependency || id.startsWith(`${dependency}/`); + }); + }; +} + function buildCoreDistEntries(): Record { return { index: "src/index.ts", @@ -167,6 +231,12 @@ function buildCoreDistEntries(): Record { } const coreDistEntries = buildCoreDistEntries(); +const stagedBundledPluginBuildEntries = bundledPluginBuildEntries.filter(({ packageJson }) => + shouldStageBundledPluginRuntimeDependencies(packageJson), +); +const rootBundledPluginBuildEntries = bundledPluginBuildEntries.filter( + ({ packageJson }) => !shouldStageBundledPluginRuntimeDependencies(packageJson), +); function buildUnifiedDistEntries(): Record { return { @@ -179,18 +249,43 @@ function buildUnifiedDistEntries(): Record { source, ]), ), - ...bundledPluginBuildEntries, + ...listBundledPluginEntrySources(rootBundledPluginBuildEntries), ...bundledHookEntries, }; } +function buildBundledPluginConfigs(): UserConfig[] { + return stagedBundledPluginBuildEntries.map(({ id, packageJson, sourceEntries }) => + nodeBuildConfig({ + clean: false, + entry: Object.fromEntries( + sourceEntries.map((entry) => [ + normalizeBundledPluginOutEntry(entry), + `extensions/${id}/${entry.replace(/^\.\//u, "")}`, + ]), + ), + outDir: `dist/extensions/${id}`, + deps: { + neverBundle: buildBundledPluginNeverBundlePredicate( + (packageJson ?? {}) as { + dependencies?: Record; + optionalDependencies?: Record; + }, + ), + }, + }), + ); +} + export default defineConfig([ nodeBuildConfig({ // Build core entrypoints, plugin-sdk subpaths, bundled plugin entrypoints, // and bundled hooks in one graph so runtime singletons are emitted once. + clean: true, entry: buildUnifiedDistEntries(), deps: { neverBundle: shouldNeverBundleDependency, }, }), + ...buildBundledPluginConfigs(), ]);