From 1c968a7bfc1ca4f24cd157fce16e2a7fa35bab21 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 30 May 2026 19:52:45 +0200 Subject: [PATCH] feat(scripts): added native leaf-package publish checks to install smoke tests - Added repeatable `--tag` argument parsing in `gen-npm-packages.ts` and threaded parsed tags into `generateNpmPackages` for targeted leaf publishing. - Exported `prepareNativeCorePackage` and expanded package manifest typing to support scripted manifest rewrites used by release workflows. - Reworked install-test smoke logic to pack the host leaf package, pack the rewritten natives core, and assert the platform leaf package resolves from the core optional dependency. --- docs/natives-addon-loader-runtime.md | 12 ++++-- docs/natives-architecture.md | 7 +++- docs/porting-to-natives.md | 2 +- packages/natives/scripts/gen-npm-packages.ts | 22 ++++++++++- scripts/ci-release-publish.ts | 5 ++- scripts/eval-bench-runs.ts | 10 +++-- scripts/install-tests/run-ci.sh | 39 +++++++++++++++++++- tsconfig.tools.json | 2 +- 8 files changed, 83 insertions(+), 16 deletions(-) diff --git a/docs/natives-addon-loader-runtime.md b/docs/natives-addon-loader-runtime.md index 5fc02a185..e511049ef 100644 --- a/docs/natives-addon-loader-runtime.md +++ b/docs/natives-addon-loader-runtime.md @@ -28,6 +28,7 @@ At module initialization, `native/index.js` computes: - **Platform tag**: `${process.platform}-${process.arch}` (for example `darwin-arm64`). - **Package version**: from `packages/natives/package.json`. - **Core directories**: + - `leafPackageDir`: directory of the platform leaf package, resolved via `require.resolve("@oh-my-pi/pi-natives-/package.json")`; `null` when no leaf is installed (e.g. local dev). - `nativeDir`: package-local `packages/natives/native`. - `execDir`: directory containing `process.execPath`. - `versionedDir`: `/`. @@ -92,10 +93,15 @@ The default unsuffixed fallback remains part of the x64 candidate list. ### Non-compiled runtime -For each filename, candidates are: +For each filename, candidates are, in order: -1. `/` -2. `/` +1. `/` (omitted when `leafPackageDir` is `null`) +2. `/` +3. `/` + +The leaf package dir comes first so the optional-dependency binary published with the release is preferred over any `.node` left in the core package's `native/` (e.g. a stale local-dev build). + +On Windows installs where `nativeDir` is inside a `node_modules` segment (`shouldStageNodeModulesAddon`), `/` staging candidates are prepended ahead of the leaf candidates so a locked `node_modules` binary can be sidestepped during `bun install -g` updates. ### Compiled runtime diff --git a/docs/natives-architecture.md b/docs/natives-architecture.md index f8271da77..062495f62 100644 --- a/docs/natives-architecture.md +++ b/docs/natives-architecture.md @@ -72,7 +72,9 @@ For x64, variant selection uses: ### Binary distribution and extraction model -`packages/natives/package.json` publishes `native/`, which contains the loader, generated declarations, generated enum patch, embedded-addon manifest stub, and prebuilt `.node` artifacts. +The published `@oh-my-pi/pi-natives` package ships **only** the loader layer in `native/`: the CommonJS loader (`index.js`), generated declarations (`index.d.ts`), the `loader-state.js`/`.d.ts` helpers, and the embedded-addon manifest stub (`embedded-addon.js`). It carries no `.node` binaries. + +Each platform's prebuilt `.node` is published as a separate optional-dependency leaf package — `@oh-my-pi/pi-natives--`, one per supported tag — which the core lists in `optionalDependencies` at the lockstep version. npm/bun install only the leaf whose `os`/`cpu` match the host. The working-tree package keeps built `.node` files under `native/` for local dev; the release-publish rewrite (`prepareNativeCorePackage` in `scripts/ci-release-publish.ts`) strips them from the core tarball, and the leaves are generated by `packages/natives/scripts/gen-npm-packages.ts` (`LEAF_TARGETS`). Adding a build target therefore requires a matching `LEAF_TARGETS` entry, or the binary never reaches npm users. For compiled binaries, loader behavior is: @@ -86,6 +88,8 @@ For compiled binaries, loader behavior is: If a populated embedded addon manifest is present, it is also treated as a compiled-binary signal. The loader can extract the matching embedded `.node` into the versioned cache directory before candidate probing. +For npm/bun installs (non-compiled), `loader-state.js` resolves the platform leaf directory via `require.resolve("@oh-my-pi/pi-natives-/package.json")` and probes its `.node` **before** the core package's `native/` directory and the executable directory. The optional-dependency binary is therefore preferred over any `.node` left in the core (e.g. a stale local-dev build). + ### Failure modes Loader failures are explicit: @@ -154,6 +158,7 @@ N-API exports are generated from Rust `#[napi]` functions/classes/objects/enums. - **Native addon**: A `.node` binary loaded via Node-API (N-API). - **Platform tag**: Runtime tuple `platform-arch` (for example `darwin-arm64`). +- **Platform leaf package**: Per-platform npm package `@oh-my-pi/pi-natives-` that carries one platform's prebuilt `.node`. The core depends on every leaf via `optionalDependencies`; the package manager installs only the host-matching one (`os`/`cpu`). - **Variant**: x64 CPU-specific build flavor (`modern` AVX2, `baseline` fallback). - **Generated binding declaration**: `native/index.d.ts` emitted by napi-rs during `build-native.ts`. - **Compiled binary mode**: Runtime mode where the CLI is bundled and native addons are resolved from embedded/cache paths before package-local paths. diff --git a/docs/porting-to-natives.md b/docs/porting-to-natives.md index 0fb9b84de..391c7b5a3 100644 --- a/docs/porting-to-natives.md +++ b/docs/porting-to-natives.md @@ -40,7 +40,7 @@ Consumers import directly from `@oh-my-pi/pi-natives`. The generated declaration - `packages/natives/scripts/build-native.ts` runs napi-rs, installs the `.node` artifact, copies generated `index.js`/`index.d.ts`, and appends enum runtime exports. - `packages/natives/native/index.js` is the loader that chooses a candidate `.node` file and returns the loaded addon. -- `packages/natives/package.json` exposes only the package root (`@oh-my-pi/pi-natives`). +- `packages/natives/package.json` exposes only the package root (`@oh-my-pi/pi-natives`) as the import surface. At publish time the binaries are split out: the core ships the loader only (no `.node`), and each platform's `.node` is published as an optional-dependency leaf package `@oh-my-pi/pi-natives-` (`scripts/ci-release-publish.ts` + `packages/natives/scripts/gen-npm-packages.ts`). This is transparent to importers — you still `import` from `@oh-my-pi/pi-natives`. **Consumer side:** diff --git a/packages/natives/scripts/gen-npm-packages.ts b/packages/natives/scripts/gen-npm-packages.ts index 0b5b9d970..7ebd035d2 100755 --- a/packages/natives/scripts/gen-npm-packages.ts +++ b/packages/natives/scripts/gen-npm-packages.ts @@ -161,6 +161,24 @@ export async function generateNpmPackages({ return leaves; } -if (import.meta.main) { - await generateNpmPackages({ dryRun: process.argv.includes("--dry-run") }); +/** Parse repeatable `--tag ` / `--tag=` flags; undefined means all targets. */ +function parseTagArgs(argv: readonly string[]): readonly string[] | undefined { + const tags: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--tag") { + const value = argv[i + 1]; + if (!value) throw new Error("--tag requires a native target tag"); + tags.push(value); + i++; + } else if (arg.startsWith("--tag=")) { + tags.push(arg.slice("--tag=".length)); + } + } + return tags.length > 0 ? tags : undefined; +} + +if (import.meta.main) { + const argv = process.argv.slice(2); + await generateNpmPackages({ dryRun: argv.includes("--dry-run"), tags: parseTagArgs(argv) }); } diff --git a/scripts/ci-release-publish.ts b/scripts/ci-release-publish.ts index 172e903ab..fcc1eab3c 100644 --- a/scripts/ci-release-publish.ts +++ b/scripts/ci-release-publish.ts @@ -49,7 +49,8 @@ type JsonValue = string | number | boolean | null | JsonObject | JsonValue[]; interface JsonObject { [key: string]: JsonValue; } -interface PackageManifest extends JsonObject { +interface PackageManifest { + [key: string]: JsonValue | undefined; name?: string; version?: string; private?: boolean; @@ -159,7 +160,7 @@ function buildNativeOptionalDependencies(version: string): JsonObject { return optionalDependencies; } -async function prepareNativeCorePackage(pkgDir: string, write: boolean): Promise { +export async function prepareNativeCorePackage(pkgDir: string, write: boolean): Promise { const manifestPath = path.join(pkgDir, "package.json"); const manifest = (await Bun.file(manifestPath).json()) as PackageManifest; if (typeof manifest.version !== "string") throw new Error(`Missing version in ${manifestPath}`); diff --git a/scripts/eval-bench-runs.ts b/scripts/eval-bench-runs.ts index 5b0777fd2..1bf40edd3 100644 --- a/scripts/eval-bench-runs.ts +++ b/scripts/eval-bench-runs.ts @@ -57,16 +57,18 @@ const SEPARATOR_DISPLAY: Record = { const args = process.argv.slice(2); const dirs: string[] = []; -let format: "table" | "md" | "csv" | "json" = "table"; -let sortBy: "sep" | "model" | "task" | "edit" | "tokens" = "sep"; +type OutputFormat = "table" | "md" | "csv" | "json"; +type SortKey = "sep" | "model" | "task" | "edit" | "tokens"; +let format: OutputFormat = "table"; +let sortBy: SortKey = "sep"; let aggregate = false; for (let i = 0; i < args.length; i++) { const a = args[i]; if (a === "--format") { - format = args[++i] as typeof format; + format = args[++i] as OutputFormat; } else if (a === "--sort") { - sortBy = args[++i] as typeof sortBy; + sortBy = args[++i] as SortKey; } else if (a === "--aggregate") { aggregate = true; } else if (!a.startsWith("--")) { diff --git a/scripts/install-tests/run-ci.sh b/scripts/install-tests/run-ci.sh index 853723ba3..5864f7a9c 100755 --- a/scripts/install-tests/run-ci.sh +++ b/scripts/install-tests/run-ci.sh @@ -66,7 +66,35 @@ SOURCE_BUN_HOME="$WORK_DIR/bun-source" section "Tarball install smoke" TARBALL_DIR="$WORK_DIR/tarballs" mkdir -p "$TARBALL_DIR" -for pkg in utils natives hashline ai mnemosyne agent tui stats coding-agent; do +host_tag="$(bun -e "process.stdout.write(\`\${process.platform}-\${process.arch}\`)")" + +# Native addon split: the published core ships only the loader (no `.node`); the +# prebuilt binary lives in a per-platform leaf package pulled in as an optional +# dependency. Reproduce that exact published topology so this smoke proves the +# installed core resolves its addon through the leaf, not a bundled binary. + +# 1. Generate + pack the host-platform leaf (carries the built `.node`). +bun --cwd=packages/natives run gen:npm --tag "$host_tag" >/dev/null +( + cd "$ROOT_DIR/packages/natives/npm/$host_tag" + bun pm pack --destination "$TARBALL_DIR" --quiet >/dev/null +) + +# 2. Pack the core with its *published* manifest: the same rewrite release uses +# drops `.node` from `files` and adds the leaf `optionalDependencies`. Always +# restore the working-tree manifest so local runs aren't left mutated. +natives_pkg_backup="$WORK_DIR/natives-package.json.orig" +cp "$ROOT_DIR/packages/natives/package.json" "$natives_pkg_backup" +core_rc=0 +{ + bun -e 'import { prepareNativeCorePackage } from "./scripts/ci-release-publish.ts"; await prepareNativeCorePackage("packages/natives", true);' && + ( cd "$ROOT_DIR/packages/natives" && bun pm pack --destination "$TARBALL_DIR" --quiet >/dev/null ) +} || core_rc=$? +cp "$natives_pkg_backup" "$ROOT_DIR/packages/natives/package.json" +[ "$core_rc" -eq 0 ] || exit "$core_rc" + +# 3. Pack the remaining workspace packages (natives core handled above). +for pkg in utils hashline ai mnemosyne agent tui stats coding-agent; do ( cd "$ROOT_DIR/packages/$pkg" bun pm pack --destination "$TARBALL_DIR" --quiet >/dev/null @@ -74,7 +102,8 @@ for pkg in utils natives hashline ai mnemosyne agent tui stats coding-agent; do done utils_tgz="$(find_tarball "$TARBALL_DIR"/oh-my-pi-pi-utils-*.tgz)" -natives_tgz="$(find_tarball "$TARBALL_DIR"/oh-my-pi-pi-natives-*.tgz)" +natives_tgz="$(find_tarball "$TARBALL_DIR"/oh-my-pi-pi-natives-[0-9]*.tgz)" +natives_leaf_tgz="$(find_tarball "$TARBALL_DIR"/oh-my-pi-pi-natives-"$host_tag"-*.tgz)" hashline_tgz="$(find_tarball "$TARBALL_DIR"/oh-my-pi-hashline-*.tgz)" ai_tgz="$(find_tarball "$TARBALL_DIR"/oh-my-pi-pi-ai-*.tgz)" mnemosyne_tgz="$(find_tarball "$TARBALL_DIR"/oh-my-pi-pi-mnemosyne-*.tgz)" @@ -96,6 +125,7 @@ mkdir -p "$TARBALL_APP_DIR" pkg.overrides = { '@oh-my-pi/pi-utils': '$utils_tgz', '@oh-my-pi/pi-natives': '$natives_tgz', + '@oh-my-pi/pi-natives-$host_tag': '$natives_leaf_tgz', '@oh-my-pi/hashline': '$hashline_tgz', '@oh-my-pi/pi-ai': '$ai_tgz', '@oh-my-pi/pi-mnemosyne': '$mnemosyne_tgz', @@ -108,6 +138,11 @@ mkdir -p "$TARBALL_APP_DIR" " bun add "$utils_tgz" "$natives_tgz" "$hashline_tgz" "$ai_tgz" "$mnemosyne_tgz" "$agent_tgz" "$tui_tgz" "$stats_tgz" "$coding_agent_tgz" + # The platform leaf must arrive through the core's optionalDependencies + + # override, not as a direct dependency — assert it landed before smoking so a + # resolution regression is distinguishable from a runtime loader bug. + leaf_dir="node_modules/@oh-my-pi/pi-natives-$host_tag" + [ -d "$leaf_dir" ] || { echo "Platform leaf package not installed: $leaf_dir"; exit 1; } smoke_cli ./node_modules/.bin/omp ) diff --git a/tsconfig.tools.json b/tsconfig.tools.json index ec05cd8a8..f0426ac9c 100644 --- a/tsconfig.tools.json +++ b/tsconfig.tools.json @@ -6,6 +6,6 @@ "emitDeclarationOnly": false, "allowImportingTsExtensions": true }, - "include": ["scripts"], + "include": ["scripts", "packages/natives/scripts/gen-npm-packages.ts"], "exclude": ["node_modules"] }