From 34e099148a6c5b6d916fa9e70167fa2364d57791 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 30 May 2026 17:08:23 +0200 Subject: [PATCH] feat(natives): added per-platform leaf package publishing for native addons - Added `gen-npm-packages.ts` script to generate per-platform leaf packages under `npm/-/`. - Updated CI release publisher to generate and publish leaf packages before rewriting the core manifest with pinned `optionalDependencies`. - Core package now ships only JS loader and declarations; installs fetch only the host platform's `.node` binary. --- .gitignore | 1 + packages/natives/CHANGELOG.md | 4 + packages/natives/README.md | 28 +++- packages/natives/package.json | 1 + packages/natives/scripts/gen-npm-packages.ts | 152 ++++++++++++++++++ packages/natives/test/issue-823-repro.test.ts | 59 +++++++ packages/natives/test/npm-packages.test.ts | 105 ++++++++++++ scripts/ci-release-publish.ts | 81 +++++++++- 8 files changed, 419 insertions(+), 12 deletions(-) create mode 100755 packages/natives/scripts/gen-npm-packages.ts create mode 100644 packages/natives/test/npm-packages.test.ts diff --git a/.gitignore b/.gitignore index fd2add32e..044b6ea46 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ pi-*.html # Generated files packages/coding-agent/src/internal-urls/docs-index.generated.ts +packages/natives/npm/ /runs/ python/omp-rpc/src/omp_rpc.egg-info/ # parallel-agent worktrees diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index a79dbc51a..921f1e29e 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Changed + +- Changed npm publishing to ship `@oh-my-pi/pi-natives` as a small core loader package plus per-platform optional dependency leaf packages, so installs fetch only the host platform's native addon instead of every supported `.node` binary. + ## [15.5.10] - 2026-05-28 ### Fixed diff --git a/packages/natives/README.md b/packages/natives/README.md index 0a88f6788..97d8ba1c5 100644 --- a/packages/natives/README.md +++ b/packages/natives/README.md @@ -49,16 +49,32 @@ bun run check ## Architecture +`@oh-my-pi/pi-natives` publishes a small core package plus generated +platform-specific optional dependency packages: + ``` crates/pi-natives/ # Rust source (workspace member) src/lib.rs # N-API exports src/sixel.rs # SIXEL terminal-image encoding Cargo.toml # Rust dependencies -native/ # Native addon binaries - pi_natives.--modern.node # x64 modern ISA (AVX2) - pi_natives.--baseline.node # x64 baseline ISA +native/ # Core loader files and local/CI native build outputs + index.js # Public native export surface + loader-state.js # Platform, ISA variant, and addon resolution + embedded-addon.js # Standalone binary embed stub/generated metadata + pi_natives.--modern.node # x64 modern ISA (local/CI artifact) + pi_natives.--baseline.node # x64 baseline ISA (local/CI artifact) pi_natives.-.node # non-x64 build artifact -src/ # TypeScript wrappers - native.ts # Native addon loader - index.ts # Public API +npm/-/ # Generated at publish time, not committed + package.json # @oh-my-pi/pi-natives-- + *.node # Only that platform's addon binary or x64 ISA variants +src/ # TypeScript wrappers and generated declarations source + native.ts + index.ts ``` + +The published core package contains only the JS loader, declarations, README, +and `package.json`. Release publishing generates one leaf package per supported +`os`/`cpu` pair and injects those leaves into the core manifest as pinned +`optionalDependencies`, so package managers install only the host platform's +native addon. x64 leaves include every built ISA variant, and the loader keeps +choosing between `baseline` and `modern` at runtime. diff --git a/packages/natives/package.json b/packages/natives/package.json index 1e0ad71de..dcdfb8401 100644 --- a/packages/natives/package.json +++ b/packages/natives/package.json @@ -37,6 +37,7 @@ "fix": "biome check --write --unsafe .", "fmt": "biome format --write .", "embed:native": "bun scripts/embed-native.ts", + "gen:npm": "bun scripts/gen-npm-packages.ts", "bench": "bun bench/grep.ts" }, "devDependencies": { diff --git a/packages/natives/scripts/gen-npm-packages.ts b/packages/natives/scripts/gen-npm-packages.ts new file mode 100755 index 000000000..4e9629ffa --- /dev/null +++ b/packages/natives/scripts/gen-npm-packages.ts @@ -0,0 +1,152 @@ +#!/usr/bin/env bun + +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +interface LeafTarget { + tag: string; + os: string; + cpu: string; +} + +export interface BuildLeafManifestInput extends LeafTarget { + files: readonly string[]; + version: string; +} + +export interface LeafManifest { + name: string; + version: string; + os: string[]; + cpu: string[]; + main: string; + files: string[]; + license: string; + repository: { + type: string; + url: string; + directory: string; + }; + engines: { + bun: string; + }; +} + +export interface GeneratedLeafPackage { + tag: string; + dir: string; + files: string[]; + manifest: LeafManifest; + missing: boolean; +} + +export interface GenerateNpmPackagesInput { + packageDir?: string; + dryRun?: boolean; + version?: string; +} + +export const LEAF_TARGETS: readonly LeafTarget[] = [ + { tag: "linux-x64", os: "linux", cpu: "x64" }, + { tag: "linux-arm64", os: "linux", cpu: "arm64" }, + { tag: "darwin-x64", os: "darwin", cpu: "x64" }, + { tag: "darwin-arm64", os: "darwin", cpu: "arm64" }, + { tag: "win32-x64", os: "win32", cpu: "x64" }, +]; + +const packageDirDefault = path.join(import.meta.dir, ".."); + +function expectedAddonFilenames(tag: string): string[] { + return tag.endsWith("-x64") + ? [`pi_natives.${tag}-baseline.node`, `pi_natives.${tag}-modern.node`, `pi_natives.${tag}.node`] + : [`pi_natives.${tag}.node`]; +} + +function discoverAddonFiles(nativeDir: string, tag: string): Promise { + return Promise.all( + expectedAddonFilenames(tag).map(async filename => + (await Bun.file(path.join(nativeDir, filename)).exists()) ? filename : null, + ), + ).then(files => files.filter(file => file !== null)); +} + +function selectPrimaryAddonFile(tag: string, files: readonly string[]): string { + const baseline = `pi_natives.${tag}-baseline.node`; + if (files.includes(baseline)) return baseline; + const defaultFile = `pi_natives.${tag}.node`; + if (files.includes(defaultFile)) return defaultFile; + return files[0]; +} + +export function buildLeafManifest({ tag, os, cpu, files, version }: BuildLeafManifestInput): LeafManifest { + const addonFiles = [...new Set(files.map(file => path.basename(file)))]; + if (addonFiles.length === 0) throw new Error(`No native addon files found for ${tag}`); + for (const file of addonFiles) { + if (!file.endsWith(".node")) throw new Error(`Leaf ${tag} includes non-addon file: ${file}`); + } + const main = selectPrimaryAddonFile(tag, addonFiles); + return { + name: `@oh-my-pi/pi-natives-${tag}`, + version, + os: [os], + cpu: [cpu], + main: `./${main}`, + files: ["*.node", "README.md"], + license: "MIT", + repository: { + type: "git", + url: "git+https://github.com/can1357/oh-my-pi.git", + directory: "packages/natives", + }, + engines: { + bun: ">=1.3.14", + }, + }; +} + +function buildReadme(tag: string, manifest: LeafManifest): string { + return `# ${manifest.name}\n\nPlatform native addon package for \`@oh-my-pi/pi-natives\` on ${tag}.\n\nThis package is generated during release and installed as an optional dependency of the core package.\n`; +} + +export async function generateNpmPackages({ + packageDir = packageDirDefault, + dryRun = false, + version, +}: GenerateNpmPackagesInput = {}): Promise { + const manifestVersion = + version ?? ((await Bun.file(path.join(packageDir, "package.json")).json()) as { version: string }).version; + const nativeDir = path.join(packageDir, "native"); + const npmDir = path.join(packageDir, "npm"); + const leaves: GeneratedLeafPackage[] = []; + + for (const target of LEAF_TARGETS) { + const files = await discoverAddonFiles(nativeDir, target.tag); + const manifestFiles = files.length > 0 ? files : [expectedAddonFilenames(target.tag)[0]]; + const manifest = buildLeafManifest({ ...target, files: manifestFiles, version: manifestVersion }); + const leafDir = path.join(npmDir, target.tag); + const missing = files.length === 0; + leaves.push({ tag: target.tag, dir: leafDir, files, manifest, missing }); + + if (dryRun) { + const fileList = missing ? "missing" : files.join(", "); + console.log(`DRY RUN generate ${manifest.name} (${fileList}) -> ${path.relative(packageDir, leafDir)}`); + console.log(JSON.stringify(manifest, null, "\t")); + continue; + } + + if (missing) throw new Error(`Missing native addon files for ${target.tag} in ${nativeDir}`); + await fs.rm(leafDir, { recursive: true, force: true }); + await fs.mkdir(leafDir, { recursive: true }); + for (const file of files) { + await fs.copyFile(path.join(nativeDir, file), path.join(leafDir, file)); + } + await Bun.write(path.join(leafDir, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`); + await Bun.write(path.join(leafDir, "README.md"), buildReadme(target.tag, manifest)); + } + + return leaves; +} + +if (import.meta.main) { + await generateNpmPackages({ dryRun: process.argv.includes("--dry-run") }); +} diff --git a/packages/natives/test/issue-823-repro.test.ts b/packages/natives/test/issue-823-repro.test.ts index 9b657bf6f..17c4389d4 100644 --- a/packages/natives/test/issue-823-repro.test.ts +++ b/packages/natives/test/issue-823-repro.test.ts @@ -138,6 +138,65 @@ describe("issue 823: standalone-binary native loader path resolution", () => { expect(candidates).not.toContain(path.join(userDataDir, "pi_natives.linux-x64-baseline.node")); }); + it("prefers platform leaf package candidates ahead of core nativeDir candidates on npm installs", () => { + const leafPackageDir = "/app/node_modules/@oh-my-pi/pi-natives-linux-x64"; + const nativeDir = "/app/node_modules/@oh-my-pi/pi-natives/native"; + const candidates = resolveLoaderCandidates({ + addonFilenames: getAddonFilenames({ tag: "linux-x64", arch: "x64", variant: "baseline" }), + isCompiledBinary: false, + leafPackageDir, + nativeDir, + execDir: "/app/node_modules/.bin", + versionedDir: "/home/u/.omp/natives/15.5.15", + userDataDir: "/home/u/.local/bin", + }); + + const leafBaseline = path.join(leafPackageDir, "pi_natives.linux-x64-baseline.node"); + const coreBaseline = path.join(nativeDir, "pi_natives.linux-x64-baseline.node"); + expect(candidates).toContain(leafBaseline); + expect(candidates.indexOf(leafBaseline)).toBeLessThan(candidates.indexOf(coreBaseline)); + }); + + it("keeps Windows staging ahead of leaf package and core nativeDir candidates", () => { + const versionedDir = "/home/u/.omp/natives/15.5.15"; + const leafPackageDir = "/app/node_modules/@oh-my-pi/pi-natives-win32-x64"; + const nativeDir = "/app/node_modules/@oh-my-pi/pi-natives/native"; + const candidates = resolveLoaderCandidates({ + addonFilenames: getAddonFilenames({ tag: "win32-x64", arch: "x64", variant: "baseline" }), + isCompiledBinary: false, + stageFromNodeModules: true, + leafPackageDir, + nativeDir, + execDir: "/app/node_modules/.bin", + versionedDir, + userDataDir: "/home/u/AppData/Local/omp", + }); + + const stagedBaseline = path.join(versionedDir, "pi_natives.win32-x64-baseline.node"); + const leafBaseline = path.join(leafPackageDir, "pi_natives.win32-x64-baseline.node"); + const coreBaseline = path.join(nativeDir, "pi_natives.win32-x64-baseline.node"); + expect(candidates.indexOf(stagedBaseline)).toBeLessThan(candidates.indexOf(leafBaseline)); + expect(candidates.indexOf(leafBaseline)).toBeLessThan(candidates.indexOf(coreBaseline)); + }); + + it("keeps the development candidate list unchanged when no leaf package is installed", () => { + const nativeDir = "/repo/packages/natives/native"; + const execDir = "/usr/bin"; + const addonFilenames = getAddonFilenames({ tag: "linux-x64", arch: "x64", variant: "baseline" }); + const candidates = resolveLoaderCandidates({ + addonFilenames, + isCompiledBinary: false, + nativeDir, + execDir, + versionedDir: "/home/u/.omp/natives/15.5.15", + userDataDir: "/home/u/.local/bin", + }); + + expect(candidates).toEqual( + addonFilenames.flatMap(filename => [path.join(nativeDir, filename), path.join(execDir, filename)]), + ); + }); + it("extracts all bundled native variants from one gzip archive and skips current files", async () => { const testDir = await fs.mkdtemp(path.join(os.tmpdir(), "natives-embedded-archive-")); try { diff --git a/packages/natives/test/npm-packages.test.ts b/packages/natives/test/npm-packages.test.ts new file mode 100644 index 000000000..c3a6229ae --- /dev/null +++ b/packages/natives/test/npm-packages.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it, spyOn } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { buildLeafManifest, generateNpmPackages } from "../scripts/gen-npm-packages"; + +describe("generated native npm leaf packages", () => { + it("builds an x64 leaf manifest that exposes all addon files without package exports", () => { + const addonFiles = ["pi_natives.linux-x64-baseline.node", "pi_natives.linux-x64-modern.node"]; + const manifest = buildLeafManifest({ + tag: "linux-x64", + os: "linux", + cpu: "x64", + files: addonFiles, + version: "15.5.15", + }); + + expect(manifest.name).toBe("@oh-my-pi/pi-natives-linux-x64"); + expect(manifest.version).toBe("15.5.15"); + expect(manifest.os).toEqual(["linux"]); + expect(manifest.cpu).toEqual(["x64"]); + expect(addonFiles).toContain(manifest.main.slice("./".length)); + expect(manifest.files).toContain("*.node"); + expect(manifest.files).toContain("README.md"); + expect("exports" in manifest).toBe(false); + }); + + it("uses the default addon as the main entry for non-x64 leaves", () => { + const addonFiles = ["pi_natives.darwin-arm64.node"]; + const manifest = buildLeafManifest({ + tag: "darwin-arm64", + os: "darwin", + cpu: "arm64", + files: addonFiles, + version: "15.5.15", + }); + + expect(manifest.name).toBe("@oh-my-pi/pi-natives-darwin-arm64"); + expect(manifest.os).toEqual(["darwin"]); + expect(manifest.cpu).toEqual(["arm64"]); + expect(manifest.main).toBe("./pi_natives.darwin-arm64.node"); + expect(addonFiles).toContain(manifest.main.slice("./".length)); + expect("exports" in manifest).toBe(false); + }); + + it("generates every leaf package by copying present addon files", async () => { + const packageDir = await fs.mkdtemp(path.join(os.tmpdir(), "pi-natives-npm-")); + try { + await fs.mkdir(path.join(packageDir, "native")); + await Bun.write(path.join(packageDir, "package.json"), JSON.stringify({ version: "15.5.15" })); + const addonFiles = [ + "pi_natives.linux-x64-baseline.node", + "pi_natives.linux-x64-modern.node", + "pi_natives.linux-arm64.node", + "pi_natives.darwin-x64-baseline.node", + "pi_natives.darwin-arm64.node", + "pi_natives.win32-x64-baseline.node", + ]; + for (const file of addonFiles) { + await Bun.write(path.join(packageDir, "native", file), file); + } + + const leaves = await generateNpmPackages({ packageDir }); + expect(leaves.map(leaf => leaf.tag)).toEqual([ + "linux-x64", + "linux-arm64", + "darwin-x64", + "darwin-arm64", + "win32-x64", + ]); + const linuxX64 = leaves.find(leaf => leaf.tag === "linux-x64"); + expect(linuxX64?.files).toEqual(["pi_natives.linux-x64-baseline.node", "pi_natives.linux-x64-modern.node"]); + expect(await Bun.file(path.join(packageDir, "npm/linux-x64/pi_natives.linux-x64-modern.node")).text()).toBe( + "pi_natives.linux-x64-modern.node", + ); + const manifest = await Bun.file(path.join(packageDir, "npm/linux-x64/package.json")).json(); + expect(manifest.main).toBe("./pi_natives.linux-x64-baseline.node"); + expect("exports" in manifest).toBe(false); + } finally { + await fs.rm(packageDir, { recursive: true, force: true }); + } + }); + + it("reports missing leaves during dry runs without writing generated packages", async () => { + const packageDir = await fs.mkdtemp(path.join(os.tmpdir(), "pi-natives-npm-dry-")); + const logSpy = spyOn(console, "log").mockImplementation(() => {}); + try { + await fs.mkdir(path.join(packageDir, "native")); + await Bun.write(path.join(packageDir, "package.json"), JSON.stringify({ version: "15.5.15" })); + await Bun.write(path.join(packageDir, "native/pi_natives.darwin-arm64.node"), "darwin"); + + const leaves = await generateNpmPackages({ packageDir, dryRun: true }); + expect(leaves.filter(leaf => leaf.missing).map(leaf => leaf.tag)).toEqual([ + "linux-x64", + "linux-arm64", + "darwin-x64", + "win32-x64", + ]); + expect(await Bun.file(path.join(packageDir, "npm/darwin-arm64/package.json")).exists()).toBe(false); + } finally { + logSpy.mockRestore(); + await fs.rm(packageDir, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/ci-release-publish.ts b/scripts/ci-release-publish.ts index 2cca43115..b04920493 100644 --- a/scripts/ci-release-publish.ts +++ b/scripts/ci-release-publish.ts @@ -18,6 +18,11 @@ import * as path from "node:path"; import { $ } from "bun"; +import { + generateNpmPackages, + LEAF_TARGETS, + type GeneratedLeafPackage, +} from "../packages/natives/scripts/gen-npm-packages.ts"; interface PublishPackage { dir: string; @@ -36,7 +41,10 @@ interface JsonObject { } interface PackageManifest extends JsonObject { name?: string; + version?: string; private?: boolean; + files?: JsonValue[]; + optionalDependencies?: JsonObject; } const repoRoot = path.join(import.meta.dir, ".."); @@ -88,7 +96,7 @@ function rewriteExports(exports: JsonValue): JsonValue { return out; } -async function rewriteManifest(pkgDir: string, extraFiles: readonly string[]): Promise { +async function rewriteManifest(pkgDir: string, extraFiles: readonly string[], write: boolean): Promise { const manifestPath = path.join(pkgDir, "package.json"); const manifest = (await Bun.file(manifestPath).json()) as PackageManifest; if (typeof manifest.types === "string" && manifest.types.startsWith("./src/")) { @@ -102,15 +110,12 @@ async function rewriteManifest(pkgDir: string, extraFiles: readonly string[]): P if (!hasDist && !files.includes(extra)) files.push(extra); } manifest.files = files; - await Bun.write(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`); + if (write) await Bun.write(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`); return manifest; } async function preparePackage(pkg: PublishPackage): Promise { const pkgDir = path.join(repoRoot, pkg.dir); - if (pkg.kind === "native") { - return (await Bun.file(path.join(pkgDir, "package.json")).json()) as PackageManifest; - } for (const argv of pkg.preBuild ?? []) { await $`${argv}`.cwd(pkgDir); } @@ -118,10 +123,74 @@ async function preparePackage(pkg: PublishPackage): Promise { for (const cfg of pkg.extraTypeConfigs ?? []) { await $`bun x tsgo -p ${cfg}`.cwd(pkgDir); } - return rewriteManifest(pkgDir, pkg.extraFiles ?? []); + return rewriteManifest(pkgDir, pkg.extraFiles ?? [], !isDryRun); +} + +function buildNativeOptionalDependencies(version: string): JsonObject { + const optionalDependencies: JsonObject = {}; + for (const target of LEAF_TARGETS) { + optionalDependencies[`@oh-my-pi/pi-natives-${target.tag}`] = version; + } + return optionalDependencies; +} + +async function prepareNativeCorePackage(pkgDir: string, write: boolean): Promise { + const manifestPath = path.join(pkgDir, "package.json"); + const manifest = (await Bun.file(manifestPath).json()) as PackageManifest; + if (typeof manifest.version !== "string") throw new Error(`Missing version in ${manifestPath}`); + manifest.optionalDependencies = buildNativeOptionalDependencies(manifest.version); + manifest.files = [ + "native/index.js", + "native/index.d.ts", + "native/loader-state.js", + "native/loader-state.d.ts", + "native/embedded-addon.js", + "README.md", + ]; + if (write) await Bun.write(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`); + return manifest; +} + +async function publishGeneratedLeafPackage(leaf: GeneratedLeafPackage): Promise { + if (isDryRun) { + console.log(`DRY RUN bun publish --access public --tolerate-republish (${path.relative(repoRoot, leaf.dir)})`); + return; + } + console.log(`Publishing ${leaf.manifest.name}…`); + const result = await $`bun publish --access public --tolerate-republish`.cwd(leaf.dir).quiet().nothrow(); + const output = `${result.stdout.toString()}${result.stderr.toString()}`.trim(); + if (output) console.log(output); + if (result.exitCode !== 0) process.exit(result.exitCode ?? 1); +} + +async function publishNativePackage(pkg: PublishPackage): Promise { + const pkgDir = path.join(repoRoot, pkg.dir); + const coreManifest = (await Bun.file(path.join(pkgDir, "package.json")).json()) as PackageManifest; + if (typeof coreManifest.version !== "string") throw new Error(`Missing version in ${pkg.dir}/package.json`); + const leaves = await generateNpmPackages({ packageDir: pkgDir, dryRun: isDryRun, version: coreManifest.version }); + for (const leaf of leaves) { + await publishGeneratedLeafPackage(leaf); + } + const manifest = await prepareNativeCorePackage(pkgDir, !isDryRun); + const name = manifest.name ?? path.basename(pkg.dir); + if (isDryRun) { + console.log(`DRY RUN native core manifest rewrite (${pkg.dir})`); + console.log(JSON.stringify({ optionalDependencies: manifest.optionalDependencies, files: manifest.files }, null, "\t")); + console.log(`DRY RUN bun publish --access public --tolerate-republish (${pkg.dir})`); + return; + } + console.log(`Publishing ${name}…`); + const result = await $`bun publish --access public --tolerate-republish`.cwd(pkgDir).quiet().nothrow(); + const output = `${result.stdout.toString()}${result.stderr.toString()}`.trim(); + if (output) console.log(output); + if (result.exitCode !== 0) process.exit(result.exitCode ?? 1); } async function publishPackage(pkg: PublishPackage): Promise { + if (pkg.kind === "native") { + await publishNativePackage(pkg); + return; + } const pkgDir = path.join(repoRoot, pkg.dir); const manifest = await preparePackage(pkg); const name = manifest.name ?? path.basename(pkg.dir);