4b2c3d524c
`pi_natives` failed to load in Bun worker threads on macOS x64 when the
host built only the `modern` (AVX2) variant. The runtime detector's
`child_process.spawnSync("sysctl", ...)` returned null from the worker
even though the build-time detector (`scripts/host-detect.ts`) succeeded
in the parent shell, so `loadNative()` resolved `variant=baseline` and
searched a file list that excluded the `pi_natives.darwin-x64-modern.node`
the build had actually produced.
Two compounding root causes in `packages/natives/native/loader-state.js`:
- `runCommand` only used `child_process.spawnSync`, which is the path
observed to fail under Bun's worker shim on darwin. The build-time
detector uses `Bun.spawnSync` and works fine.
- The darwin branch looked up `sysctl` via PATH. Login shells supply
`/usr/sbin` so the build picks it up, but worker/embedded spawn
contexts can ship without it.
Fix:
- `runCommand` now prefers `Bun.spawnSync` (matches the build-time
detector) and falls back to `child_process.spawnSync` for non-Bun
embeds.
- The darwin branch tries `/usr/sbin/sysctl` before bare `sysctl`.
- New private env key `__PI_NATIVE_VARIANT_CACHE`: once any context
resolves the variant (the main thread does first), it is written
there. Bun workers and child subprocesses inherit `process.env` at
spawn, so they read the cache and skip detection — sidestepping the
worker-context spawn flakiness end-to-end.
- New exported pure helper `selectCpuVariant({ arch, override, env,
detectAvx2 })` codifies the override > cache > detect order and
returns the cache write hints so the helper itself stays
side-effect-free.
Regression test `packages/natives/test/issue-3238-repro.test.ts` pins
every branch of the resolution order, including the file-list shape
that surfaced the bug.
Fixes #3238
755 lines
27 KiB
JavaScript
755 lines
27 KiB
JavaScript
import * as childProcess from "node:child_process";
|
|
import * as fs from "node:fs";
|
|
import { createRequire } from "node:module";
|
|
import * as os from "node:os";
|
|
import * as path from "node:path";
|
|
import * as zlib from "node:zlib";
|
|
import packageJson from "../package.json" with { type: "json" };
|
|
import { embeddedAddon } from "./embedded-addon.js";
|
|
|
|
/**
|
|
* Native addon loader for `@oh-my-pi/pi-natives`.
|
|
*
|
|
* Owns every step between "Node imports `native/index.js`" and "the right
|
|
* `pi_natives.<platform>-<arch>*.node` is required, validated, and returned":
|
|
* platform/variant detection, candidate-path resolution, on-disk staging from
|
|
* `node_modules` (Windows update safety), embedded-addon extraction (Bun
|
|
* standalone binaries), version-sentinel validation, and the aggregated error
|
|
* surface for diagnostic-friendly failures.
|
|
*
|
|
* `native/index.js` is reduced to one `loadNative()` call plus the generated
|
|
* surface-area exports between `MARKER_START`/`MARKER_END` (rewritten by
|
|
* `scripts/gen-enums.ts`); everything else lives here so the pure helpers stay
|
|
* unit-testable without triggering the side-effectful module-load path.
|
|
*
|
|
* Background (issue #823): `bun build --compile --define PI_COMPILED=true`
|
|
* substitutes the bare identifier `PI_COMPILED`, NOT `process.env.PI_COMPILED`,
|
|
* so a runtime read of the env var returns `undefined`. Older CommonJS loader
|
|
* code also saw the original build-host absolute path in `__filename`; ESM
|
|
* `import.meta.url` is rewritten to the bunfs URL. The embedded-addon
|
|
* presence (true iff the build pipeline ran `embed:native`, false in the
|
|
* post-build `--reset` stub) is the authoritative compiled-mode signal.
|
|
*/
|
|
|
|
const SUPPORTED_PLATFORMS = ["linux-x64", "linux-arm64", "darwin-x64", "darwin-arm64", "win32-x64"];
|
|
|
|
/**
|
|
* Streaming startup marker, enabled by `PI_DEBUG_STARTUP`. Local copy of the
|
|
* pi-utils helper (this loader cannot depend on pi-utils). Synchronous on
|
|
* purpose: extraction/dlopen hangs must still leave the `:start` marker.
|
|
* @param {string} text
|
|
*/
|
|
function startupMarker(text) {
|
|
if (!process.env.PI_DEBUG_STARTUP) return;
|
|
try {
|
|
fs.writeSync(2, `[startup] ${text}\n`);
|
|
} catch {
|
|
// stderr unavailable; markers are best-effort
|
|
}
|
|
}
|
|
|
|
function getNativesDir() {
|
|
const xdgDataHome = process.env.XDG_DATA_HOME;
|
|
if (xdgDataHome && fs.existsSync(path.join(xdgDataHome, "omp"))) {
|
|
return path.join(xdgDataHome, "omp", "natives");
|
|
}
|
|
return path.join(os.homedir(), ".omp", "natives");
|
|
}
|
|
|
|
function resolveLeafPackageDir(platformTag) {
|
|
try {
|
|
const require_ = createRequire(import.meta.url);
|
|
return path.dirname(require_.resolve(`@oh-my-pi/pi-natives-${platformTag}/package.json`));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// =========================================================================
|
|
// Pure helpers — re-exported for unit tests in `packages/natives/test/`.
|
|
// =========================================================================
|
|
|
|
/**
|
|
* @param {{
|
|
* embeddedAddon: { platformTag: string; version: string; files: unknown[] } | null | undefined;
|
|
* env: Record<string, string | undefined>;
|
|
* importMetaUrl: string | null | undefined;
|
|
* }} input
|
|
* @returns {boolean}
|
|
*/
|
|
export function detectCompiledBinary({ embeddedAddon, env, importMetaUrl }) {
|
|
if (embeddedAddon) return true;
|
|
if (env && env.PI_COMPILED) return true;
|
|
if (typeof importMetaUrl === "string") {
|
|
if (importMetaUrl.includes("$bunfs")) return true;
|
|
if (importMetaUrl.includes("~BUN")) return true;
|
|
if (importMetaUrl.includes("%7EBUN")) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* @param {{ tag: string; arch: string; variant: "modern" | "baseline" | null | undefined }} input
|
|
* @returns {string[]}
|
|
*/
|
|
export function getAddonFilenames({ tag, arch, variant }) {
|
|
const defaultFilename = `pi_natives.${tag}.node`;
|
|
if (arch !== "x64" || !variant) return [defaultFilename];
|
|
const baselineFilename = `pi_natives.${tag}-baseline.node`;
|
|
const modernFilename = `pi_natives.${tag}-modern.node`;
|
|
if (variant === "modern") {
|
|
return [modernFilename, baselineFilename, defaultFilename];
|
|
}
|
|
return [baselineFilename, defaultFilename];
|
|
}
|
|
|
|
/**
|
|
* Decide whether the loader should mirror the package's `native/<filename>.node`
|
|
* into the per-version cache directory (`~/.omp/natives/<version>/`) before loading.
|
|
*
|
|
* Windows-only safety net for `bun install -g` updates: when a previous `omp`
|
|
* process is running, bun cannot overwrite the locked `.node` inside
|
|
* `node_modules/@oh-my-pi/pi-natives/native/`, leaving an old binary next to a
|
|
* newer `index.js` and producing `<sym> is not a function` crashes on the next
|
|
* launch. Staging into the version-pinned cache:
|
|
* 1. Gives every package version its own filesystem path, so concurrent omp
|
|
* processes never collide on the same file.
|
|
* 2. Makes the running process keep its handle on the cache copy, freeing bun
|
|
* to overwrite the `node_modules` copy on subsequent updates.
|
|
* Disabled on non-Windows (no file-lock problem), in workspace dev (`nativeDir`
|
|
* is not inside a `node_modules` segment), and for compiled binaries (handled
|
|
* by `maybeExtractEmbeddedAddon`).
|
|
*
|
|
* @param {{ platform: NodeJS.Platform | string; isCompiledBinary: boolean; nativeDir: string }} input
|
|
* @returns {boolean}
|
|
*/
|
|
export function shouldStageNodeModulesAddon({ platform, isCompiledBinary, nativeDir }) {
|
|
if (platform !== "win32") return false;
|
|
if (isCompiledBinary) return false;
|
|
// Check both separators independently of the host's `path.sep`: this helper
|
|
// is shared by the loader (running on Windows with `\`) and the test suite
|
|
// (typically running on POSIX hosts when CI executes the regression test).
|
|
return nativeDir.includes("\\node_modules\\") || nativeDir.includes("/node_modules/");
|
|
}
|
|
|
|
/**
|
|
* @param {{
|
|
* addonFilenames: string[];
|
|
* isCompiledBinary: boolean;
|
|
* stageFromNodeModules?: boolean;
|
|
* nativeDir: string;
|
|
* leafPackageDir?: string | null;
|
|
* execDir: string;
|
|
* versionedDir: string;
|
|
* userDataDir: string;
|
|
* }} input
|
|
* @returns {string[]}
|
|
*/
|
|
export function resolveLoaderCandidates({
|
|
addonFilenames,
|
|
isCompiledBinary,
|
|
stageFromNodeModules = false,
|
|
nativeDir,
|
|
leafPackageDir = null,
|
|
execDir,
|
|
versionedDir,
|
|
userDataDir,
|
|
}) {
|
|
const baseReleaseCandidates = addonFilenames.flatMap(filename => [
|
|
path.join(nativeDir, filename),
|
|
path.join(execDir, filename),
|
|
]);
|
|
const leafCandidates = leafPackageDir ? addonFilenames.map(filename => path.join(leafPackageDir, filename)) : [];
|
|
const compiledCandidates = addonFilenames.flatMap(filename => [
|
|
path.join(versionedDir, filename),
|
|
path.join(userDataDir, filename),
|
|
]);
|
|
const stagedCandidates = stageFromNodeModules ? addonFilenames.map(filename => path.join(versionedDir, filename)) : [];
|
|
let releaseCandidates;
|
|
if (isCompiledBinary) {
|
|
releaseCandidates = [...compiledCandidates, ...baseReleaseCandidates];
|
|
} else if (stageFromNodeModules) {
|
|
releaseCandidates = [...stagedCandidates, ...leafCandidates, ...baseReleaseCandidates];
|
|
} else {
|
|
releaseCandidates = [...leafCandidates, ...baseReleaseCandidates];
|
|
}
|
|
return [...new Set(releaseCandidates)];
|
|
}
|
|
|
|
// =========================================================================
|
|
|
|
/**
|
|
* Remove version-pinned native cache directories older than the loaded package.
|
|
* Best-effort by design: permission errors and concurrent processes must not
|
|
* abort startup after the native addon has already loaded successfully.
|
|
*
|
|
* @param {{ nativesDir: string; currentVersion: string }} input
|
|
* @returns {string[]}
|
|
*/
|
|
export function cleanupStaleNativeVersions({ nativesDir, currentVersion }) {
|
|
const removed = [];
|
|
let entries;
|
|
try {
|
|
entries = fs.readdirSync(nativesDir, { withFileTypes: true });
|
|
} catch {
|
|
return removed;
|
|
}
|
|
|
|
for (const entry of entries) {
|
|
if (!entry.isDirectory() || entry.name === currentVersion) continue;
|
|
const targetPath = path.join(nativesDir, entry.name);
|
|
try {
|
|
fs.rmSync(targetPath, { recursive: true, force: true });
|
|
removed.push(targetPath);
|
|
} catch {
|
|
// Stale caches are opportunistic cleanup only.
|
|
}
|
|
}
|
|
return removed;
|
|
}
|
|
|
|
// Side-effectful loader. Everything below runs only when `loadNative()` is
|
|
// called from `native/index.js` — tests that only import the pure helpers
|
|
// above pay nothing for variant detection, subprocess spawns, or fs probes.
|
|
// =========================================================================
|
|
|
|
/**
|
|
* Hidden env key for the resolved x64 variant. Once any context (main thread,
|
|
* worker, subprocess) finishes variant detection, the result is written here
|
|
* so every Bun worker and child process spawned afterwards inherits the same
|
|
* verdict and skips re-detection. See `selectCpuVariant` for the lookup order.
|
|
*/
|
|
const VARIANT_CACHE_ENV_KEY = "__PI_NATIVE_VARIANT_CACHE";
|
|
|
|
/**
|
|
* Spawn `command` with `args` and capture stdout. Prefers `Bun.spawnSync`
|
|
* because Bun's `child_process.spawnSync` shim has been observed to return
|
|
* non-zero / null in worker threads on macOS even when the same binary works
|
|
* fine from the parent — the failure mode behind issue #3238, where the worker
|
|
* silently falls back to the "baseline" variant. Falls back to the Node shim
|
|
* for non-Bun embeds.
|
|
*/
|
|
function runCommand(command, args) {
|
|
if (typeof Bun !== "undefined" && typeof Bun.spawnSync === "function") {
|
|
try {
|
|
const result = Bun.spawnSync([command, ...args], { stdout: "pipe", stderr: "pipe" });
|
|
if (result.exitCode === 0) {
|
|
return result.stdout.toString("utf-8").trim();
|
|
}
|
|
} catch {
|
|
// fall through to childProcess
|
|
}
|
|
}
|
|
try {
|
|
const result = childProcess.spawnSync(command, args, { encoding: "utf-8" });
|
|
if (result.error) return null;
|
|
if (result.status !== 0) return null;
|
|
return (result.stdout || "").trim();
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function getVariantOverride() {
|
|
const value = process.env.PI_NATIVE_VARIANT;
|
|
if (!value) return null;
|
|
if (value === "modern" || value === "baseline") return value;
|
|
return null;
|
|
}
|
|
|
|
function detectAvx2Support() {
|
|
if (process.arch !== "x64") {
|
|
return false;
|
|
}
|
|
|
|
if (process.platform === "linux") {
|
|
try {
|
|
const cpuInfo = fs.readFileSync("/proc/cpuinfo", "utf8");
|
|
return /\bavx2\b/i.test(cpuInfo);
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
if (process.platform === "darwin") {
|
|
// Try the absolute path before bare `sysctl`: PATH may not include
|
|
// `/usr/sbin` in worker/embedded spawn contexts (issue #3238).
|
|
for (const sysctlBin of ["/usr/sbin/sysctl", "sysctl"]) {
|
|
const leaf7 = runCommand(sysctlBin, ["-n", "machdep.cpu.leaf7_features"]);
|
|
if (leaf7 && /\bAVX2\b/i.test(leaf7)) return true;
|
|
const features = runCommand(sysctlBin, ["-n", "machdep.cpu.features"]);
|
|
if (features && /\bAVX2\b/i.test(features)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
if (process.platform === "win32") {
|
|
const output = runCommand("powershell.exe", [
|
|
"-NoProfile",
|
|
"-NonInteractive",
|
|
"-Command",
|
|
"[System.Runtime.Intrinsics.X86.Avx2]::IsSupported",
|
|
]);
|
|
return output && output.toLowerCase() === "true";
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Pure variant-selection helper, exposed for unit tests. Resolution order:
|
|
*
|
|
* 1. `override` (user-facing `PI_NATIVE_VARIANT` env var). Always wins.
|
|
* 2. The private `__PI_NATIVE_VARIANT_CACHE` env var, populated by the first
|
|
* context that detected at runtime. Lets child workers / subprocesses
|
|
* inherit the main thread's verdict instead of re-spawning `sysctl` etc.
|
|
* from a worker context where the spawn may fail (issue #3238).
|
|
* 3. `detectAvx2()` — the slow path, called at most once per process.
|
|
*
|
|
* Non-x64 architectures return `{ variant: null }` and never set the cache.
|
|
* When detection runs, the result is surfaced as `cacheEnvKey`/`cacheEnvValue`
|
|
* so the caller can write `process.env` (the pure helper itself stays
|
|
* side-effect-free, which keeps it easy to test).
|
|
*
|
|
* @param {{
|
|
* arch: string;
|
|
* override: "modern" | "baseline" | null | undefined;
|
|
* env: Record<string, string | undefined>;
|
|
* detectAvx2: () => boolean;
|
|
* }} input
|
|
* @returns {{
|
|
* variant: "modern" | "baseline" | null;
|
|
* source: "non-x64" | "override" | "cache" | "detect";
|
|
* cacheEnvKey?: string;
|
|
* cacheEnvValue?: string;
|
|
* }}
|
|
*/
|
|
export function selectCpuVariant({ arch, override, env, detectAvx2 }) {
|
|
if (arch !== "x64") return { variant: null, source: "non-x64" };
|
|
if (override === "modern" || override === "baseline") {
|
|
return { variant: override, source: "override" };
|
|
}
|
|
const cached = env[VARIANT_CACHE_ENV_KEY];
|
|
if (cached === "modern" || cached === "baseline") {
|
|
return { variant: cached, source: "cache" };
|
|
}
|
|
const variant = detectAvx2() ? "modern" : "baseline";
|
|
return {
|
|
variant,
|
|
source: "detect",
|
|
cacheEnvKey: VARIANT_CACHE_ENV_KEY,
|
|
cacheEnvValue: variant,
|
|
};
|
|
}
|
|
|
|
function resolveCpuVariant(override) {
|
|
const result = selectCpuVariant({
|
|
arch: process.arch,
|
|
override,
|
|
env: process.env,
|
|
detectAvx2: detectAvx2Support,
|
|
});
|
|
if (result.cacheEnvKey) {
|
|
process.env[result.cacheEnvKey] = result.cacheEnvValue;
|
|
}
|
|
return result.variant;
|
|
}
|
|
|
|
function selectEmbeddedAddonFile(selectedVariant) {
|
|
if (!embeddedAddon) return null;
|
|
const defaultFile = embeddedAddon.files.find(file => file.variant === "default") || null;
|
|
if (process.arch !== "x64") return defaultFile || embeddedAddon.files[0] || null;
|
|
if (selectedVariant === "modern") {
|
|
return (
|
|
embeddedAddon.files.find(file => file.variant === "modern") ||
|
|
embeddedAddon.files.find(file => file.variant === "baseline") ||
|
|
null
|
|
);
|
|
}
|
|
return embeddedAddon.files.find(file => file.variant === "baseline") || null;
|
|
}
|
|
|
|
function readTarString(buffer, offset, length) {
|
|
const end = Math.min(offset + length, buffer.length);
|
|
let stringEnd = offset;
|
|
while (stringEnd < end && buffer[stringEnd] !== 0) stringEnd++;
|
|
return buffer.toString("utf8", offset, stringEnd);
|
|
}
|
|
|
|
function readTarOctal(buffer, offset, length) {
|
|
const value = readTarString(buffer, offset, length).trim();
|
|
if (!value) return 0;
|
|
const parsed = Number.parseInt(value, 8);
|
|
if (!Number.isFinite(parsed)) {
|
|
throw new Error(`Invalid tar octal value: ${value}`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function isZeroTarBlock(buffer, offset) {
|
|
for (let index = 0; index < 512; index++) {
|
|
if (buffer[offset + index] !== 0) return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
function getTarEntryName(header) {
|
|
const name = readTarString(header, 0, 100);
|
|
const prefix = readTarString(header, 345, 155);
|
|
return prefix ? `${prefix}/${name}` : name;
|
|
}
|
|
|
|
function isSafeEmbeddedAddonFilename(filename) {
|
|
return filename.length > 0 && path.basename(filename) === filename && !filename.includes("/") && !filename.includes("\\");
|
|
}
|
|
|
|
function isEmbeddedAddonFileCurrent(targetPath, file) {
|
|
try {
|
|
const stat = fs.statSync(targetPath);
|
|
if (!stat.isFile()) return false;
|
|
return typeof file.size !== "number" || stat.size === file.size;
|
|
} catch (err) {
|
|
if (err && err.code === "ENOENT") return false;
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
function writeEmbeddedAddonFile(targetPath, content) {
|
|
const tempPath = `${targetPath}.tmp.${process.pid}.${Date.now()}`;
|
|
try {
|
|
fs.writeFileSync(tempPath, content, { mode: 0o755 });
|
|
fs.renameSync(tempPath, targetPath);
|
|
} catch (err) {
|
|
try {
|
|
fs.unlinkSync(tempPath);
|
|
} catch {
|
|
// Best-effort cleanup only.
|
|
}
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
export function extractEmbeddedAddonArchive({ archivePath, files, targetDir }) {
|
|
const pending = new Map();
|
|
for (const file of files) {
|
|
if (!isSafeEmbeddedAddonFilename(file.filename)) {
|
|
throw new Error(`Unsafe embedded addon filename: ${file.filename}`);
|
|
}
|
|
const targetPath = path.join(targetDir, file.filename);
|
|
if (!isEmbeddedAddonFileCurrent(targetPath, file)) {
|
|
pending.set(file.filename, file);
|
|
}
|
|
}
|
|
if (pending.size === 0) return [];
|
|
|
|
const archive = zlib.gunzipSync(fs.readFileSync(archivePath));
|
|
const writtenPaths = [];
|
|
let offset = 0;
|
|
|
|
while (offset + 512 <= archive.length) {
|
|
if (isZeroTarBlock(archive, offset)) break;
|
|
const header = archive.subarray(offset, offset + 512);
|
|
const filename = getTarEntryName(header);
|
|
const size = readTarOctal(header, 124, 12);
|
|
const typeflag = header[156] === 0 ? "0" : String.fromCharCode(header[156]);
|
|
offset += 512;
|
|
|
|
if (offset + size > archive.length) {
|
|
throw new Error(`Truncated embedded addon archive entry: ${filename}`);
|
|
}
|
|
|
|
if (!isSafeEmbeddedAddonFilename(filename)) {
|
|
throw new Error(`Unsafe embedded addon archive entry: ${filename}`);
|
|
}
|
|
if (typeflag !== "0") {
|
|
throw new Error(`Unsupported embedded addon archive entry type ${typeflag}: ${filename}`);
|
|
}
|
|
|
|
const file = pending.get(filename);
|
|
if (file) {
|
|
if (typeof file.size === "number" && file.size !== size) {
|
|
throw new Error(`Embedded addon size mismatch for ${filename}: expected ${file.size}, got ${size}`);
|
|
}
|
|
const targetPath = path.join(targetDir, filename);
|
|
writeEmbeddedAddonFile(targetPath, archive.subarray(offset, offset + size));
|
|
pending.delete(filename);
|
|
writtenPaths.push(targetPath);
|
|
}
|
|
|
|
offset += Math.ceil(size / 512) * 512;
|
|
}
|
|
|
|
if (pending.size > 0) {
|
|
throw new Error(`Embedded addon archive missing: ${[...pending.keys()].join(", ")}`);
|
|
}
|
|
|
|
return writtenPaths;
|
|
}
|
|
|
|
function maybeExtractEmbeddedAddon(ctx, errors) {
|
|
if (!ctx.isCompiledBinary || !embeddedAddon) return null;
|
|
if (embeddedAddon.platformTag !== ctx.platformTag || embeddedAddon.version !== ctx.packageVersion) return null;
|
|
|
|
const selectedEmbeddedFile = selectEmbeddedAddonFile(ctx.selectedVariant);
|
|
if (!selectedEmbeddedFile) return null;
|
|
const targetPath = path.join(ctx.versionedDir, selectedEmbeddedFile.filename);
|
|
|
|
startupMarker("native:extractEmbeddedAddon:start");
|
|
try {
|
|
fs.mkdirSync(ctx.versionedDir, { recursive: true });
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
errors.push(`embedded addon dir: ${message}`);
|
|
return null;
|
|
}
|
|
|
|
if (embeddedAddon.archive) {
|
|
try {
|
|
extractEmbeddedAddonArchive({
|
|
archivePath: embeddedAddon.archive.filePath,
|
|
files: embeddedAddon.files,
|
|
targetDir: ctx.versionedDir,
|
|
});
|
|
if (isEmbeddedAddonFileCurrent(targetPath, selectedEmbeddedFile)) {
|
|
return targetPath;
|
|
}
|
|
errors.push(`embedded addon archive (${embeddedAddon.archive.filename}): missing ${selectedEmbeddedFile.filename}`);
|
|
return null;
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
errors.push(`embedded addon archive (${embeddedAddon.archive.filename}): ${message}`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
if (isEmbeddedAddonFileCurrent(targetPath, selectedEmbeddedFile)) {
|
|
return targetPath;
|
|
}
|
|
if (!selectedEmbeddedFile.filePath) {
|
|
errors.push(`embedded addon metadata missing file path for ${selectedEmbeddedFile.filename}`);
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const buffer = fs.readFileSync(selectedEmbeddedFile.filePath);
|
|
fs.writeFileSync(targetPath, buffer);
|
|
return targetPath;
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
errors.push(`embedded addon write (${selectedEmbeddedFile.filename}): ${message}`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Mirror `leafPackageDir ?? nativeDir` addon binaries to
|
|
* `versionedDir/<filename>.node` on Windows installs so the running process
|
|
* cache path, never on the `node_modules` copy that bun must overwrite on
|
|
* update. No-op on non-Windows, in workspace dev, and for compiled binaries —
|
|
* see `shouldStageNodeModulesAddon` for the gating rules.
|
|
*/
|
|
function maybeStageNodeModulesAddon(ctx, errors) {
|
|
if (!ctx.stageFromNodeModules) return null;
|
|
|
|
let stagedPath = null;
|
|
for (const filename of ctx.addonFilenames) {
|
|
const sourcePath = path.join(ctx.leafPackageDir ?? ctx.nativeDir, filename);
|
|
const targetPath = path.join(ctx.versionedDir, filename);
|
|
|
|
if (fs.existsSync(targetPath)) {
|
|
stagedPath = stagedPath || targetPath;
|
|
continue;
|
|
}
|
|
if (!fs.existsSync(sourcePath)) continue;
|
|
|
|
try {
|
|
fs.mkdirSync(ctx.versionedDir, { recursive: true });
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
errors.push(`staged addon dir: ${message}`);
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
// `copyFileSync` is atomic on Windows (CopyFileW) and avoids holding
|
|
// two large buffers in JS for the read/write dance.
|
|
fs.copyFileSync(sourcePath, targetPath);
|
|
stagedPath = stagedPath || targetPath;
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
errors.push(`staged addon copy (${filename}): ${message}`);
|
|
}
|
|
}
|
|
return stagedPath;
|
|
}
|
|
|
|
function validateLoadedBindings(ctx, bindings, candidate) {
|
|
// In workspace dev (running out of `packages/natives/native/` rather than a
|
|
// `node_modules` install or a compiled bundle) the local `.node` only gains
|
|
// the renamed sentinel after `bun --cwd=packages/natives run build`. Skip
|
|
// validation there so a stale post-pull dev tree boots while the rebuild
|
|
// completes; install and compiled-binary paths still validate.
|
|
if (ctx.isWorkspaceLoad) return;
|
|
if (typeof bindings[ctx.versionSentinelExport] === "function") return;
|
|
throw new Error(
|
|
`Loaded ${candidate} but it does not expose the @oh-my-pi/pi-natives@${ctx.packageVersion} ` +
|
|
`version sentinel \`${ctx.versionSentinelExport}\`. The .node file on disk is from a different ` +
|
|
"release than this loader — reinstall to re-sync.",
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Install the addon's bounded Tokio runtime now that `dlopen` has returned and
|
|
* the dynamic-loader lock is released. The Rust `#[module_init]` deliberately
|
|
* does NOT build the runtime — spawning worker threads under the loader lock
|
|
* deadlocks on some hosts — so it exposes `__ompInstallTokioRuntime` for the
|
|
* loader to call once, before any async native runs. Best-effort: older addons
|
|
* predating this export simply fall back to napi-rs's default runtime.
|
|
*/
|
|
function installNativeTokioRuntime(bindings) {
|
|
const install = bindings.__ompInstallTokioRuntime;
|
|
if (typeof install !== "function") return;
|
|
try {
|
|
install();
|
|
startupMarker("native:tokioRuntime:installed");
|
|
} catch (err) {
|
|
startupMarker(`native:tokioRuntime:failed:${err instanceof Error ? err.message : String(err)}`);
|
|
}
|
|
}
|
|
|
|
|
|
function buildHelpMessage(ctx) {
|
|
if (ctx.isCompiledBinary) {
|
|
const expectedPaths = ctx.addonFilenames.map(filename => ` ${path.join(ctx.versionedDir, filename)}`).join("\n");
|
|
const downloadHints = ctx.addonFilenames
|
|
.map(filename => {
|
|
const downloadUrl = `https://github.com/can1357/oh-my-pi/releases/latest/download/${filename}`;
|
|
const targetPath = path.join(ctx.versionedDir, filename);
|
|
return ` curl -fsSL "${downloadUrl}" -o "${targetPath}"`;
|
|
})
|
|
.join("\n");
|
|
return (
|
|
`The compiled binary should extract one of:\n${expectedPaths}\n\n` +
|
|
`If missing, delete ${ctx.versionedDir} and re-run, or download manually:\n${downloadHints}`
|
|
);
|
|
}
|
|
return (
|
|
"If installed via npm/bun, try reinstalling: bun install @oh-my-pi/pi-natives\n" +
|
|
"If developing locally, build with: bun --cwd=packages/natives run build\n" +
|
|
"Optional x64 variants: TARGET_VARIANT=baseline|modern bun --cwd=packages/natives run build"
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Initialize the loader context: resolves every path, variant, and policy
|
|
* decision once so the inner load loop stays a pure require/validate pipeline.
|
|
* Called from `loadNative()` rather than at module scope so importing pure
|
|
* helpers from this file doesn't trigger AVX2 detection or filesystem probes.
|
|
*/
|
|
function initLoaderContext() {
|
|
const platformTag = `${process.platform}-${process.arch}`;
|
|
const packageVersion = packageJson.version;
|
|
const nativeDir = path.join(import.meta.dir, "..", "native");
|
|
const execDir = path.dirname(process.execPath);
|
|
const nativesDir = getNativesDir();
|
|
const versionedDir = path.join(nativesDir, packageVersion);
|
|
const userDataDir =
|
|
process.platform === "win32"
|
|
? path.join(process.env.LOCALAPPDATA || path.join(os.homedir(), "AppData", "Local"), "omp")
|
|
: path.join(os.homedir(), ".local", "bin");
|
|
|
|
const isCompiledBinary = detectCompiledBinary({
|
|
embeddedAddon,
|
|
env: process.env,
|
|
importMetaUrl: import.meta.url,
|
|
});
|
|
const leafPackageDir = isCompiledBinary ? null : resolveLeafPackageDir(platformTag);
|
|
const stageFromNodeModules = shouldStageNodeModulesAddon({
|
|
platform: process.platform,
|
|
isCompiledBinary,
|
|
nativeDir,
|
|
});
|
|
|
|
const selectedVariant = resolveCpuVariant(getVariantOverride());
|
|
const addonFilenames = getAddonFilenames({ tag: platformTag, arch: process.arch, variant: selectedVariant });
|
|
const addonLabel = selectedVariant ? `${platformTag} (${selectedVariant})` : platformTag;
|
|
|
|
const candidates = resolveLoaderCandidates({
|
|
addonFilenames,
|
|
isCompiledBinary,
|
|
stageFromNodeModules,
|
|
nativeDir,
|
|
leafPackageDir,
|
|
execDir,
|
|
versionedDir,
|
|
userDataDir,
|
|
});
|
|
|
|
// Version sentinel emitted by the Rust addon under a `js_name` that encodes
|
|
// the package version (`__piNativesV{major}_{minor}_{patch}`).
|
|
// `scripts/release.ts` bumps the name in `crates/pi-natives/src/lib.rs` in
|
|
// lock-step with the version, so a `.node` from a different release
|
|
// physically cannot expose the symbol this loader is looking for. That
|
|
// turns the silent `<sym> is not a function` crash from a Windows
|
|
// locked-file update into an actionable load-time error.
|
|
const versionSentinelExport = `__piNativesV${packageVersion.replace(/[^A-Za-z0-9]/g, "_")}`;
|
|
const isWorkspaceLoad =
|
|
!isCompiledBinary && !nativeDir.includes("\\node_modules\\") && !nativeDir.includes("/node_modules/");
|
|
|
|
return {
|
|
platformTag,
|
|
packageVersion,
|
|
nativeDir,
|
|
leafPackageDir,
|
|
versionedDir,
|
|
isCompiledBinary,
|
|
stageFromNodeModules,
|
|
selectedVariant,
|
|
addonFilenames,
|
|
addonLabel,
|
|
candidates,
|
|
versionSentinelExport,
|
|
isWorkspaceLoad,
|
|
nativesDir,
|
|
};
|
|
}
|
|
|
|
export function loadNative() {
|
|
startupMarker("native:loadNative:start");
|
|
const ctx = initLoaderContext();
|
|
const require_ = createRequire(import.meta.url);
|
|
|
|
const errors = [];
|
|
const embeddedCandidate = maybeExtractEmbeddedAddon(ctx, errors);
|
|
const stagedCandidate = embeddedCandidate ? null : maybeStageNodeModulesAddon(ctx, errors);
|
|
const prepended = [embeddedCandidate, stagedCandidate].filter(c => typeof c === "string");
|
|
const runtimeCandidates = prepended.length > 0 ? [...prepended, ...ctx.candidates] : ctx.candidates;
|
|
|
|
for (const candidate of runtimeCandidates) {
|
|
try {
|
|
startupMarker(`native:require:${path.basename(candidate)}`);
|
|
const bindings = require_(candidate);
|
|
validateLoadedBindings(ctx, bindings, candidate);
|
|
installNativeTokioRuntime(bindings);
|
|
cleanupStaleNativeVersions({ nativesDir: ctx.nativesDir, currentVersion: ctx.packageVersion });
|
|
startupMarker("native:loadNative:done");
|
|
return bindings;
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
errors.push(`${candidate}: ${message}`);
|
|
}
|
|
}
|
|
|
|
if (!SUPPORTED_PLATFORMS.includes(ctx.platformTag)) {
|
|
throw new Error(
|
|
`Unsupported platform: ${ctx.platformTag}\n` +
|
|
`Supported platforms: ${SUPPORTED_PLATFORMS.join(", ")}\n` +
|
|
"If you need support for this platform, please open an issue.",
|
|
);
|
|
}
|
|
const details = errors.map(error => `- ${error}`).join("\n");
|
|
throw new Error(
|
|
`Failed to load pi_natives native addon for ${ctx.addonLabel}.\n\nTried:\n${details}\n\n${buildHelpMessage(ctx)}`,
|
|
);
|
|
}
|