diff --git a/Cargo.toml b/Cargo.toml index 4ff4ae66b..0bedaedfa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,8 +26,8 @@ panic = "abort" inherits = "release" lto = "thin" codegen-units = 16 -debug = "line-tables-only" -strip = "none" +debug = false +strip = "symbols" split-debuginfo = "off" [profile.local] diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index ba4053eb4..c61191281 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -1,6 +1,18 @@ # Changelog ## [Unreleased] +### Changed + +- Changed native addon extraction to skip re-extracting cached `.node` files when their size already matches embedded archive metadata +- Changed standalone binaries to embed native addons as a compressed tarball and unpack them into the versioned native cache on first run instead of embedding each `.node` file uncompressed. + +### Fixed + +- Fixed CI native addon builds retaining ELF debug and symbol sections in release artifacts; stripped builds are now verified to reject `.debug_*`, `.zdebug_*`, `.symtab`, and `.strtab` sections. + +### Security + +- Hardened embedded addon archive extraction by rejecting unsafe entry names and non-file archive entries before writing binaries to disk ## [15.5.4] - 2026-05-27 ### Added diff --git a/packages/natives/native/embedded-addon.js b/packages/natives/native/embedded-addon.js index 66a045fd4..78717dc85 100644 --- a/packages/natives/native/embedded-addon.js +++ b/packages/natives/native/embedded-addon.js @@ -8,6 +8,14 @@ * @typedef {Object} EmbeddedAddonFile * @property {EmbeddedAddonVariant} variant * @property {string} filename + * @property {number} size + * @property {string=} filePath + */ + +/** + * @typedef {Object} EmbeddedAddonArchive + * @property {"tar.gz"} format + * @property {string} filename * @property {string} filePath */ @@ -16,6 +24,7 @@ * @property {string} platformTag * @property {string} version * @property {EmbeddedAddonFile[]} files + * @property {EmbeddedAddonArchive=} archive */ /** @type {EmbeddedAddon|null} */ diff --git a/packages/natives/native/loader-state.d.ts b/packages/natives/native/loader-state.d.ts index 819bcc6b2..eb41d5daa 100644 --- a/packages/natives/native/loader-state.d.ts +++ b/packages/natives/native/loader-state.d.ts @@ -1,6 +1,13 @@ export interface EmbeddedAddonFile { variant: "modern" | "baseline" | "default"; filename: string; + size?: number; + filePath?: string; +} + +export interface EmbeddedAddonArchive { + format: "tar.gz"; + filename: string; filePath: string; } @@ -8,6 +15,7 @@ export interface EmbeddedAddon { platformTag: string; version: string; files: EmbeddedAddonFile[]; + archive?: EmbeddedAddonArchive; } export interface DetectCompiledBinaryInput { @@ -46,4 +54,11 @@ export interface ResolveLoaderCandidatesInput { export function resolveLoaderCandidates(input: ResolveLoaderCandidatesInput): string[]; +export interface ExtractEmbeddedAddonArchiveInput { + archivePath: string; + files: EmbeddedAddonFile[]; + targetDir: string; +} + +export function extractEmbeddedAddonArchive(input: ExtractEmbeddedAddonArchiveInput): string[]; export function loadNative(): Record; diff --git a/packages/natives/native/loader-state.js b/packages/natives/native/loader-state.js index 0fb3a07dd..35fff4e62 100644 --- a/packages/natives/native/loader-state.js +++ b/packages/natives/native/loader-state.js @@ -3,6 +3,7 @@ import * as fs from "node:fs"; import { createRequire } from "node:module"; import * as os from "node:os"; import * as path from "node:path"; +import * as zlib from "node:zlib"; import packageJson from "../package.json" with { type: "json" }; import { embeddedAddon } from "./embedded-addon.js"; @@ -228,6 +229,123 @@ function selectEmbeddedAddonFile(selectedVariant) { return embeddedAddon.files.find(file => file.variant === "baseline") || null; } +function readTarString(buffer, offset, length) { + const end = Math.min(offset + length, buffer.length); + let stringEnd = offset; + while (stringEnd < end && buffer[stringEnd] !== 0) stringEnd++; + return buffer.toString("utf8", offset, stringEnd); +} + +function readTarOctal(buffer, offset, length) { + const value = readTarString(buffer, offset, length).trim(); + if (!value) return 0; + const parsed = Number.parseInt(value, 8); + if (!Number.isFinite(parsed)) { + throw new Error(`Invalid tar octal value: ${value}`); + } + return parsed; +} + +function isZeroTarBlock(buffer, offset) { + for (let index = 0; index < 512; index++) { + if (buffer[offset + index] !== 0) return false; + } + return true; +} + +function getTarEntryName(header) { + const name = readTarString(header, 0, 100); + const prefix = readTarString(header, 345, 155); + return prefix ? `${prefix}/${name}` : name; +} + +function isSafeEmbeddedAddonFilename(filename) { + return filename.length > 0 && path.basename(filename) === filename && !filename.includes("/") && !filename.includes("\\"); +} + +function isEmbeddedAddonFileCurrent(targetPath, file) { + try { + const stat = fs.statSync(targetPath); + if (!stat.isFile()) return false; + return typeof file.size !== "number" || stat.size === file.size; + } catch (err) { + if (err && err.code === "ENOENT") return false; + throw err; + } +} + +function writeEmbeddedAddonFile(targetPath, content) { + const tempPath = `${targetPath}.tmp.${process.pid}.${Date.now()}`; + try { + fs.writeFileSync(tempPath, content, { mode: 0o755 }); + fs.renameSync(tempPath, targetPath); + } catch (err) { + try { + fs.unlinkSync(tempPath); + } catch { + // Best-effort cleanup only. + } + throw err; + } +} + +export function extractEmbeddedAddonArchive({ archivePath, files, targetDir }) { + const pending = new Map(); + for (const file of files) { + if (!isSafeEmbeddedAddonFilename(file.filename)) { + throw new Error(`Unsafe embedded addon filename: ${file.filename}`); + } + const targetPath = path.join(targetDir, file.filename); + if (!isEmbeddedAddonFileCurrent(targetPath, file)) { + pending.set(file.filename, file); + } + } + if (pending.size === 0) return []; + + const archive = zlib.gunzipSync(fs.readFileSync(archivePath)); + const writtenPaths = []; + let offset = 0; + + while (offset + 512 <= archive.length) { + if (isZeroTarBlock(archive, offset)) break; + const header = archive.subarray(offset, offset + 512); + const filename = getTarEntryName(header); + const size = readTarOctal(header, 124, 12); + const typeflag = header[156] === 0 ? "0" : String.fromCharCode(header[156]); + offset += 512; + + if (offset + size > archive.length) { + throw new Error(`Truncated embedded addon archive entry: ${filename}`); + } + + if (!isSafeEmbeddedAddonFilename(filename)) { + throw new Error(`Unsafe embedded addon archive entry: ${filename}`); + } + if (typeflag !== "0") { + throw new Error(`Unsupported embedded addon archive entry type ${typeflag}: ${filename}`); + } + + const file = pending.get(filename); + if (file) { + if (typeof file.size === "number" && file.size !== size) { + throw new Error(`Embedded addon size mismatch for ${filename}: expected ${file.size}, got ${size}`); + } + const targetPath = path.join(targetDir, filename); + writeEmbeddedAddonFile(targetPath, archive.subarray(offset, offset + size)); + pending.delete(filename); + writtenPaths.push(targetPath); + } + + offset += Math.ceil(size / 512) * 512; + } + + if (pending.size > 0) { + throw new Error(`Embedded addon archive missing: ${[...pending.keys()].join(", ")}`); + } + + return writtenPaths; +} + function maybeExtractEmbeddedAddon(ctx, errors) { if (!ctx.isCompiledBinary || !embeddedAddon) return null; if (embeddedAddon.platformTag !== ctx.platformTag || embeddedAddon.version !== ctx.packageVersion) return null; @@ -244,9 +362,32 @@ function maybeExtractEmbeddedAddon(ctx, errors) { return null; } - if (fs.existsSync(targetPath)) { + if (embeddedAddon.archive) { + try { + extractEmbeddedAddonArchive({ + archivePath: embeddedAddon.archive.filePath, + files: embeddedAddon.files, + targetDir: ctx.versionedDir, + }); + if (isEmbeddedAddonFileCurrent(targetPath, selectedEmbeddedFile)) { + return targetPath; + } + errors.push(`embedded addon archive (${embeddedAddon.archive.filename}): missing ${selectedEmbeddedFile.filename}`); + return null; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + errors.push(`embedded addon archive (${embeddedAddon.archive.filename}): ${message}`); + return null; + } + } + + if (isEmbeddedAddonFileCurrent(targetPath, selectedEmbeddedFile)) { return targetPath; } + if (!selectedEmbeddedFile.filePath) { + errors.push(`embedded addon metadata missing file path for ${selectedEmbeddedFile.filename}`); + return null; + } try { const buffer = fs.readFileSync(selectedEmbeddedFile.filePath); diff --git a/packages/natives/scripts/build-native.ts b/packages/natives/scripts/build-native.ts index 23edd2aa6..edb56b4f9 100644 --- a/packages/natives/scripts/build-native.ts +++ b/packages/natives/scripts/build-native.ts @@ -145,6 +145,134 @@ async function installGeneratedBindings(outputDir: string): Promise { } } +async function isElfFile(filePath: string): Promise { + const handle = await fs.open(filePath, "r"); + try { + const buffer = Buffer.alloc(4); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + return bytesRead === 4 && buffer[0] === 0x7f && buffer[1] === 0x45 && buffer[2] === 0x4c && buffer[3] === 0x46; + } finally { + await handle.close(); + } +} + +function readElfUint(buffer: Buffer, offset: number, byteLength: 2 | 4 | 8, littleEndian: boolean): number { + if (offset < 0 || offset + byteLength > buffer.length) { + throw new Error("ELF section table is truncated."); + } + if (byteLength === 2) return littleEndian ? buffer.readUInt16LE(offset) : buffer.readUInt16BE(offset); + if (byteLength === 4) return littleEndian ? buffer.readUInt32LE(offset) : buffer.readUInt32BE(offset); + const value = littleEndian ? buffer.readBigUInt64LE(offset) : buffer.readBigUInt64BE(offset); + const numberValue = Number(value); + if (!Number.isSafeInteger(numberValue)) { + throw new Error(`ELF integer exceeds JavaScript's safe range: ${value}`); + } + return numberValue; +} + +function readCString(buffer: Buffer, offset: number): string { + if (offset < 0 || offset >= buffer.length) return ""; + let end = offset; + while (end < buffer.length && buffer[end] !== 0) end++; + return buffer.toString("utf8", offset, end); +} + +function readElfSectionNames(buffer: Buffer): string[] { + if (buffer.length < 0x40 || buffer[0] !== 0x7f || buffer[1] !== 0x45 || buffer[2] !== 0x4c || buffer[3] !== 0x46) { + return []; + } + const elfClass = buffer[4]; + const endian = buffer[5]; + if (elfClass !== 1 && elfClass !== 2) throw new Error(`Unsupported ELF class: ${elfClass}`); + if (endian !== 1 && endian !== 2) throw new Error(`Unsupported ELF endian marker: ${endian}`); + + const is64Bit = elfClass === 2; + const littleEndian = endian === 1; + const sectionHeaderOffset = is64Bit + ? readElfUint(buffer, 0x28, 8, littleEndian) + : readElfUint(buffer, 0x20, 4, littleEndian); + const sectionHeaderEntrySize = readElfUint(buffer, is64Bit ? 0x3a : 0x2e, 2, littleEndian); + const sectionHeaderCount = readElfUint(buffer, is64Bit ? 0x3c : 0x30, 2, littleEndian); + const sectionNameTableIndex = readElfUint(buffer, is64Bit ? 0x3e : 0x32, 2, littleEndian); + if (sectionHeaderOffset === 0 || sectionHeaderCount === 0) return []; + if (sectionNameTableIndex >= sectionHeaderCount) { + throw new Error("ELF section name table index is out of bounds."); + } + + const sectionHeadersEnd = sectionHeaderOffset + sectionHeaderEntrySize * sectionHeaderCount; + if (sectionHeadersEnd > buffer.length) { + throw new Error("ELF section headers extend past the end of the file."); + } + + const sectionHeader = (index: number) => sectionHeaderOffset + sectionHeaderEntrySize * index; + const sectionNameTableHeader = sectionHeader(sectionNameTableIndex); + const nameTableOffset = readElfUint( + buffer, + sectionNameTableHeader + (is64Bit ? 0x18 : 0x10), + is64Bit ? 8 : 4, + littleEndian, + ); + const nameTableSize = readElfUint( + buffer, + sectionNameTableHeader + (is64Bit ? 0x20 : 0x14), + is64Bit ? 8 : 4, + littleEndian, + ); + if (nameTableOffset + nameTableSize > buffer.length) { + throw new Error("ELF section name table extends past the end of the file."); + } + + const names: string[] = []; + const nameTable = buffer.subarray(nameTableOffset, nameTableOffset + nameTableSize); + for (let index = 0; index < sectionHeaderCount; index++) { + const nameOffset = readElfUint(buffer, sectionHeader(index), 4, littleEndian); + names.push(readCString(nameTable, nameOffset)); + } + return names; +} + +const forbiddenStrippedElfSections = new Set([".symtab", ".strtab"]); + +function getForbiddenElfSections(sectionNames: string[]): string[] { + return sectionNames.filter( + sectionName => + forbiddenStrippedElfSections.has(sectionName) || + sectionName.startsWith(".debug_") || + sectionName.startsWith(".zdebug_"), + ); +} + +async function runStripTool(addonPath: string): Promise { + const toolSpecs = [ + { command: "llvm-strip", args: ["--strip-unneeded"] }, + { command: "strip", args: ["--strip-unneeded"] }, + ]; + for (const tool of toolSpecs) { + const executable = Bun.which(tool.command); + if (!executable) continue; + const proc = Bun.spawn([executable, ...tool.args, addonPath], { + stdout: "pipe", + stderr: "pipe", + }); + const exitCode = await proc.exited; + if (exitCode === 0) return; + } +} + +async function stripAndVerifyNativeAddon(addonPath: string): Promise { + if (profileLabel !== "ci") return; + if (!(await isElfFile(addonPath))) return; + + await runStripTool(addonPath); + const sectionNames = readElfSectionNames(await fs.readFile(addonPath)); + const forbiddenSections = getForbiddenElfSections(sectionNames); + if (forbiddenSections.length > 0) { + throw new Error( + `Native addon ${path.basename(addonPath)} still contains stripped-release forbidden ELF sections: ${forbiddenSections.join(", ")}`, + ); + } +} + const isCI = Boolean(Bun.env.CI); const useLocalProfile = !isCI && !isCrossCompile; const profileLabel = useLocalProfile ? "local" : "ci"; @@ -213,6 +341,7 @@ try { } const builtAddonPath = await resolveBuiltAddonPath(buildOutputDir, canonicalAddonFilename); + await stripAndVerifyNativeAddon(builtAddonPath); if (builtAddonPath !== canonicalAddonPath) { console.log(`Normalizing native addon filename: ${path.basename(builtAddonPath)} → ${canonicalAddonFilename}`); await installBinary(builtAddonPath, canonicalAddonPath); diff --git a/packages/natives/scripts/embed-native.ts b/packages/natives/scripts/embed-native.ts index a8779a2eb..40d7e779d 100644 --- a/packages/natives/scripts/embed-native.ts +++ b/packages/natives/scripts/embed-native.ts @@ -5,17 +5,23 @@ const reset = process.argv.includes("--reset"); const outputPath = path.join(import.meta.dir, "../native/embedded-addon.js"); const packageJsonPath = path.join(import.meta.dir, "../package.json"); const nativeDir = path.join(import.meta.dir, "../native"); +const archivePrefix = "embedded-addons."; +const archiveSuffix = ".tar.gz"; -const stubContent = ` -// AUTOGENERATED FILE -- DO NOT EDIT DIRECTLY -// See scripts/embed-native.ts - -/** @typedef {"modern" | "baseline" | "default"} EmbeddedAddonVariant */ +const embeddedAddonTypedefs = `/** @typedef {"modern" | "baseline" | "default"} EmbeddedAddonVariant */ /** * @typedef {Object} EmbeddedAddonFile * @property {EmbeddedAddonVariant} variant * @property {string} filename + * @property {number} size + * @property {string=} filePath + */ + +/** + * @typedef {Object} EmbeddedAddonArchive + * @property {"tar.gz"} format + * @property {string} filename * @property {string} filePath */ @@ -24,13 +30,30 @@ const stubContent = ` * @property {string} platformTag * @property {string} version * @property {EmbeddedAddonFile[]} files - */ + * @property {EmbeddedAddonArchive=} archive + */`; + +const stubContent = ` +// AUTOGENERATED FILE -- DO NOT EDIT DIRECTLY +// See scripts/embed-native.ts + +${embeddedAddonTypedefs} /** @type {EmbeddedAddon|null} */ export const embeddedAddon = null; `; if (reset) { await Bun.write(outputPath, stubContent); + try { + const entries = await fs.readdir(nativeDir); + await Promise.all( + entries + .filter(entry => entry.startsWith(archivePrefix) && entry.endsWith(archiveSuffix)) + .map(entry => fs.unlink(path.join(nativeDir, entry))), + ); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err; + } process.exit(0); } @@ -39,15 +62,11 @@ interface CandidateAddon { filename: string; } -async function fileExists(filePath: string): Promise { - try { - await fs.stat(filePath); - return true; - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") return false; - throw err; - } +interface AvailableAddon extends CandidateAddon { + path: string; + size: number; } + const targetPlatform = Bun.env.TARGET_PLATFORM || process.platform; const targetArch = Bun.env.TARGET_ARCH || process.arch; const platformTag = `${targetPlatform}-${targetArch}`; @@ -59,11 +78,14 @@ const candidates: CandidateAddon[] = ] : [{ variant: "default", filename: `pi_natives.${platformTag}.node` }]; -const available: CandidateAddon[] = []; +const available: AvailableAddon[] = []; for (const candidate of candidates) { const candidatePath = path.join(nativeDir, candidate.filename); - if (await fileExists(candidatePath)) { - available.push(candidate); + try { + const stat = await fs.stat(candidatePath); + available.push({ ...candidate, path: candidatePath, size: stat.size }); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err; } } @@ -73,17 +95,18 @@ if (available.length === 0) { } const packageJson = (await Bun.file(packageJsonPath).json()) as { version: string }; -const imports = available - .map( - (candidate, index) => - `import addonPath${index} from ${JSON.stringify(`../native/${candidate.filename}`)} with { type: "file" };`, - ) - .join("\n"); +const archiveFilename = `${archivePrefix}${platformTag}${archiveSuffix}`; +const archivePath = path.join(nativeDir, archiveFilename); +const archiveEntries: Record = {}; +for (const addon of available) { + archiveEntries[addon.filename] = await fs.readFile(addon.path); +} +await Bun.write(archivePath, await new Bun.Archive(archiveEntries, { compress: "gzip", level: 9 }).bytes()); const files = available .map( - (candidate, index) => - `\t\t{ variant: ${JSON.stringify(candidate.variant)}, filename: ${JSON.stringify(candidate.filename)}, filePath: addonPath${index} },`, + addon => + `\t\t{ variant: ${JSON.stringify(addon.variant)}, filename: ${JSON.stringify(addon.filename)}, size: ${addon.size} },`, ) .join("\n"); @@ -91,27 +114,18 @@ const content = ` // AUTOGENERATED FILE -- DO NOT EDIT DIRECTLY // See scripts/embed-native.ts -/** @typedef {"modern" | "baseline" | "default"} EmbeddedAddonVariant */ +${embeddedAddonTypedefs} -/** - * @typedef {Object} EmbeddedAddonFile - * @property {EmbeddedAddonVariant} variant - * @property {string} filename - * @property {string} filePath - */ - -/** - * @typedef {Object} EmbeddedAddon - * @property {string} platformTag - * @property {string} version - * @property {EmbeddedAddonFile[]} files - */ - -${imports} +import archivePath from ${JSON.stringify(`../native/${archiveFilename}`)} with { type: "file" }; export const embeddedAddon = { \tplatformTag: ${JSON.stringify(platformTag)}, \tversion: ${JSON.stringify(packageJson.version)}, +\tarchive: { +\t\tformat: "tar.gz", +\t\tfilename: ${JSON.stringify(archiveFilename)}, +\t\tfilePath: archivePath, +\t}, \tfiles: [ ${files} \t], diff --git a/packages/natives/test/issue-823-repro.test.ts b/packages/natives/test/issue-823-repro.test.ts index 1560d94b8..9b657bf6f 100644 --- a/packages/natives/test/issue-823-repro.test.ts +++ b/packages/natives/test/issue-823-repro.test.ts @@ -26,8 +26,16 @@ */ import { describe, expect, it } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; import * as path from "node:path"; -import { detectCompiledBinary, getAddonFilenames, resolveLoaderCandidates } from "../native/loader-state.js"; +import { + detectCompiledBinary, + type EmbeddedAddonFile, + extractEmbeddedAddonArchive, + getAddonFilenames, + resolveLoaderCandidates, +} from "../native/loader-state.js"; describe("issue 823: standalone-binary native loader path resolution", () => { it("detects compiled-binary mode from embedded-addon presence when env and url markers are absent", () => { @@ -129,4 +137,42 @@ describe("issue 823: standalone-binary native loader path resolution", () => { expect(candidates).not.toContain(path.join(versionedDir, "pi_natives.linux-x64-baseline.node")); expect(candidates).not.toContain(path.join(userDataDir, "pi_natives.linux-x64-baseline.node")); }); + + it("extracts all bundled native variants from one gzip archive and skips current files", async () => { + const testDir = await fs.mkdtemp(path.join(os.tmpdir(), "natives-embedded-archive-")); + try { + const archivePath = path.join(testDir, "embedded-addons.linux-x64.tar.gz"); + const targetDir = path.join(testDir, "cache"); + await fs.mkdir(targetDir); + + const modern = Buffer.from("modern native addon"); + const baseline = Buffer.from("baseline native addon"); + const modernFilename = "pi_natives.linux-x64-modern.node"; + const baselineFilename = "pi_natives.linux-x64-baseline.node"; + await Bun.write( + archivePath, + await new Bun.Archive( + { + [modernFilename]: modern, + [baselineFilename]: baseline, + }, + { compress: "gzip", level: 9 }, + ).bytes(), + ); + + const files: EmbeddedAddonFile[] = [ + { variant: "modern", filename: modernFilename, size: modern.length }, + { variant: "baseline", filename: baselineFilename, size: baseline.length }, + ]; + + const written = extractEmbeddedAddonArchive({ archivePath, files, targetDir }); + expect(written.map(filePath => path.basename(filePath)).sort()).toEqual([baselineFilename, modernFilename]); + expect(await fs.readFile(path.join(targetDir, modernFilename), "utf8")).toBe("modern native addon"); + expect(await fs.readFile(path.join(targetDir, baselineFilename), "utf8")).toBe("baseline native addon"); + + expect(extractEmbeddedAddonArchive({ archivePath, files, targetDir })).toEqual([]); + } finally { + await fs.rm(testDir, { recursive: true, force: true }); + } + }); });