From 1e52c837c832c5992cf540e47998fa811d930807 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sun, 1 Feb 2026 05:17:06 +0100 Subject: [PATCH] feat(natives): added native addon distribution and platform validation to CLI update mechanism - Added native addon distribution and installation support to the CLI update mechanism, enabling platform-specific native modules to be downloaded and installed alongside the main binary. - Added platform validation in native addon loader to verify supported platforms (linux-x64, linux-arm64, darwin-x64, darwin-arm64, win32-x64) with helpful error messages for unsupported configurations. - Enhanced native addon loading error messages to provide platform-specific troubleshooting guidance for both npm/bun installations and local development environments. - Updated installation scripts (install.sh, install.ps1) to download and install native addon files alongside the main binary during CLI installation. --- .github/workflows/ci.yml | 14 ++++++ packages/coding-agent/CHANGELOG.md | 3 +- packages/coding-agent/package.json | 2 +- packages/coding-agent/src/cli/update-cli.ts | 48 +++++++++++++++++++-- packages/natives/src/native.ts | 19 +++++++- scripts/install-tests/binary.dockerfile | 15 +++++-- scripts/install-tests/source.dockerfile | 9 +++- scripts/install-tests/tarball.dockerfile | 9 +++- scripts/install.ps1 | 18 +++++--- scripts/install.sh | 17 +++++--- 10 files changed, 130 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7ca3dd44..7c0b20496 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,20 @@ jobs: pattern: pi-natives-* path: packages/natives/native merge-multiple: true + - name: Verify native addons + run: | + echo "Native addons downloaded:" + ls -la packages/natives/native/ + echo "" + echo "Expected platforms: linux-x64, linux-arm64, darwin-x64, darwin-arm64, win32-x64" + for plat in linux-x64 linux-arm64 darwin-x64 darwin-arm64 win32-x64; do + if [ -f "packages/natives/native/pi_natives.${plat}.node" ]; then + echo "✓ pi_natives.${plat}.node" + else + echo "✗ pi_natives.${plat}.node MISSING" + exit 1 + fi + done - name: Build binaries run: | diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 0d15702e6..ce9664427 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,7 +1,6 @@ # Changelog ## [Unreleased] - ### Added - Added `maxResults` parameter to find tool to limit result set at the native layer @@ -14,6 +13,8 @@ ### Changed +- Updated binary build process to exclude worker files from compilation, reducing binary size +- Modified update mechanism to download and install native addon alongside CLI binary for platform-specific functionality - Updated find tool to emit streaming match updates via callback, allowing real-time progress feedback during file searches - Modified find tool to use native match metadata (mtime, fileType) from WASM layer instead of redundant filesystem stats, improving performance - Restructured Task tool documentation to emphasize context quality and explicit API contracts for subagent success diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 57a99c88b..cf427e4b2 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -74,7 +74,7 @@ "scripts": { "check": "tsgo -p tsconfig.json", "format-prompts": "bun scripts/format-prompts.ts", - "build:binary": "cd ../.. && bun build --compile --define OMP_COMPILED=true --root . ./packages/coding-agent/src/cli.ts ./packages/natives/src/grep/worker.ts ./packages/natives/src/html/worker.ts ./packages/natives/src/image/worker.ts --outfile packages/coding-agent/dist/omp", + "build:binary": "cd ../.. && bun build --compile --define OMP_COMPILED=true --root . ./packages/coding-agent/src/cli.ts --outfile packages/coding-agent/dist/omp", "generate-template": "bun scripts/generate-template.ts", "test": "bun test" }, diff --git a/packages/coding-agent/src/cli/update-cli.ts b/packages/coding-agent/src/cli/update-cli.ts index ed78bccb0..5f7d58b9e 100644 --- a/packages/coding-agent/src/cli/update-cli.ts +++ b/packages/coding-agent/src/cli/update-cli.ts @@ -135,6 +135,24 @@ function getBinaryName(): string { return `${APP_NAME}-${os}-${archName}`; } +/** + * Get the appropriate native addon name for this platform. + * Uses process.platform directly (linux, darwin, win32). + */ +function getNativeAddonName(): string { + const platform = process.platform; + const arch = process.arch; + + if (!["linux", "darwin", "win32"].includes(platform)) { + throw new Error(`Unsupported platform: ${platform}`); + } + if (!["x64", "arm64"].includes(arch)) { + throw new Error(`Unsupported architecture: ${arch}`); + } + + return `pi_natives.${platform}-${arch}.node`; +} + /** * Update via bun package manager. */ @@ -154,16 +172,24 @@ async function updateViaBun(): Promise { */ async function updateViaBinary(release: ReleaseInfo): Promise { const binaryName = getBinaryName(); + const nativeAddonName = getNativeAddonName(); + const asset = release.assets.find(a => a.name === binaryName); + const nativeAsset = release.assets.find(a => a.name === nativeAddonName); if (!asset) { throw new Error(`No binary found for ${binaryName}`); } + if (!nativeAsset) { + throw new Error(`No native addon found for ${nativeAddonName}`); + } const execPath = process.execPath; - const _execDir = path.dirname(execPath); + const execDir = path.dirname(execPath); const tempPath = `${execPath}.new`; const backupPath = `${execPath}.bak`; + const nativePath = path.join(execDir, nativeAddonName); + const nativeTempPath = `${nativePath}.new`; console.log(chalk.dim(`Downloading ${binaryName}...`)); @@ -177,6 +203,18 @@ async function updateViaBinary(release: ReleaseInfo): Promise { const nodeStream = Readable.fromWeb(response.body as import("stream/web").ReadableStream); await pipeline(nodeStream, fileStream); + // Download native addon + console.log(chalk.dim(`Downloading ${nativeAddonName}...`)); + + const nativeResponse = await fetch(nativeAsset.url, { redirect: "follow" }); + if (!nativeResponse.ok || !nativeResponse.body) { + throw new Error(`Native addon download failed: ${nativeResponse.statusText}`); + } + + const nativeFileStream = fs.createWriteStream(nativeTempPath, { mode: 0o755 }); + const nativeNodeStream = Readable.fromWeb(nativeResponse.body as import("stream/web").ReadableStream); + await pipeline(nativeNodeStream, nativeFileStream); + // Replace current binary console.log(chalk.dim("Installing update...")); @@ -187,11 +225,12 @@ async function updateViaBinary(release: ReleaseInfo): Promise { if (!isEnoent(err)) throw err; } await fs.promises.rename(execPath, backupPath); - await fs.promises.rename(tempPath, execPath); - await fs.promises.unlink(backupPath); + // Replace native addon (no backup needed, just overwrite) + await fs.promises.rename(nativeTempPath, nativePath); + console.log(chalk.green(`\n${theme.status.success} Updated to ${release.version}`)); console.log(chalk.dim(`Restart ${APP_NAME} to use the new version`)); } catch (err) { @@ -201,6 +240,9 @@ async function updateViaBinary(release: ReleaseInfo): Promise { if (fs.existsSync(tempPath)) { await fs.promises.unlink(tempPath); } + if (fs.existsSync(nativeTempPath)) { + await fs.promises.unlink(nativeTempPath); + } throw err; } } diff --git a/packages/natives/src/native.ts b/packages/natives/src/native.ts index aef2645cd..89cba0b91 100644 --- a/packages/natives/src/native.ts +++ b/packages/natives/src/native.ts @@ -62,6 +62,9 @@ const platformTag = `${process.platform}-${process.arch}`; const nativeDir = path.join(import.meta.dir, "..", "native"); const repoRoot = path.join(import.meta.dir, "..", "..", ".."); const execDir = path.dirname(process.execPath); + +const SUPPORTED_PLATFORMS = ["linux-x64", "linux-arm64", "darwin-x64", "darwin-arm64", "win32-x64"]; + const candidates = [ path.join(nativeDir, `pi_natives.${platformTag}.node`), path.join(nativeDir, "pi_natives.node"), @@ -85,8 +88,22 @@ function loadNative(): NativeBindings { } } + // Check if this is an unsupported platform + if (!SUPPORTED_PLATFORMS.includes(platformTag)) { + throw new Error( + `Unsupported platform: ${platformTag}\n` + + `Supported platforms: ${SUPPORTED_PLATFORMS.join(", ")}\n` + + "If you need support for this platform, please open an issue.", + ); + } + const details = errors.map(error => `- ${error}`).join("\n"); - throw new Error(`Failed to load pi_natives native addon. Tried:\n${details}`); + throw new Error( + `Failed to load pi_natives native addon for ${platformTag}.\n\n` + + `Tried:\n${details}\n\n` + + "If installed via npm/bun, try reinstalling: bun install @oh-my-pi/pi-natives\n" + + "If developing locally, build with: bun --cwd=packages/natives run build:native", + ); } function validateNative(bindings: NativeBindings, source: string): void { diff --git a/scripts/install-tests/binary.dockerfile b/scripts/install-tests/binary.dockerfile index cfa81a95e..afa99735b 100644 --- a/scripts/install-tests/binary.dockerfile +++ b/scripts/install-tests/binary.dockerfile @@ -1,22 +1,29 @@ # Test binary build from local source FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y curl ca-certificates unzip && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y curl ca-certificates unzip build-essential && rm -rf /var/lib/apt/lists/* # Install bun RUN curl -fsSL https://bun.sh/install | bash ENV PATH="/root/.bun/bin:$PATH" +# Install Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly +ENV PATH="/root/.cargo/bin:$PATH" + # Copy local repo WORKDIR /repo COPY . . -# Build binary +# Build native addon and binary RUN bun install --frozen-lockfile +RUN bun --cwd=packages/natives run build:native RUN cd packages/coding-agent && bun run build:binary -# Install binary to PATH -RUN mkdir -p /root/.local/bin && cp packages/coding-agent/dist/omp /root/.local/bin/ +# Install binary and native addon to PATH +RUN mkdir -p /root/.local/bin && \ + cp packages/coding-agent/dist/omp /root/.local/bin/ && \ + cp packages/natives/native/pi_natives.linux-x64.node /root/.local/bin/ ENV PATH="/root/.local/bin:$PATH" # Verify diff --git a/scripts/install-tests/source.dockerfile b/scripts/install-tests/source.dockerfile index d83d33e94..cfa6ee55a 100644 --- a/scripts/install-tests/source.dockerfile +++ b/scripts/install-tests/source.dockerfile @@ -1,18 +1,23 @@ # Test --source install from local repo FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y curl ca-certificates unzip && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y curl ca-certificates unzip build-essential && rm -rf /var/lib/apt/lists/* # Install bun RUN curl -fsSL https://bun.sh/install | bash ENV PATH="/root/.bun/bin:$PATH" +# Install Rust (needed to build native addon) +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly +ENV PATH="/root/.cargo/bin:$PATH" + # Copy local repo WORKDIR /repo COPY . . -# Install dependencies and link globally +# Install dependencies, build native addon, and link globally RUN bun install --frozen-lockfile +RUN bun --cwd=packages/natives run build:native RUN cd packages/coding-agent && bun link # Verify diff --git a/scripts/install-tests/tarball.dockerfile b/scripts/install-tests/tarball.dockerfile index 113da6ce1..a2bae49d4 100644 --- a/scripts/install-tests/tarball.dockerfile +++ b/scripts/install-tests/tarball.dockerfile @@ -2,12 +2,16 @@ # Uses verdaccio as local registry to test full publish/install cycle FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y curl ca-certificates unzip jq procps && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y curl ca-certificates unzip jq procps build-essential && rm -rf /var/lib/apt/lists/* # Install bun RUN curl -fsSL https://bun.sh/install | bash ENV PATH="/root/.bun/bin:$PATH" +# Install Rust (needed to build native addon) +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly +ENV PATH="/root/.cargo/bin:$PATH" + # Install Node.js (needed for verdaccio and npm) RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ && apt-get install -y nodejs \ @@ -22,6 +26,7 @@ COPY . . # Build the project RUN bun install --frozen-lockfile +RUN bun --cwd=packages/natives run build:native # Create verdaccio config (allow anonymous publish) RUN mkdir -p /root/.config/verdaccio && cat > /root/.config/verdaccio/config.yaml <<'EOF' @@ -63,7 +68,7 @@ RUN cat > /repo/scripts/publish-local.sh <<'SCRIPT' set -e REGISTRY="http://localhost:4873" -PACKAGES=(utils ai agent tui stats coding-agent) +PACKAGES=(utils natives ai agent tui stats coding-agent) # Build version map from all package.json files declare -A VERSION_MAP diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 9f42f99d9..e63a0a175 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -20,6 +20,7 @@ $Repo = "can1357/oh-my-pi" $Package = "@oh-my-pi/pi-coding-agent" $InstallDir = if ($env:OMP_INSTALL_DIR) { $env:OMP_INSTALL_DIR } else { "$env:LOCALAPPDATA\omp" } $BinaryName = "omp-windows-x64.exe" +$NativeAddonName = "pi_natives.win32-x64.node" $MinimumBunVersion = "1.3.7" function Test-BunInstalled { @@ -254,16 +255,23 @@ function Install-Binary { } Write-Host "Using version: $Latest" - # Download binary - $Url = "https://github.com/$Repo/releases/download/$Latest/$BinaryName" - Write-Host "Downloading $BinaryName..." - New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null + + # Download binary + $BinaryUrl = "https://github.com/$Repo/releases/download/$Latest/$BinaryName" + Write-Host "Downloading $BinaryName..." $OutPath = Join-Path $InstallDir "omp.exe" - Invoke-WebRequest -Uri $Url -OutFile $OutPath + Invoke-WebRequest -Uri $BinaryUrl -OutFile $OutPath + + # Download native addon + $NativeUrl = "https://github.com/$Repo/releases/download/$Latest/$NativeAddonName" + Write-Host "Downloading $NativeAddonName..." + $NativeOutPath = Join-Path $InstallDir $NativeAddonName + Invoke-WebRequest -Uri $NativeUrl -OutFile $NativeOutPath Write-Host "" Write-Host "✓ Installed omp to $OutPath" -ForegroundColor Green + Write-Host "✓ Installed native addon to $NativeOutPath" -ForegroundColor Green # Add to PATH if not already there $UserPath = [Environment]::GetEnvironmentVariable("Path", "User") diff --git a/scripts/install.sh b/scripts/install.sh index 2581d6b09..73a168d08 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -200,6 +200,7 @@ install_binary() { esac BINARY="omp-${PLATFORM}-${ARCH}" + NATIVE_ADDON="pi_natives.${PLATFORM}-${ARCH}.node" # Get release tag if [ -n "$REF" ]; then @@ -222,16 +223,22 @@ install_binary() { fi echo "Using version: $LATEST" - # Download binary - URL="https://github.com/${REPO}/releases/download/${LATEST}/${BINARY}" - echo "Downloading ${BINARY}..." - mkdir -p "$INSTALL_DIR" - curl -fsSL "$URL" -o "${INSTALL_DIR}/omp" + + # Download binary + BINARY_URL="https://github.com/${REPO}/releases/download/${LATEST}/${BINARY}" + echo "Downloading ${BINARY}..." + curl -fsSL "$BINARY_URL" -o "${INSTALL_DIR}/omp" chmod +x "${INSTALL_DIR}/omp" + # Download native addon + NATIVE_URL="https://github.com/${REPO}/releases/download/${LATEST}/${NATIVE_ADDON}" + echo "Downloading ${NATIVE_ADDON}..." + curl -fsSL "$NATIVE_URL" -o "${INSTALL_DIR}/${NATIVE_ADDON}" + echo "" echo "✓ Installed omp to ${INSTALL_DIR}/omp" + echo "✓ Installed native addon to ${INSTALL_DIR}/${NATIVE_ADDON}" # Check if in PATH case ":$PATH:" in