diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..e271c3e56 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,78 @@ +# Default build context filter, used by every build whose Dockerfile has no +# `.dockerignore` shadow next to it — today the install smokes in +# scripts/install-tests (`podman build -f scripts/install-tests/*.dockerfile .`). +# The pi and robomp images shadow this file with Dockerfile.dockerignore and +# Dockerfile.robomp.dockerignore; keep the three roughly in sync. +# +# Patterns are depth-agnostic (`**/`) to match .gitignore semantics: dockerignore +# anchors a bare `target/` at the context root only, so nested build dirs +# (go-port/*/target, packages/*/dist) otherwise ride along — gigabytes per build. + +# Heavy build outputs. +**/target/ +bazel-* +**/node_modules +**/dist/ +runs/ +.xwin-cache/ +packages/coding-agent/binaries/ +packages/natives/npm/ +packages/natives/native/.build/ +**/.cache/ +packages/snapcompact/research/results/ +# Host-built addons: every image that needs one compiles it from source. +**/*.node + +# Per-host scratch the pi codebase uses for parallel agents / worktrees. +.fallow/ +.worktrees/ +.wt/ +.opencode/ +.pi_config/ +.omp/plugins/ + +# VCS, editors, IDEs. +.git/ +.npm/ +.vscode/ +.zed/ +.idea/ + +# OS + transient noise. +.DS_Store +*.swp +*.swo +*~ +*.tmp + +# Logs + profiling artifacts. +*.log +*.cpuprofile +*.heapprofile +*.heapsnapshot +CPU.* + +# Build / test side outputs. +**/*.tsbuildinfo +**/coverage/ +.nyc_output/ +**/__pycache__/ +compaction-results/ +changes/ + +# Generated, regenerated in-image. +packages/ai/test/.temp-images/ +python/omp-rpc/src/omp_rpc.egg-info/ +python/robomp/data/ +python/robomp/.cache/ +python/robomp/src/robomp/static/ +python/robomp/web/dist/ + +# Scratch files the repo creates ad-hoc. +syntax.jsonl +out.jsonl +out.html +pi-*.html + +# Secrets. Should never be in an image regardless. +.env diff --git a/Cargo.toml b/Cargo.toml index 1699423b1..5f170d2d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,18 @@ [workspace] -members = ["crates/pi-*", "crates/vendor/*"] +# Explicit, not `crates/pi-*`: a glob member that resolves to a directory +# without a Cargo.toml (a deleted crate whose dir survives `git reset --hard` +# because an ignored file stayed behind) is a hard error that fails the whole +# workspace, not a skipped entry. +members = [ + "crates/pi-ast", + "crates/pi-builtins", + "crates/pi-iso", + "crates/pi-natives", + "crates/pi-shell", + "crates/pi-voice", + "crates/pi-walker", + "crates/vendor/brush-core", +] resolver = "3" [workspace.package] diff --git a/Dockerfile b/Dockerfile index 3cd9f5a02..6c8d44b97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,33 +25,41 @@ ARG BUN_VERSION=1.3.14 ############################ -# 1) natives-builder — Rust + Bun → pi_natives.linux-.node +# 1) natives-builder — Rust + Bun → pi_natives.linux-.node (local cargo) ############################ FROM rust:1.86-slim-bookworm AS natives-builder ARG BUN_VERSION + +# The addon is built with cargo/napi-rs (OMP_NATIVE_BUILD_BACKEND=cargo) +# instead of Bazel: the image is one fixed host target, so Bazel's hermetic +# cross toolchains and crate_universe splice buy nothing while costing a +# bazelisk download plus a full analysis phase on every build. `ci` profile = +# release codegen, thin LTO, stripped. ENV BUN_INSTALL=/opt/bun \ PATH=/opt/bun/bin:/usr/local/cargo/bin:/usr/local/bin:/usr/bin:/bin \ - CARGO_TERM_COLOR=never + CARGO_TERM_COLOR=never \ + OMP_NATIVE_BUILD_BACKEND=cargo \ + OMP_NATIVE_CARGO_PROFILE=ci # clang/libclang-dev: bindgen for pipewire-sys/libspa-sys (Linux desktop capture); # cmake/make/ninja-build: audiopus_sys builds bundled libopus via CMake. -# bazelisk: hermetic bazel launcher for the native addon build (17.1.5+). RUN apt-get update \ && apt-get install -y --no-install-recommends \ curl ca-certificates pkg-config libssl-dev unzip git \ clang libclang-dev cmake make ninja-build \ - && rm -rf /var/lib/apt/lists/* \ - && curl -fsSL -o /usr/local/bin/bazelisk \ - "https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-$(dpkg --print-architecture)" \ - && chmod +x /usr/local/bin/bazelisk \ - && ln -s /usr/local/bin/bazelisk /usr/local/bin/bazel + && rm -rf /var/lib/apt/lists/* RUN curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}" \ && /opt/bun/bin/bun --version WORKDIR /pi +# Layer 0 — the pinned nightly toolchain. Its own layer so a source or manifest +# edit never re-downloads ~5 rustup components. +COPY rust-toolchain.toml /pi/ +RUN rustup show + # Layer 1 — manifests + lockfiles only. Source edits under packages/*/src and # crates/*/src won't bust `bun install` below. `--parents` preserves the # matched path under /pi/ (requires syntax 1.7-labs). @@ -75,13 +83,12 @@ RUN bun install --frozen-lockfile --ignore-scripts COPY . /pi/ # Layer 4 — compile pi-natives to a Linux N-API addon. Persistent caches keep -# repeat builds incremental: cargo's package index + git-deps + the workspace -# target dir. -RUN --mount=type=cache,target=/root/.cargo/registry \ - --mount=type=cache,target=/root/.cargo/git \ +# repeat builds incremental: cargo's package index + git-deps (CARGO_HOME is +# /usr/local/cargo in the rust image, not ~/.cargo) + the workspace target dir. +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/pi/target \ set -eux; \ - rustup show; \ bun --cwd=packages/natives run build; \ mkdir -p /out; \ cp packages/natives/native/pi_natives.linux-*.node /out/ diff --git a/Dockerfile.dockerignore b/Dockerfile.dockerignore index 7d6c0840d..05fa0925b 100644 --- a/Dockerfile.dockerignore +++ b/Dockerfile.dockerignore @@ -2,13 +2,21 @@ # .dockerignore for this file only. Robomp uses Dockerfile.robomp + # Dockerfile.robomp.dockerignore alongside. -# Heavy build outputs — must never reach the build context. `target/` alone is -# >100 GB on a dev machine; `bazel-*` symlinks point at the bazel output base. -target/ +# Heavy build outputs — must never reach the build context. Patterns are +# depth-agnostic (`**/`) to match .gitignore semantics: dockerignore anchors +# bare `target/`/`dist/` at the context root only, which leaked +# packages/*/dist (~600 MB) and go-port/*/target (~1.4 GB) into every build. +# `bazel-*` symlinks point at the bazel output base. +**/target/ bazel-* **/node_modules -dist/ +**/dist/ runs/ +.xwin-cache/ +packages/coding-agent/binaries/ +packages/natives/npm/ +**/.cache/ +packages/snapcompact/research/results/ # Per-host scratch the pi codebase uses for parallel agents / worktrees. .fallow/ @@ -40,18 +48,20 @@ runs/ CPU.* # Build / test side outputs. -*.tsbuildinfo -coverage/ +**/*.tsbuildinfo +**/coverage/ .nyc_output/ -__pycache__/ +**/__pycache__/ compaction-results/ changes/ # Generated files (the in-image build regenerates them). packages/coding-agent/src/internal-urls/docs-index.generated.ts packages/natives/native/.build/ -packages/natives/native/pi_natives.darwin-*.node -packages/natives/native/pi_natives.dev.node +# Every host-built addon: natives-builder compiles its own, and a stale host +# `pi_natives.linux-*.node` riding in would be copied to /out alongside it. +# (Also drops the ~760 MB of dev addons a working checkout accumulates.) +**/*.node packages/ai/test/.temp-images/ python/omp-rpc/src/omp_rpc.egg-info/ python/robomp/data/ diff --git a/Dockerfile.robomp.dockerignore b/Dockerfile.robomp.dockerignore index bd5bc3e72..21ac7290d 100644 --- a/Dockerfile.robomp.dockerignore +++ b/Dockerfile.robomp.dockerignore @@ -5,11 +5,23 @@ # the cost of per-Dockerfile shadows (no shared file to factor common rules # into). Keep them roughly in sync. -# Heavy build outputs — must never reach the build context. -target/ +# Heavy build outputs — must never reach the build context. Depth-agnostic for +# the same reason as Dockerfile.dockerignore: bare `target/`/`dist/` only match +# the context root, so nested build dirs (go-port/*/target, packages/*/dist) +# were shipped on every build. +**/target/ +bazel-* **/node_modules -dist/ +**/dist/ runs/ +.xwin-cache/ +packages/coding-agent/binaries/ +packages/natives/npm/ +packages/natives/native/ +**/.cache/ +packages/snapcompact/research/results/ +# The addon comes from PI_BASE; nothing in this context needs one. +**/*.node # Per-host scratch the pi codebase uses for parallel agents / worktrees. .fallow/ diff --git a/docs/native-crates.md b/docs/native-crates.md index 116e566c4..8f206f4f5 100644 --- a/docs/native-crates.md +++ b/docs/native-crates.md @@ -2,7 +2,7 @@ Contributor map for Rust workspace members under `crates/`. They are implementation details behind `@oh-my-pi/pi-natives` and its embedded shell; package consumers use JavaScript entrypoints, not these crate APIs. -The root `Cargo.toml` includes `crates/pi-*` and `crates/vendor/*` as workspace members. It also patches crates.io `brush-core` to the vendored copy. +The root `Cargo.toml` lists every crate under `crates/` explicitly in `workspace.members` — add new crates there. It also patches crates.io `brush-core` to the vendored copy. ## First-party crates diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index 6165a6cda..7ddd27f28 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -2,6 +2,15 @@ ## [Unreleased] +### Changed + +- Docker images (`Dockerfile`, `scripts/install-tests/*.dockerfile`) build the native addon through the cargo/napi-rs backend (`OMP_NATIVE_BUILD_BACKEND=cargo`) instead of Bazel: a single fixed host target gains nothing from hermetic cross toolchains, and none of those images shipped bazelisk. `OMP_NATIVE_CARGO_PROFILE` picks the profile for that path (images use `ci`, local default stays `local`). + +### Fixed + +- Fixed the root Cargo workspace failing to load when a stale directory exists under `crates/` — e.g. a deleted crate whose directory survived `git reset --hard`. `members` no longer globs `crates/pi-*`, so a directory without a `Cargo.toml` can no longer break every cargo and Bazel build. +- Fixed Docker build contexts shipping nested build output: `.dockerignore` patterns are anchored at the context root, so bare `target/` and `dist/` matched neither `go-port/*/target` (~1.4 GB) nor `packages/*/dist` (~600 MB). + ## [17.3.1] - 2026-08-13 ### Fixed diff --git a/packages/natives/scripts/build-bindings.ts b/packages/natives/scripts/build-bindings.ts index 2c0182294..276624441 100644 --- a/packages/natives/scripts/build-bindings.ts +++ b/packages/natives/scripts/build-bindings.ts @@ -1,9 +1,12 @@ /** - * Dev-only napi build that regenerates the TypeScript bindings - * (native/index.d.ts) and the runtime enum exports. Shipping addons are built - * by Bazel (`bun run build` → scripts/bazel-natives.ts); run this - * (`bun run build:bindings`) only when the Rust API changes its exported - * typedefs. Host target only, local cargo profile — no cross-compilation. + * Local napi build: regenerates the TypeScript bindings (native/index.d.ts) + * and the runtime enum exports, then installs the host addon. Release addons + * come from Bazel (`bun run build` → scripts/bazel-natives.ts); this path also + * serves hosts Bazel cannot run on (Windows, the Docker image) via + * `OMP_NATIVE_BUILD_BACKEND=cargo`. Host target only — no cross-compilation. + * + * `OMP_NATIVE_CARGO_PROFILE` selects the cargo profile (default `local`: + * incremental, unstripped). Image builds set `ci` for a stripped addon. */ import * as fsSync from "node:fs"; @@ -203,6 +206,10 @@ if (!napiBinEntry) { } const napiBin = path.join(path.dirname(napiManifestPath), napiBinEntry); +// Profiles live in the root Cargo.toml; `local` trades size for iteration +// speed, `ci` strips and drops incremental state. +const cargoProfile = Bun.env.OMP_NATIVE_CARGO_PROFILE?.trim() || "local"; + const napiArgs = [ "build", "--manifest-path", @@ -216,7 +223,7 @@ const napiArgs = [ "-o", buildOutputDir, "--profile", - "local", + cargoProfile, ]; // napi-rs / cargo route much failure detail to stdout (e.g. `cargo metadata` diff --git a/scripts/install-tests/binary.dockerfile b/scripts/install-tests/binary.dockerfile index da804617a..2cf4aa05a 100644 --- a/scripts/install-tests/binary.dockerfile +++ b/scripts/install-tests/binary.dockerfile @@ -7,9 +7,11 @@ RUN apt-get update && apt-get install -y curl ca-certificates unzip build-essent RUN curl -fsSL https://bun.sh/install | bash ENV PATH="/root/.bun/bin:$PATH" -# Install Rust +# Install Rust. The natives build defaults to Bazel; this image has no +# bazelisk and needs only the host addon, so route it through cargo/napi-rs. RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly -ENV PATH="/root/.cargo/bin:$PATH" +ENV PATH="/root/.cargo/bin:$PATH" \ + OMP_NATIVE_BUILD_BACKEND=cargo # Copy local repo WORKDIR /repo diff --git a/scripts/install-tests/source.dockerfile b/scripts/install-tests/source.dockerfile index f4c3c0000..b0ee944ae 100644 --- a/scripts/install-tests/source.dockerfile +++ b/scripts/install-tests/source.dockerfile @@ -7,9 +7,12 @@ RUN apt-get update && apt-get install -y curl ca-certificates unzip build-essent RUN curl -fsSL https://bun.sh/install | bash ENV PATH="/root/.bun/bin:$PATH" -# Install Rust (needed to build native addon) +# Install Rust (needed to build native addon). The natives build defaults to +# Bazel; this image has no bazelisk and needs only the host addon, so route it +# through the cargo/napi-rs backend. RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly -ENV PATH="/root/.cargo/bin:$PATH" +ENV PATH="/root/.cargo/bin:$PATH" \ + OMP_NATIVE_BUILD_BACKEND=cargo # Copy local repo WORKDIR /repo diff --git a/scripts/install-tests/tarball.dockerfile b/scripts/install-tests/tarball.dockerfile index a68df0196..28bf6dbb9 100644 --- a/scripts/install-tests/tarball.dockerfile +++ b/scripts/install-tests/tarball.dockerfile @@ -8,9 +8,12 @@ RUN apt-get update && apt-get install -y curl ca-certificates unzip jq procps bu RUN curl -fsSL https://bun.sh/install | bash ENV PATH="/root/.bun/bin:$PATH" -# Install Rust (needed to build native addon) +# Install Rust (needed to build native addon). The natives build defaults to +# Bazel; this image has no bazelisk and needs only the host addon, so route it +# through the cargo/napi-rs backend. RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly -ENV PATH="/root/.cargo/bin:$PATH" +ENV PATH="/root/.cargo/bin:$PATH" \ + OMP_NATIVE_BUILD_BACKEND=cargo # Install Node.js (needed for verdaccio and npm) RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \