diff --git a/.github/actions/build-native-kata/action.yml b/.github/actions/build-native-kata/action.yml new file mode 100644 index 000000000..6f2223ade --- /dev/null +++ b/.github/actions/build-native-kata/action.yml @@ -0,0 +1,134 @@ +name: Build native addon (omp-kata) +description: > + Build the pi_natives cdylib on the preloaded omp-kata runner image, using + baked toolchains and the shared RustFS-backed sccache instead of per-job tool + setup downloads. + +inputs: + hash: + description: Rust source hash used in the artifact name + required: true + platform: + description: Target platform (linux, darwin, win32) + required: true + arch: + description: Target arch (x64, arm64) + required: true + variant: + description: Optional build variant (baseline, modern); required for native x64 builds. + required: false + default: "" + target: + description: Optional rustc target triple for cross-compilation + required: false + default: "" + rust_checks: + description: Run clippy/rustfmt checks (only one matrix entry should set this) + required: false + default: "false" + save_cache: + description: Kept for interface parity with the GitHub-hosted action; unused here. + required: false + default: "false" + +runs: + using: composite + steps: + - uses: ./.github/actions/ensure-rust-toolchain + with: + toolchain: nightly-2026-04-29 + components: ${{ inputs.rust_checks == 'true' && 'clippy,rustfmt' || '' }} + target: ${{ inputs.target }} + - name: Configure native Rust flags + if: inputs.target == '' + shell: bash + env: + TARGET_ARCH: ${{ inputs.arch }} + TARGET_VARIANT: ${{ inputs.variant }} + run: | + case "$TARGET_ARCH:$TARGET_VARIANT" in + x64:modern) + rustflags="-C target-cpu=x86-64-v3" + ;; + x64:baseline) + rustflags="-C target-cpu=x86-64-v2" + ;; + x64:*) + echo "::error::x64 native builds require variant=modern or variant=baseline" + exit 1 + ;; + *) + if [ -n "${RUSTFLAGS:-}" ]; then + echo "Using caller-provided RUSTFLAGS=$RUSTFLAGS" + exit 0 + fi + rustflags="-C target-cpu=native" + ;; + esac + + echo "RUSTFLAGS=$rustflags" >> "$GITHUB_ENV" + echo "Configured RUSTFLAGS=$rustflags" + - uses: ./.github/actions/ensure-sccache + with: + version: "0.15.0" + - name: Enable sccache for cargo + shell: bash + run: | + { + echo "RUSTC_WRAPPER=sccache" + echo "CARGO_INCREMENTAL=0" + } >> "$GITHUB_ENV" + echo "sccache backend: shared S3 ($SCCACHE_BUCKET @ $SCCACHE_ENDPOINT)" + - uses: ./.github/actions/ensure-cargo-tool + if: inputs.target == '' + with: + binary: cargo-nextest + crate: cargo-nextest + - uses: ./.github/actions/bun-install + - uses: ./.github/actions/ensure-zig + if: inputs.target != '' && !endsWith(inputs.target, '-msvc') + with: + version: "0.16.0" + - uses: ./.github/actions/ensure-cargo-tool + if: inputs.target != '' && !endsWith(inputs.target, '-msvc') + with: + binary: cargo-zigbuild + crate: cargo-zigbuild + - uses: ./.github/actions/ensure-cargo-tool + if: endsWith(inputs.target, '-msvc') + with: + binary: cargo-xwin + crate: cargo-xwin + - name: Cache cargo-xwin Windows SDK + if: endsWith(inputs.target, '-msvc') + uses: actions/cache@v4 + with: + path: ~/.cache/cargo-xwin + key: cargo-xwin-${{ runner.os }}-v1 + - name: Accept xwin license + if: endsWith(inputs.target, '-msvc') + shell: bash + run: echo "XWIN_ACCEPT_LICENSE=1" >> "$GITHUB_ENV" + - name: Rust checks + if: inputs.rust_checks == 'true' + shell: bash + run: bun run check:rs + - name: Test workspace (Rust) + if: inputs.target == '' && inputs.platform != 'darwin' + shell: bash + run: bun run test:rs + - name: Build native addon(s) + shell: bash + env: + CROSS_TARGET: ${{ inputs.target }} + TARGET_PLATFORM: ${{ inputs.platform }} + TARGET_ARCH: ${{ inputs.arch }} + TARGET_VARIANTS: ${{ inputs.variant }} + run: bun run ci:build:native + - name: Upload native addon(s) + uses: actions/upload-artifact@v4 + with: + name: pi-natives-${{ inputs.platform }}-${{ inputs.arch }}${{ inputs.variant && format('-{0}', inputs.variant) || '' }}-h${{ inputs.hash }} + path: packages/natives/native/pi_natives.${{ inputs.platform }}-${{ inputs.arch }}*.node + if-no-files-found: error + retention-days: 90 diff --git a/.github/actions/ensure-cargo-tool/action.yml b/.github/actions/ensure-cargo-tool/action.yml new file mode 100644 index 000000000..dcde8d72f --- /dev/null +++ b/.github/actions/ensure-cargo-tool/action.yml @@ -0,0 +1,38 @@ +name: "ensure cargo tool" +description: Ensure a cargo-installed CLI is present. + +inputs: + binary: + required: true + description: Binary name expected on PATH + crate: + required: false + default: "" + description: Crate name to cargo install; defaults to the binary name + version: + required: false + default: "" + description: Optional crate version + +runs: + using: composite + steps: + - shell: bash + env: + BINARY: ${{ inputs.binary }} + CRATE: ${{ inputs.crate }} + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if command -v "$BINARY" >/dev/null 2>&1; then + echo "Using baked cargo tool: $BINARY" + exit 0 + fi + + crate="${CRATE:-$BINARY}" + install_args=(install --locked "$crate") + if [ -n "$VERSION" ]; then + install_args+=(--version "$VERSION") + fi + cargo "${install_args[@]}" + echo "Installed cargo tool: $BINARY" diff --git a/.github/actions/ensure-rust-toolchain/action.yml b/.github/actions/ensure-rust-toolchain/action.yml new file mode 100644 index 000000000..7946a9670 --- /dev/null +++ b/.github/actions/ensure-rust-toolchain/action.yml @@ -0,0 +1,59 @@ +name: "ensure rust toolchain" +description: > + Ensure a pinned rustup toolchain, optional components, and an optional target + are present, then prepend the real toolchain bin dir to PATH. + +inputs: + toolchain: + required: true + description: Rust toolchain name (for example nightly-2026-04-29) + components: + required: false + default: "" + description: Optional comma-separated rustup components + target: + required: false + default: "" + description: Optional rustup target triple + +runs: + using: composite + steps: + - shell: bash + env: + TOOLCHAIN: ${{ inputs.toolchain }} + COMPONENTS: ${{ inputs.components }} + TARGET: ${{ inputs.target }} + run: | + set -euo pipefail + if ! command -v rustup >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain "$TOOLCHAIN" --profile minimal + fi + + if ! rustc +"$TOOLCHAIN" --version >/dev/null 2>&1; then + rustup toolchain install "$TOOLCHAIN" --profile minimal --no-self-update + fi + rustup default "$TOOLCHAIN" + + missing_components=() + if [ -n "$COMPONENTS" ]; then + IFS=',' read -r -a wanted_components <<< "$COMPONENTS" + for component in "${wanted_components[@]}"; do + [ -z "$component" ] && continue + if ! rustup component list --toolchain "$TOOLCHAIN" --installed | grep -qx "$component"; then + missing_components+=("$component") + fi + done + fi + if [ ${#missing_components[@]} -gt 0 ]; then + rustup component add --toolchain "$TOOLCHAIN" "${missing_components[@]}" + fi + + if [ -n "$TARGET" ] && ! rustup target list --toolchain "$TOOLCHAIN" --installed | grep -qx "$TARGET"; then + rustup target add --toolchain "$TOOLCHAIN" "$TARGET" + fi + + toolchain_bin="$(dirname "$(rustup which cargo --toolchain "$TOOLCHAIN")")" + echo "$toolchain_bin" >> "$GITHUB_PATH" + echo "Using Rust toolchain: $(rustc +"$TOOLCHAIN" --version)" diff --git a/.github/actions/ensure-sccache/action.yml b/.github/actions/ensure-sccache/action.yml new file mode 100644 index 000000000..cb10f9909 --- /dev/null +++ b/.github/actions/ensure-sccache/action.yml @@ -0,0 +1,47 @@ +name: "ensure sccache" +description: Ensure a pinned sccache binary is on PATH. + +inputs: + version: + required: false + default: "0.15.0" + description: sccache release version without the leading v + +runs: + using: composite + steps: + - shell: bash + env: + SCCACHE_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if command -v sccache >/dev/null 2>&1; then + current="$(sccache --version | awk '{print $2}')" + if [ "$current" = "$SCCACHE_VERSION" ]; then + echo "Using baked sccache $current" + exit 0 + fi + fi + + case "$(uname -s)-$(uname -m)" in + Linux-x86_64) triple=x86_64-unknown-linux-musl ;; + Darwin-arm64) triple=aarch64-apple-darwin ;; + Darwin-x86_64) triple=x86_64-apple-darwin ;; + *) triple="" ;; + esac + + if [ -n "$triple" ]; then + url="https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-${triple}.tar.gz" + tmpdir="${RUNNER_TEMP:-/tmp}/sccache-${SCCACHE_VERSION}" + bindir="${HOME}/.local/bin" + rm -rf "$tmpdir" + mkdir -p "$tmpdir" "$bindir" + curl -fsSL "$url" | tar -xz -C "$tmpdir" + install -m755 "$tmpdir"/sccache-v${SCCACHE_VERSION}-${triple}/sccache "$bindir/sccache" + echo "$bindir" >> "$GITHUB_PATH" + echo "Installed sccache $("$bindir/sccache" --version)" + exit 0 + fi + + cargo install --locked sccache --version "$SCCACHE_VERSION" + echo "Installed sccache $(sccache --version)" diff --git a/.github/actions/ensure-zig/action.yml b/.github/actions/ensure-zig/action.yml new file mode 100644 index 000000000..64afd5d82 --- /dev/null +++ b/.github/actions/ensure-zig/action.yml @@ -0,0 +1,40 @@ +name: "ensure zig" +description: Ensure a pinned Zig binary is on PATH. + +inputs: + version: + required: true + description: Zig release version + +runs: + using: composite + steps: + - shell: bash + env: + ZIG_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if command -v zig >/dev/null 2>&1 && [ "$(zig version)" = "$ZIG_VERSION" ]; then + echo "Using baked zig $ZIG_VERSION" + exit 0 + fi + + case "$(uname -s)-$(uname -m)" in + Linux-x86_64) archive="zig-x86_64-linux-${ZIG_VERSION}" ;; + Darwin-arm64) archive="zig-aarch64-macos-${ZIG_VERSION}" ;; + Darwin-x86_64) archive="zig-x86_64-macos-${ZIG_VERSION}" ;; + *) + echo "Unsupported zig host: $(uname -s)-$(uname -m)" >&2 + exit 1 + ;; + esac + + tmpdir="${RUNNER_TEMP:-/tmp}/zig-${ZIG_VERSION}" + bindir="${HOME}/.local/bin" + rm -rf "$tmpdir" + mkdir -p "$tmpdir" "$bindir" + curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/${archive}.tar.xz" -o "$tmpdir/zig.tar.xz" + tar -xJf "$tmpdir/zig.tar.xz" -C "$tmpdir" + install -m755 "$tmpdir/${archive}/zig" "$bindir/zig" + echo "$bindir" >> "$GITHUB_PATH" + echo "Installed zig $("$bindir/zig" version)" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca3698462..1a7f56b1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -212,7 +212,7 @@ jobs: - { variant: modern } steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/build-native + - uses: ./.github/actions/build-native-kata with: hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} platform: linux @@ -224,7 +224,7 @@ jobs: # Pre-warm the cross-platform native build cache on `main`, in addition to # building the artifacts that ship in releases. Skipped on main when # native_artifact_lookup already found a recent run with all artifacts intact. - native_cross_platform: + native_cross_platform_kata: name: "Native: ${{ matrix.platform }} ${{ matrix.arch }}" needs: [release_metadata, native_artifact_lookup] if: ${{ needs.release_metadata.outputs.is-release == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.native_artifact_lookup.outputs.cross-platform-run-id == '') }} @@ -233,9 +233,29 @@ jobs: matrix: include: - { os: omp-kata, platform: linux, arch: arm64, target: aarch64-unknown-linux-gnu } + - { os: omp-kata, platform: win32, arch: x64, target: x86_64-pc-windows-msvc, variant: baseline } + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-native-kata + with: + hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} + platform: ${{ matrix.platform }} + arch: ${{ matrix.arch }} + variant: ${{ matrix.variant }} + target: ${{ matrix.target }} + save_cache: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + native_cross_platform_macos: + name: "Native: ${{ matrix.platform }} ${{ matrix.arch }}" + needs: [release_metadata, native_artifact_lookup] + if: ${{ needs.release_metadata.outputs.is-release == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.native_artifact_lookup.outputs.cross-platform-run-id == '') }} + strategy: + fail-fast: false + matrix: + include: - { os: macos-15-intel, platform: darwin, arch: x64, variant: baseline } - { os: macos-14, platform: darwin, arch: arm64 } - - { os: omp-kata, platform: win32, arch: x64, target: x86_64-pc-windows-msvc, variant: baseline } runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -247,7 +267,6 @@ jobs: variant: ${{ matrix.variant }} target: ${{ matrix.target }} save_cache: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - test_workspace: name: Test TS workspace fast runs-on: omp-kata @@ -464,31 +483,12 @@ jobs: runs-on: omp-kata steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@nightly + - uses: ./.github/actions/ensure-rust-toolchain with: toolchain: nightly-2026-04-29 - - name: Decide Rust artifact cache - id: rust_artifact_cache - shell: bash - run: | - if [ -n "${SCCACHE_BUCKET:-}" ]; then - echo "use_rust_cache=false" >> "$GITHUB_OUTPUT" - echo "Rust target cache: skipped on shared-sccache runners" - else - echo "use_rust_cache=true" >> "$GITHUB_OUTPUT" - echo "Rust target cache: GitHub Actions" - fi - - uses: Swatinem/rust-cache@v2 - if: steps.rust_artifact_cache.outputs.use_rust_cache == 'true' + - uses: ./.github/actions/ensure-sccache with: - shared-key: install-methods-linux-x64 - cache-on-failure: true - save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - cache-workspace-crates: true - # On omp-kata, shared S3-backed sccache already carries the compile reuse - # and avoids a second GitHub-cache handshake for target/. - - name: Setup sccache - uses: mozilla-actions/sccache-action@v0.0.10 + version: "0.15.0" - name: Enable sccache for cargo # Conditional backend: self-hosted omp-kata injects a shared S3 # (RustFS) sccache via pod env; GitHub-hosted runners keep the GHA @@ -513,8 +513,9 @@ jobs: release_binary: name: "Release binary: ${{ matrix.target_id }}" if: ${{ needs.release_metadata.outputs.is-release == 'true' && !cancelled() && - needs.native_linux_x64.result == 'success' && needs.native_cross_platform.result == - 'success' && needs.test_workspace.result == 'success' && + needs.native_linux_x64.result == 'success' && needs.native_cross_platform_kata.result == + 'success' && needs.native_cross_platform_macos.result == 'success' && + needs.test_workspace.result == 'success' && needs.test_coding_agent_singleton.result == 'success' && needs.test_ts_native.result == 'success' && needs.test_coding_agent_ui.result == 'success' && @@ -522,7 +523,7 @@ jobs: needs.test_coding_agent_native.result == 'success' && needs.test_smoke.result == 'success' && needs.check.result == 'success' && needs.install_methods.result == 'success' }} - needs: [release_metadata, check, native_linux_x64, native_cross_platform, test_workspace, test_coding_agent_singleton, test_ts_native, test_coding_agent_ui, test_coding_agent_runtime, test_coding_agent_native, test_smoke, install_methods, native_artifact_lookup] + needs: [release_metadata, check, native_linux_x64, native_cross_platform_kata, native_cross_platform_macos, test_workspace, test_coding_agent_singleton, test_ts_native, test_coding_agent_ui, test_coding_agent_runtime, test_coding_agent_native, test_smoke, install_methods, native_artifact_lookup] strategy: fail-fast: false matrix: diff --git a/infra/docs/03-runner-image.md b/infra/docs/03-runner-image.md index 68aa420f3..374b273a6 100644 --- a/infra/docs/03-runner-image.md +++ b/infra/docs/03-runner-image.md @@ -26,15 +26,17 @@ All host commands below run on `` (the single k3s node) as root. The base `ghcr.io/actions/actions-runner:latest` is a clean Ubuntu 24.04 runner. On a normal (GitHub-hosted-style) runner, the CI workflow installs its system dependencies at the start of every job: the cairo/pango native stack for canvas -builds, `fd`/`ripgrep`/`imagemagick`, `bun`, and a pinned Rust nightly with the -cross targets. Inside a Kata microVM that is destroyed after a single job, paying -that apt/bun/rustup cost on **every** job is pure latency - the microVM starts -cold each time. +builds, `fd`/`ripgrep`/`imagemagick`, `bun`, `sccache`, Zig, the cargo-native +helper CLIs (`cargo-nextest`, `cargo-zigbuild`, `cargo-xwin`), and a pinned Rust +nightly with the cross targets/components. Inside a Kata microVM that is +destroyed after a single job, paying that apt/bun/rustup/tool-download cost on +**every** job is pure latency - the microVM starts cold each time. The preloaded image moves that work to build time. Every ephemeral runner then -starts with the toolchain already present: apt deps are not re-fetched, `bun` and -`cargo`/`rustc` are on `PATH`, and the pinned Rust toolchain is already the -default so target/component installs in CI become no-ops. +starts with the toolchain already present: apt deps are not re-fetched, `bun`, +`cargo`/`rustc`, `sccache`, `zig`, and the cargo helper CLIs are on `PATH`, and +the pinned Rust toolchain is already the default so target/component installs in +CI become no-ops. ### Stay in sync with `setup-system-deps` @@ -79,7 +81,8 @@ reproduced verbatim (it contains no secrets or redactable host identifiers; the # tool and release workflows expect it # - C/build toolchain the native + canvas builds need # - bun (system-wide, on PATH) -# - rust nightly toolchain (pinned) + clippy/rustfmt + linux-arm64/windows-msvc targets +# - sccache + Zig + cargo-nextest/cargo-zigbuild/cargo-xwin for native builds +# - rust nightly (pinned) + clippy/rustfmt/rust-analyzer + linux-arm64/windows-msvc targets # # Rebuild + reimport (see /root/omp-kata-runner.md) after bumping the ARGs below # or the apt set. Keep the apt set in sync with .github/actions/setup-system-deps. @@ -87,19 +90,23 @@ FROM ghcr.io/actions/actions-runner:latest ARG RUST_NIGHTLY=nightly-2026-04-29 ARG BUN_VERSION=1.3.14 +ARG SCCACHE_VERSION=0.15.0 +ARG ZIG_VERSION=0.16.0 USER root ENV DEBIAN_FRONTEND=noninteractive # Mirrors the "Install system deps" block in .github/workflows/ci.yml plus the -# C/build toolchain (native + canvas builds) and the GitHub CLI. The gh apt repo -# is added first so `gh` installs in the same apt transaction. +# native/cross toolchain (clang/lld/llvm), the baked cache/tooling binaries, and +# the GitHub CLI. The gh apt repo is added first so `gh` installs in the same apt +# transaction. RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg -o /usr/share/keyrings/githubcli-archive-keyring.gpg \ && chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list \ && apt-get update \ && apt-get install -y \ build-essential pkg-config curl ca-certificates git unzip xz-utils zstd gh \ + clang lld llvm \ libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \ fd-find ripgrep imagemagick \ && ln -sf "$(command -v fdfind)" /usr/local/bin/fd \ @@ -111,17 +118,34 @@ ENV BUN_INSTALL=/usr/local RUN curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}" \ && bun --version -# rust toolchain for the runner user; rustup default == pinned nightly so -# dtolnay/rust-toolchain@nightly and target/component adds are no-ops in CI. +# Pinned native-build helpers, system-wide. +RUN curl -fsSL "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \ + | tar -xz -C /tmp \ + && install -m755 "/tmp/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" /usr/local/bin/sccache \ + && rm -rf "/tmp/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl" +RUN curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz \ + && tar -xJf /tmp/zig.tar.xz -C /opt \ + && ln -sf "/opt/zig-x86_64-linux-${ZIG_VERSION}/zig" /usr/local/bin/zig \ + && rm -f /tmp/zig.tar.xz + +# rust toolchain + cargo helpers for the runner user; rustup default == pinned +# nightly so Rust setup becomes a no-op on the preloaded image. USER runner ENV RUSTUP_HOME=/home/runner/.rustup \ CARGO_HOME=/home/runner/.cargo \ PATH=/home/runner/.cargo/bin:/usr/local/bin:${PATH} RUN curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ | sh -s -- -y --default-toolchain "${RUST_NIGHTLY}" --profile minimal \ - && rustup component add clippy rustfmt \ + && rustup component add clippy rustfmt rust-analyzer \ && rustup target add aarch64-unknown-linux-gnu x86_64-pc-windows-msvc \ - && cargo --version && rustc --version + && cargo install --locked cargo-nextest cargo-zigbuild cargo-xwin \ + && cargo --version \ + && rustc --version \ + && sccache --version \ + && zig version \ + && cargo nextest --version \ + && cargo zigbuild --version \ + && cargo xwin --version ``` ### Stage-by-stage annotation @@ -152,9 +176,11 @@ In order: tool. - `apt-get install` pulls three groups: - **build toolchain / utilities:** `build-essential pkg-config curl - ca-certificates git unzip xz-utils zstd gh`. `build-essential` + `pkg-config` - are needed by the native and canvas builds; `zstd` is the codec the bun and - sccache cache tarballs use (see [04-arc-and-caching.md](./04-arc-and-caching.md)). + ca-certificates git unzip xz-utils zstd gh clang lld llvm`. + `build-essential` + `pkg-config` are needed by the native and canvas builds; + `zstd` is the codec the bun and sccache cache tarballs use (see + [04-arc-and-caching.md](./04-arc-and-caching.md)); `clang lld llvm` are the + MSVC-cross prerequisites that used to be apt-installed per job. - **canvas / cairo native stack:** `libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev` - the `-dev` headers the canvas/rsvg native modules compile against. @@ -174,16 +200,25 @@ In order: `bun-v${BUN_VERSION}`, and `bun --version` fails the build if the install is broken. +**Pinned native-build helpers (two root `RUN`s).** `sccache` is downloaded as a +version-pinned GitHub release tarball and installed to `/usr/local/bin`; Zig is +downloaded as the pinned release archive, unpacked under `/opt`, and symlinked +into `/usr/local/bin/zig`. Baking these two removes the per-job +`mozilla-actions/sccache-action` and `mlugg/setup-zig` downloads from the +self-hosted path. + **Rust toolchain (`USER runner` + rustup `RUN`).** The toolchain is installed as the **`runner` user** - the UID jobs execute as - so cargo/rustc are owned by and visible to the job without sudo. `RUSTUP_HOME`/`CARGO_HOME` are pinned under `/home/runner`, and `~/.cargo/bin` is prepended to `PATH`. rustup installs the pinned nightly as the **default toolchain** (`--profile minimal`), then adds the -`clippy` and `rustfmt` components and the `aarch64-unknown-linux-gnu` -(Linux arm64) and `x86_64-pc-windows-msvc` (Windows cross) targets. Because the -default toolchain already *is* the pinned nightly with these components/targets, -the corresponding `rustup` steps in CI become no-ops - the warm-start payoff. -`cargo --version && rustc --version` is the final build-time sanity check. +`clippy`, `rustfmt`, and `rust-analyzer` components plus the +`aarch64-unknown-linux-gnu` (Linux arm64) and `x86_64-pc-windows-msvc` (Windows +cross) targets. The same layer also `cargo install`s the Rust-native helper CLIs +`cargo-nextest`, `cargo-zigbuild`, and `cargo-xwin`, so the self-hosted native +build path no longer fetches those tools job-by-job. Because the default toolchain +already *is* the pinned nightly with these components/targets, the corresponding +Rust setup steps in CI become no-ops - the warm-start payoff. --- @@ -217,10 +252,10 @@ DOCKER_BUILDKIT=1 docker build -t "$IMAGE" -t omp-kata-runner:preloaded . echo "==> [2/5] verifying baked tools" docker run --rm --entrypoint bash "$IMAGE" -lc ' set -e - for b in gh fd rg magick bun cargo rustc pkg-config zstd; do + for b in gh fd rg magick bun cargo rustc pkg-config zstd clang lld sccache zig cargo-nextest cargo-zigbuild cargo-xwin; do command -v "$b" >/dev/null || { echo "MISSING: $b"; exit 1; } done - echo "tools OK | $(bun --version) | $(rustc --version) | gh $(gh --version | head -1 | cut -d" " -f3)" + echo "tools OK | bun $(bun --version) | rust $(rustc --version) | sccache $(sccache --version | awk '\''{print $2}'\'') | zig $(zig version) | gh $(gh --version | head -1 | cut -d\" \" -f3)" ' echo "==> [3/5] importing into k3s containerd (k8s.io namespace)" @@ -259,10 +294,10 @@ BuildKit + the docker layer cache make an unchanged rebuild near-instant. **[2/5] verify baked tools.** Runs the freshly built image with a bash entrypoint and asserts every expected binary is on `PATH` -(`gh fd rg magick bun cargo rustc pkg-config zstd`), failing the whole script if -any is missing, then prints the bun / rustc / gh versions. This catches a broken -apt set, missing shim, or bad toolchain pin **before** anything touches the -cluster. +(`gh fd rg magick bun cargo rustc pkg-config zstd clang lld sccache zig cargo-nextest cargo-zigbuild cargo-xwin`), +failing the whole script if any is missing, then prints the key version tuple +(bun / rust / sccache / zig / gh). This catches a broken apt set, missing shim, +or bad toolchain pin **before** anything touches the cluster. **[3/5] import into k3s containerd.** `docker save "$IMAGE" | k3s ctr -n k8s.io images import --platform linux/amd64 -` diff --git a/infra/reload-runner.sh b/infra/reload-runner.sh index bb6ded287..e51f41d62 100755 --- a/infra/reload-runner.sh +++ b/infra/reload-runner.sh @@ -56,10 +56,10 @@ DOCKER_BUILDKIT=1 docker build -t "$IMAGE" -t omp-kata-runner:preloaded . echo "==> [2/5] verifying baked tools" docker run --rm --entrypoint bash "$IMAGE" -lc ' set -e - for b in gh fd rg magick bun cargo rustc pkg-config zstd; do + for b in gh fd rg magick bun cargo rustc pkg-config zstd clang lld sccache zig cargo-nextest cargo-zigbuild cargo-xwin; do command -v "$b" >/dev/null || { echo "MISSING: $b"; exit 1; } done - echo "tools OK | $(bun --version) | $(rustc --version) | gh $(gh --version | head -1 | cut -d" " -f3)" + echo "tools OK | bun $(bun --version) | rust $(rustc --version) | sccache $(sccache --version | awk '\''{print $2}'\'') | zig $(zig version) | gh $(gh --version | head -1 | cut -d\" \" -f3)" ' echo "==> [3/5] importing into k3s containerd (k8s.io namespace)" diff --git a/infra/runner.Dockerfile b/infra/runner.Dockerfile index 6daa54481..7067fb61e 100644 --- a/infra/runner.Dockerfile +++ b/infra/runner.Dockerfile @@ -9,7 +9,8 @@ # tool and release workflows expect it # - C/build toolchain the native + canvas builds need # - bun (system-wide, on PATH) -# - rust nightly toolchain (pinned) + clippy/rustfmt + linux-arm64/windows-msvc targets +# - sccache + Zig + cargo-nextest/cargo-zigbuild/cargo-xwin for native builds +# - rust nightly (pinned) + clippy/rustfmt/rust-analyzer + linux-arm64/windows-msvc targets # # Rebuild + reimport (see /root/omp-kata-runner.md) after bumping the ARGs below # or the apt set. Keep the apt set in sync with .github/actions/setup-system-deps. @@ -17,19 +18,23 @@ FROM ghcr.io/actions/actions-runner:latest ARG RUST_NIGHTLY=nightly-2026-04-29 ARG BUN_VERSION=1.3.14 +ARG SCCACHE_VERSION=0.15.0 +ARG ZIG_VERSION=0.16.0 USER root ENV DEBIAN_FRONTEND=noninteractive # Mirrors the "Install system deps" block in .github/workflows/ci.yml plus the -# C/build toolchain (native + canvas builds) and the GitHub CLI. The gh apt repo -# is added first so `gh` installs in the same apt transaction. +# native/cross toolchain (clang/lld/llvm), the baked cache/tooling binaries, and +# the GitHub CLI. The gh apt repo is added first so `gh` installs in the same apt +# transaction. RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg -o /usr/share/keyrings/githubcli-archive-keyring.gpg \ && chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list \ && apt-get update \ && apt-get install -y \ build-essential pkg-config curl ca-certificates git unzip xz-utils zstd gh \ + clang lld llvm \ libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \ fd-find ripgrep imagemagick \ && ln -sf "$(command -v fdfind)" /usr/local/bin/fd \ @@ -41,14 +46,31 @@ ENV BUN_INSTALL=/usr/local RUN curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}" \ && bun --version -# rust toolchain for the runner user; rustup default == pinned nightly so -# dtolnay/rust-toolchain@nightly and target/component adds are no-ops in CI. +# Pinned native-build helpers, system-wide. +RUN curl -fsSL "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \ + | tar -xz -C /tmp \ + && install -m755 "/tmp/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" /usr/local/bin/sccache \ + && rm -rf "/tmp/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl" +RUN curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz \ + && tar -xJf /tmp/zig.tar.xz -C /opt \ + && ln -sf "/opt/zig-x86_64-linux-${ZIG_VERSION}/zig" /usr/local/bin/zig \ + && rm -f /tmp/zig.tar.xz + +# rust toolchain + cargo helpers for the runner user; rustup default == pinned +# nightly so Rust setup becomes a no-op on the preloaded image. USER runner ENV RUSTUP_HOME=/home/runner/.rustup \ CARGO_HOME=/home/runner/.cargo \ PATH=/home/runner/.cargo/bin:/usr/local/bin:${PATH} RUN curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ | sh -s -- -y --default-toolchain "${RUST_NIGHTLY}" --profile minimal \ - && rustup component add clippy rustfmt \ + && rustup component add clippy rustfmt rust-analyzer \ && rustup target add aarch64-unknown-linux-gnu x86_64-pc-windows-msvc \ - && cargo --version && rustc --version + && cargo install --locked cargo-nextest cargo-zigbuild cargo-xwin \ + && cargo --version \ + && rustc --version \ + && sccache --version \ + && zig version \ + && cargo nextest --version \ + && cargo zigbuild --version \ + && cargo xwin --version diff --git a/packages/mnemopi/test/embedding-model-reconcile.test.ts b/packages/mnemopi/test/embedding-model-reconcile.test.ts index 0da7e7cfe..ae03af7b9 100644 --- a/packages/mnemopi/test/embedding-model-reconcile.test.ts +++ b/packages/mnemopi/test/embedding-model-reconcile.test.ts @@ -8,9 +8,9 @@ import { Database } from "bun:sqlite"; import { describe, expect, it } from "bun:test"; -import { RUN_EMBEDDINGS } from "./setup"; import { initBeam } from "@oh-my-pi/pi-mnemopi/core/beam"; import { Mnemopi } from "@oh-my-pi/pi-mnemopi/core/memory"; +import { RUN_EMBEDDINGS } from "./setup"; const OLD_MODEL = "BAAI/bge-small-en-v1.5"; const NEW_MODEL = "intfloat/multilingual-e5-large"; diff --git a/packages/mnemopi/test/embeddings-multilingual.test.ts b/packages/mnemopi/test/embeddings-multilingual.test.ts index 29bc36140..da3e5eabd 100644 --- a/packages/mnemopi/test/embeddings-multilingual.test.ts +++ b/packages/mnemopi/test/embeddings-multilingual.test.ts @@ -1,5 +1,4 @@ import { describe, expect, it } from "bun:test"; -import { RUN_EMBEDDINGS } from "./setup"; import { cosineSimilarity, embed, @@ -8,6 +7,7 @@ import { resetEmbeddingProviderForTests, setEmbeddingProviderForTests, } from "@oh-my-pi/pi-mnemopi/core/embeddings"; +import { RUN_EMBEDDINGS } from "./setup"; function withEnvValue(key: string, value: string | undefined, fn: () => T): T { const previous = process.env[key]; diff --git a/packages/mnemopi/test/issue-1832-embedding-population.test.ts b/packages/mnemopi/test/issue-1832-embedding-population.test.ts index 4f40f5a56..7df3fd700 100644 --- a/packages/mnemopi/test/issue-1832-embedding-population.test.ts +++ b/packages/mnemopi/test/issue-1832-embedding-population.test.ts @@ -16,7 +16,6 @@ import { describe, expect, it } from "bun:test"; import { randomBytes } from "node:crypto"; import { rmSync } from "node:fs"; import { tmpdir } from "node:os"; -import { RUN_EMBEDDINGS } from "./setup"; import { cmdRemember } from "@oh-my-pi/pi-mnemopi/cli"; import { BeamMemory } from "@oh-my-pi/pi-mnemopi/core/beam"; import { Mnemopi } from "@oh-my-pi/pi-mnemopi/core/memory"; @@ -24,6 +23,7 @@ import { type ResolvedMnemopiRuntimeOptions, withMnemopiRuntimeOptions, } from "@oh-my-pi/pi-mnemopi/core/runtime-options"; +import { RUN_EMBEDDINGS } from "./setup"; interface EmbeddingRow { readonly memory_id: string; diff --git a/packages/mnemopi/test/optional-embeddings.test.ts b/packages/mnemopi/test/optional-embeddings.test.ts index c3f1a24b0..46c710598 100644 --- a/packages/mnemopi/test/optional-embeddings.test.ts +++ b/packages/mnemopi/test/optional-embeddings.test.ts @@ -1,6 +1,4 @@ import { afterEach, describe, expect, it } from "bun:test"; -import { getFastembedCacheDir } from "@oh-my-pi/pi-utils"; -import { RUN_EMBEDDINGS } from "./setup"; import { available, embed, @@ -12,7 +10,9 @@ import { } from "@oh-my-pi/pi-mnemopi/core/embeddings"; import { Mnemopi } from "@oh-my-pi/pi-mnemopi/core/memory"; import { withMnemopiRuntimeOptions } from "@oh-my-pi/pi-mnemopi/core/runtime-options"; +import { getFastembedCacheDir } from "@oh-my-pi/pi-utils"; import packageJson from "../package.json" with { type: "json" }; +import { RUN_EMBEDDINGS } from "./setup"; const ENV_KEYS = [ "NODE_ENV", diff --git a/packages/mnemopi/test/setup.ts b/packages/mnemopi/test/setup.ts index 1f87e484b..368c996c5 100644 --- a/packages/mnemopi/test/setup.ts +++ b/packages/mnemopi/test/setup.ts @@ -64,7 +64,6 @@ class FakeLocalLlmBackend implements LlmBackend { } export const RUN_EMBEDDINGS = Bun.env.EMBEDDINGS === "1"; - beforeEach(() => { // Real embeddings (fastembed + onnxruntime-node, ~270MB peers) install on // demand via `bun install` on first use. Default the suite to the lightweight