can1357
59619623e1
- Add scripts/ci-target-cache.ts to snapshot and restore Cargo target directories to S3 storage on omp-kata runners. - Update .github/actions/build-native/action.yml to support target cache restoration, saving, and compiler launcher configurations. - Add skip_validation input in GitHub workflow actions to bypass clippy and Rust test checks on release runs.
374 lines
18 KiB
YAML
374 lines
18 KiB
YAML
name: Build native addon
|
|
description: >
|
|
Build the pi_natives cdylib for one platform/arch/variant and upload it as a
|
|
hash-tagged artifact. Self-detects the runner via $SCCACHE_BUCKET (injected
|
|
only on the self-hosted omp-kata pods): on-infra it uses the image's baked
|
|
toolchains + the RustFS-backed sccache; on GitHub-hosted runners it installs
|
|
the toolchains and uses Swatinem target/ cache + the GitHub Actions sccache
|
|
backend. PRs run on GitHub-hosted runners, so the on-infra path only ever
|
|
serves trusted push/main + release builds.
|
|
|
|
inputs:
|
|
hash:
|
|
description: Rust source hash used in the artifact name
|
|
required: true
|
|
platform:
|
|
description: Target platform (linux, darwin, win32)
|
|
required: true
|
|
arch:
|
|
description: Target arch (x64, arm64)
|
|
required: true
|
|
variant:
|
|
description: Optional build variant (baseline, modern); required for native x64 builds.
|
|
required: false
|
|
default: ""
|
|
target:
|
|
description: Optional rustc target triple for cross-compilation
|
|
required: false
|
|
default: ""
|
|
glibc:
|
|
description: >
|
|
Optional glibc floor (e.g. "2.17") for linux-gnu builds. Routes the build
|
|
through cargo-zigbuild against that floor without affecting the rustup
|
|
target or the host-arch native test steps. Combine with `target` to pin a
|
|
cross-arch linux build, or set alone for a host-arch (x64) linux build.
|
|
required: false
|
|
default: ""
|
|
libc:
|
|
description: Optional Linux libc artifact qualifier (for example, musl)
|
|
required: false
|
|
default: ""
|
|
rust_checks:
|
|
description: Run clippy/rustfmt checks (only one matrix entry should set this)
|
|
required: false
|
|
default: "false"
|
|
skip_validation:
|
|
description: >
|
|
Skip clippy/rustfmt and the Rust test suite. Set on release runs: the
|
|
version-bump commit only changes version strings, and the tagged
|
|
content's Rust code already passed validation on its main-push run.
|
|
required: false
|
|
default: "false"
|
|
save_cache:
|
|
description: >
|
|
Whether to write build caches: Swatinem/rust-cache on GitHub-hosted
|
|
runners, the RustFS target/ snapshot on omp-kata.
|
|
required: false
|
|
default: "false"
|
|
|
|
runs:
|
|
using: composite
|
|
steps:
|
|
- name: Detect runner environment
|
|
id: detect
|
|
shell: bash
|
|
run: |
|
|
# $SCCACHE_BUCKET is injected only on the self-hosted omp-kata runner
|
|
# pods (envFrom sccache-s3); its presence is the repo's single
|
|
# "on can.internal infra?" signal. On-infra: baked toolchains, RustFS
|
|
# sccache, no GitHub target/ cache. Off-infra (GitHub-hosted): install
|
|
# toolchains, Swatinem target/ cache, GitHub Actions sccache backend.
|
|
if [ -n "${SCCACHE_BUCKET:-}" ]; then
|
|
echo "on_infra=true" >> "$GITHUB_OUTPUT"
|
|
echo "runner: self-hosted omp-kata (baked tools + RustFS sccache)"
|
|
else
|
|
echo "on_infra=false" >> "$GITHUB_OUTPUT"
|
|
echo "runner: GitHub-hosted (install tools + Swatinem cache + GHA sccache)"
|
|
fi
|
|
|
|
- name: Resolve build targets
|
|
id: resolve
|
|
shell: bash
|
|
env:
|
|
TARGET: ${{ inputs.target }}
|
|
GLIBC: ${{ inputs.glibc }}
|
|
ARCH: ${{ inputs.arch }}
|
|
run: |
|
|
set -euo pipefail
|
|
# `cross_target` is what the napi build feeds cargo: cargo-zigbuild reads
|
|
# the glibc floor as a `.<major>.<minor>` triple suffix. `bare_target` is
|
|
# what rustup needs — never glibc-suffixed (rustup rejects the suffix)
|
|
# and empty for host-arch builds whose target is already installed.
|
|
base="$TARGET"
|
|
if [ -z "$base" ]; then
|
|
case "$ARCH" in
|
|
x64) base="x86_64-unknown-linux-gnu" ;;
|
|
arm64) base="aarch64-unknown-linux-gnu" ;;
|
|
esac
|
|
fi
|
|
cross_target=""
|
|
if [ -n "$GLIBC" ]; then
|
|
if [ -z "$base" ]; then
|
|
echo "::error::glibc floor '$GLIBC' set but no linux-gnu base triple for arch '$ARCH'"
|
|
exit 1
|
|
fi
|
|
cross_target="${base}.${GLIBC}"
|
|
elif [ -n "$TARGET" ]; then
|
|
cross_target="$TARGET"
|
|
fi
|
|
bare_target="${TARGET%%.*}"
|
|
{
|
|
echo "cross_target=$cross_target"
|
|
echo "bare_target=$bare_target"
|
|
} >> "$GITHUB_OUTPUT"
|
|
echo "Resolved cross_target='$cross_target' bare_target='$bare_target'"
|
|
|
|
# --- Rust toolchain -----------------------------------------------------
|
|
- name: Ensure baked Rust toolchain (omp-kata)
|
|
if: steps.detect.outputs.on_infra == 'true'
|
|
uses: ./.github/actions/ensure-rust-toolchain
|
|
with:
|
|
toolchain: nightly-2026-04-29
|
|
components: ${{ inputs.rust_checks == 'true' && 'clippy,rustfmt' || '' }}
|
|
target: ${{ steps.resolve.outputs.bare_target }}
|
|
- name: Install Rust toolchain (GitHub-hosted)
|
|
if: steps.detect.outputs.on_infra == 'false'
|
|
uses: dtolnay/rust-toolchain@nightly
|
|
with:
|
|
toolchain: nightly-2026-04-29
|
|
components: ${{ inputs.rust_checks == 'true' && 'clippy, rustfmt' || '' }}
|
|
targets: ${{ steps.resolve.outputs.bare_target }}
|
|
- name: Install Linux build prerequisites (GitHub-hosted)
|
|
if: steps.detect.outputs.on_infra == 'false' && runner.os == 'Linux'
|
|
shell: bash
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y build-essential
|
|
# audiopus_sys builds bundled libopus via CMake (Ninja generator for MSVC
|
|
# cross); baked into the omp-kata image and GitHub-hosted images, so this
|
|
# only self-heals runners that predate the bake.
|
|
- uses: ./.github/actions/ensure-cmake
|
|
- name: Prepend rustup toolchain bin to PATH (GitHub-hosted)
|
|
if: steps.detect.outputs.on_infra == 'false'
|
|
shell: bash
|
|
run: |
|
|
# Homebrew on macOS runners ships rustup-init with shadow proxies for
|
|
# `cargo`/`rustc`/etc. that error out as the installer ("unexpected
|
|
# argument 'metadata' found"). Force the real toolchain binaries to win
|
|
# on PATH. (ensure-rust-toolchain already does this on omp-kata.)
|
|
toolchain_bin="$(dirname "$(rustup which cargo)")"
|
|
echo "$toolchain_bin" >> "$GITHUB_PATH"
|
|
echo "Prepended $toolchain_bin to PATH"
|
|
|
|
# --- Rust flags (shared) ------------------------------------------------
|
|
- name: Configure native Rust flags
|
|
if: inputs.target == '' || inputs.arch == 'x64'
|
|
shell: bash
|
|
env:
|
|
TARGET_ARCH: ${{ inputs.arch }}
|
|
TARGET_VARIANT: ${{ inputs.variant }}
|
|
run: |
|
|
case "$TARGET_ARCH:$TARGET_VARIANT" in
|
|
x64:modern)
|
|
rustflags="-C target-cpu=x86-64-v3"
|
|
;;
|
|
x64:baseline)
|
|
rustflags="-C target-cpu=x86-64-v2"
|
|
;;
|
|
x64:*)
|
|
echo "::error::x64 native builds require variant=modern or variant=baseline"
|
|
exit 1
|
|
;;
|
|
*)
|
|
if [ -n "${RUSTFLAGS:-}" ]; then
|
|
echo "Using caller-provided RUSTFLAGS=$RUSTFLAGS"
|
|
exit 0
|
|
fi
|
|
# Non-x64 native builds (darwin arm64) keep the target's default
|
|
# CPU features. `-C target-cpu=native` both baked the CI host's
|
|
# CPU features into shipped artifacts and trips ring 0.17's
|
|
# aarch64-apple const assertion (CAPS_STATIC == MIN_STATIC_FEATURES)
|
|
# once extra static features are enabled.
|
|
echo "Using default target CPU features (no RUSTFLAGS)"
|
|
exit 0
|
|
;;
|
|
esac
|
|
|
|
echo "RUSTFLAGS=$rustflags" >> "$GITHUB_ENV"
|
|
echo "Configured RUSTFLAGS=$rustflags"
|
|
|
|
# --- target/ cache (GitHub-hosted only) ---------------------------------
|
|
# Swatinem keys target/ off its restore-time environment, so it must run
|
|
# after RUSTFLAGS is set: if x64 target-cpu were only selected later, cargo
|
|
# would invalidate the restored target/ while rust-cache saw an exact key
|
|
# and refused to save the rebuilt artifacts (macOS x64 baseline rebuilds
|
|
# forever). The native source hash is in the shared key too: rust-cache's
|
|
# lockfile scan misses the workspace-root Cargo.toml version Cargo
|
|
# fingerprints, so a version bump could otherwise get an exact hit for
|
|
# artifacts Cargo must rebuild. On omp-kata the reuse layers are the
|
|
# mounted Cargo registry, RustFS sccache, and the RustFS target/ snapshot
|
|
# (see "Restore target/ cache" below), so Swatinem stays GitHub-only.
|
|
- name: Cache Rust target/ (GitHub-hosted)
|
|
if: steps.detect.outputs.on_infra == 'false'
|
|
uses: Swatinem/rust-cache@v2
|
|
with:
|
|
shared-key: native-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }}-h${{ inputs.hash }}
|
|
cache-on-failure: true
|
|
save-if: ${{ inputs.save_cache == 'true' }}
|
|
cache-workspace-crates: true
|
|
|
|
# --- sccache ------------------------------------------------------------
|
|
- name: Ensure baked sccache (omp-kata)
|
|
if: steps.detect.outputs.on_infra == 'true'
|
|
uses: ./.github/actions/ensure-sccache
|
|
with:
|
|
version: "0.15.0"
|
|
- name: Setup sccache (GitHub-hosted)
|
|
if: steps.detect.outputs.on_infra == 'false'
|
|
uses: mozilla-actions/sccache-action@v0.0.10
|
|
- name: Enable sccache for cargo
|
|
# CARGO_INCREMENTAL=0 is required: sccache silently skips caching when
|
|
# incremental is enabled, turning the wrapper into a no-op. The backend is
|
|
# conditional: omp-kata reads the shared S3 (RustFS) config from the
|
|
# inherited pod env; GitHub-hosted runners use the GHA cache backend.
|
|
shell: bash
|
|
env:
|
|
CROSS_TARGET: ${{ steps.resolve.outputs.cross_target }}
|
|
run: |
|
|
{
|
|
echo "RUSTC_WRAPPER=sccache"
|
|
echo "CARGO_INCREMENTAL=0"
|
|
} >> "$GITHUB_ENV"
|
|
# Route CMake-built C (audiopus_sys' bundled opus) through sccache
|
|
# too — build scripts bypass RUSTC_WRAPPER. Non-cross builds only:
|
|
# cross builds compile C with zig cc / clang-cl wrapper scripts that
|
|
# sccache may fail to classify, which would hard-fail the compile.
|
|
if [ -z "$CROSS_TARGET" ]; then
|
|
{
|
|
echo "CMAKE_C_COMPILER_LAUNCHER=sccache"
|
|
echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache"
|
|
} >> "$GITHUB_ENV"
|
|
fi
|
|
if [ -n "${SCCACHE_BUCKET:-}" ]; then
|
|
echo "sccache backend: shared S3 ($SCCACHE_BUCKET @ $SCCACHE_ENDPOINT)"
|
|
else
|
|
echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
|
|
echo "sccache backend: GitHub Actions cache"
|
|
fi
|
|
|
|
# --- cargo-nextest (native test runner; non-cross builds only) ----------
|
|
- name: Ensure baked cargo-nextest (omp-kata)
|
|
if: steps.detect.outputs.on_infra == 'true' && inputs.target == ''
|
|
uses: ./.github/actions/ensure-cargo-tool
|
|
with:
|
|
binary: cargo-nextest
|
|
crate: cargo-nextest
|
|
- name: Install cargo-nextest (GitHub-hosted)
|
|
if: steps.detect.outputs.on_infra == 'false' && inputs.target == ''
|
|
uses: taiki-e/install-action@v2
|
|
with:
|
|
tool: nextest
|
|
|
|
- uses: ./.github/actions/bun-install
|
|
|
|
# --- Cross-compile toolchains -------------------------------------------
|
|
# Non-MSVC targets (e.g. aarch64-unknown-linux-gnu) build with
|
|
# cargo-zigbuild (needs zig); MSVC targets (e.g. x86_64-pc-windows-msvc)
|
|
# build with cargo-xwin (needs clang/lld/llvm). The napi CLI's
|
|
# --cross-compile flag picks the backend; we just install what it needs.
|
|
# Cross builds only run on push/main + release (omp-kata), so the
|
|
# GitHub-hosted cross branches exist for portability and never fire here.
|
|
- name: Ensure baked zig (omp-kata, non-MSVC cross)
|
|
if: steps.detect.outputs.on_infra == 'true' && steps.resolve.outputs.cross_target != '' && !endsWith(steps.resolve.outputs.cross_target, '-msvc')
|
|
uses: ./.github/actions/ensure-zig
|
|
with:
|
|
version: "0.16.0"
|
|
- name: Setup zig (GitHub-hosted, non-MSVC cross)
|
|
if: steps.detect.outputs.on_infra == 'false' && steps.resolve.outputs.cross_target != '' && !endsWith(steps.resolve.outputs.cross_target, '-msvc')
|
|
uses: mlugg/setup-zig@v2
|
|
with:
|
|
version: 0.16.0
|
|
- name: Ensure baked cargo-zigbuild (omp-kata, non-MSVC cross)
|
|
if: steps.detect.outputs.on_infra == 'true' && steps.resolve.outputs.cross_target != '' && !endsWith(steps.resolve.outputs.cross_target, '-msvc')
|
|
uses: ./.github/actions/ensure-cargo-tool
|
|
with:
|
|
binary: cargo-zigbuild
|
|
crate: cargo-zigbuild
|
|
- name: Install cargo-zigbuild (GitHub-hosted, non-MSVC cross)
|
|
if: steps.detect.outputs.on_infra == 'false' && steps.resolve.outputs.cross_target != '' && !endsWith(steps.resolve.outputs.cross_target, '-msvc')
|
|
uses: taiki-e/install-action@v2
|
|
with:
|
|
tool: cargo-zigbuild
|
|
- name: Install LLVM tooling (GitHub-hosted, MSVC cross)
|
|
if: steps.detect.outputs.on_infra == 'false' && endsWith(steps.resolve.outputs.cross_target, '-msvc')
|
|
shell: bash
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y clang lld llvm
|
|
- name: Ensure baked cargo-xwin (omp-kata, MSVC cross)
|
|
if: steps.detect.outputs.on_infra == 'true' && endsWith(steps.resolve.outputs.cross_target, '-msvc')
|
|
uses: ./.github/actions/ensure-cargo-tool
|
|
with:
|
|
binary: cargo-xwin
|
|
crate: cargo-xwin
|
|
- name: Install cargo-xwin (GitHub-hosted, MSVC cross)
|
|
if: steps.detect.outputs.on_infra == 'false' && endsWith(steps.resolve.outputs.cross_target, '-msvc')
|
|
uses: taiki-e/install-action@v2
|
|
with:
|
|
tool: cargo-xwin
|
|
- name: Cache cargo-xwin Windows SDK
|
|
if: endsWith(steps.resolve.outputs.cross_target, '-msvc')
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/cargo-xwin
|
|
key: cargo-xwin-${{ runner.os }}-v1
|
|
- name: Accept xwin license
|
|
if: endsWith(steps.resolve.outputs.cross_target, '-msvc')
|
|
shell: bash
|
|
run: echo "XWIN_ACCEPT_LICENSE=1" >> "$GITHUB_ENV"
|
|
|
|
# --- target/ cache (omp-kata) --------------------------------------------
|
|
# sccache only covers rustc invocations; build-script outputs (57
|
|
# tree-sitter grammar C compiles, bundled opus via CMake, ring asm) and
|
|
# cargo's fingerprint/link work bypass it. Snapshot target/ to the same
|
|
# RustFS S3 bucket, keyed per platform/libc/arch/variant + toolchain and
|
|
# overwritten on each save so storage stays bounded at one snapshot per
|
|
# key. GitHub-hosted runners get the same effect from Swatinem above.
|
|
- name: Restore target/ cache (omp-kata)
|
|
if: steps.detect.outputs.on_infra == 'true'
|
|
shell: bash
|
|
env:
|
|
TARGET_CACHE_KEY: native-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }}
|
|
run: bun scripts/ci-target-cache.ts restore "$TARGET_CACHE_KEY"
|
|
|
|
# --- Checks, build, upload (shared) -------------------------------------
|
|
- name: Rust checks
|
|
if: inputs.rust_checks == 'true' && inputs.skip_validation != 'true'
|
|
shell: bash
|
|
run: bun run check:rs
|
|
- name: Test workspace (Rust)
|
|
# macOS has no `#[cfg(target_os = "macos")]` tests in the workspace, and
|
|
# Windows-only tests are no longer exercised in CI (win32-x64 cross-builds
|
|
# on Linux). Skipping the duplicate Linux runs on macOS saves ~10 min of
|
|
# parallel runner time.
|
|
if: inputs.target == '' && inputs.platform != 'darwin' && inputs.skip_validation != 'true'
|
|
shell: bash
|
|
run: bun run test:rs
|
|
- name: Build native addon(s)
|
|
shell: bash
|
|
env:
|
|
CROSS_TARGET: ${{ steps.resolve.outputs.cross_target }}
|
|
TARGET_PLATFORM: ${{ inputs.platform }}
|
|
TARGET_ARCH: ${{ inputs.arch }}
|
|
TARGET_VARIANTS: ${{ inputs.variant }}
|
|
run: bun run ci:build:native
|
|
- name: sccache stats
|
|
shell: bash
|
|
run: sccache --show-stats || true
|
|
- name: Upload native addon(s)
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: pi-natives-${{ inputs.platform }}-${{ inputs.libc && format('{0}-', inputs.libc) || '' }}${{ inputs.arch }}${{ inputs.variant && format('-{0}', inputs.variant) || '' }}-h${{ inputs.hash }}
|
|
path: packages/natives/native/pi_natives.${{ inputs.platform }}-${{ inputs.arch }}*.node
|
|
if-no-files-found: error
|
|
# Explicit so the native_artifact_lookup canary keeps working even if
|
|
# org defaults shift; bump if Rust source ever stays stable for >90 days
|
|
# of main pushes and you want to avoid rebuilds.
|
|
retention-days: 90
|
|
- name: Save target/ cache (omp-kata)
|
|
if: steps.detect.outputs.on_infra == 'true' && inputs.save_cache == 'true'
|
|
shell: bash
|
|
env:
|
|
TARGET_CACHE_KEY: native-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }}
|
|
run: bun scripts/ci-target-cache.ts save "$TARGET_CACHE_KEY"
|