ci: added CI workflows and actions for pinned toolchain-native builds
- Added composite GitHub actions to ensure rust toolchains and cargo helpers. - Added a kata-native build action with variant checks and platform artifact uploads. - Reworked CI matrices to split native cross-platform jobs and gate releases accordingly. - Updated runner bootstrap and image to preinstall pinned build tools for CI consistency.
This commit is contained in:
@@ -0,0 +1,134 @@
|
||||
name: Build native addon (omp-kata)
|
||||
description: >
|
||||
Build the pi_natives cdylib on the preloaded omp-kata runner image, using
|
||||
baked toolchains and the shared RustFS-backed sccache instead of per-job tool
|
||||
setup downloads.
|
||||
|
||||
inputs:
|
||||
hash:
|
||||
description: Rust source hash used in the artifact name
|
||||
required: true
|
||||
platform:
|
||||
description: Target platform (linux, darwin, win32)
|
||||
required: true
|
||||
arch:
|
||||
description: Target arch (x64, arm64)
|
||||
required: true
|
||||
variant:
|
||||
description: Optional build variant (baseline, modern); required for native x64 builds.
|
||||
required: false
|
||||
default: ""
|
||||
target:
|
||||
description: Optional rustc target triple for cross-compilation
|
||||
required: false
|
||||
default: ""
|
||||
rust_checks:
|
||||
description: Run clippy/rustfmt checks (only one matrix entry should set this)
|
||||
required: false
|
||||
default: "false"
|
||||
save_cache:
|
||||
description: Kept for interface parity with the GitHub-hosted action; unused here.
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: ./.github/actions/ensure-rust-toolchain
|
||||
with:
|
||||
toolchain: nightly-2026-04-29
|
||||
components: ${{ inputs.rust_checks == 'true' && 'clippy,rustfmt' || '' }}
|
||||
target: ${{ inputs.target }}
|
||||
- name: Configure native Rust flags
|
||||
if: inputs.target == ''
|
||||
shell: bash
|
||||
env:
|
||||
TARGET_ARCH: ${{ inputs.arch }}
|
||||
TARGET_VARIANT: ${{ inputs.variant }}
|
||||
run: |
|
||||
case "$TARGET_ARCH:$TARGET_VARIANT" in
|
||||
x64:modern)
|
||||
rustflags="-C target-cpu=x86-64-v3"
|
||||
;;
|
||||
x64:baseline)
|
||||
rustflags="-C target-cpu=x86-64-v2"
|
||||
;;
|
||||
x64:*)
|
||||
echo "::error::x64 native builds require variant=modern or variant=baseline"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
if [ -n "${RUSTFLAGS:-}" ]; then
|
||||
echo "Using caller-provided RUSTFLAGS=$RUSTFLAGS"
|
||||
exit 0
|
||||
fi
|
||||
rustflags="-C target-cpu=native"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "RUSTFLAGS=$rustflags" >> "$GITHUB_ENV"
|
||||
echo "Configured RUSTFLAGS=$rustflags"
|
||||
- uses: ./.github/actions/ensure-sccache
|
||||
with:
|
||||
version: "0.15.0"
|
||||
- name: Enable sccache for cargo
|
||||
shell: bash
|
||||
run: |
|
||||
{
|
||||
echo "RUSTC_WRAPPER=sccache"
|
||||
echo "CARGO_INCREMENTAL=0"
|
||||
} >> "$GITHUB_ENV"
|
||||
echo "sccache backend: shared S3 ($SCCACHE_BUCKET @ $SCCACHE_ENDPOINT)"
|
||||
- uses: ./.github/actions/ensure-cargo-tool
|
||||
if: inputs.target == ''
|
||||
with:
|
||||
binary: cargo-nextest
|
||||
crate: cargo-nextest
|
||||
- uses: ./.github/actions/bun-install
|
||||
- uses: ./.github/actions/ensure-zig
|
||||
if: inputs.target != '' && !endsWith(inputs.target, '-msvc')
|
||||
with:
|
||||
version: "0.16.0"
|
||||
- uses: ./.github/actions/ensure-cargo-tool
|
||||
if: inputs.target != '' && !endsWith(inputs.target, '-msvc')
|
||||
with:
|
||||
binary: cargo-zigbuild
|
||||
crate: cargo-zigbuild
|
||||
- uses: ./.github/actions/ensure-cargo-tool
|
||||
if: endsWith(inputs.target, '-msvc')
|
||||
with:
|
||||
binary: cargo-xwin
|
||||
crate: cargo-xwin
|
||||
- name: Cache cargo-xwin Windows SDK
|
||||
if: endsWith(inputs.target, '-msvc')
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/cargo-xwin
|
||||
key: cargo-xwin-${{ runner.os }}-v1
|
||||
- name: Accept xwin license
|
||||
if: endsWith(inputs.target, '-msvc')
|
||||
shell: bash
|
||||
run: echo "XWIN_ACCEPT_LICENSE=1" >> "$GITHUB_ENV"
|
||||
- name: Rust checks
|
||||
if: inputs.rust_checks == 'true'
|
||||
shell: bash
|
||||
run: bun run check:rs
|
||||
- name: Test workspace (Rust)
|
||||
if: inputs.target == '' && inputs.platform != 'darwin'
|
||||
shell: bash
|
||||
run: bun run test:rs
|
||||
- name: Build native addon(s)
|
||||
shell: bash
|
||||
env:
|
||||
CROSS_TARGET: ${{ inputs.target }}
|
||||
TARGET_PLATFORM: ${{ inputs.platform }}
|
||||
TARGET_ARCH: ${{ inputs.arch }}
|
||||
TARGET_VARIANTS: ${{ inputs.variant }}
|
||||
run: bun run ci:build:native
|
||||
- name: Upload native addon(s)
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: pi-natives-${{ inputs.platform }}-${{ inputs.arch }}${{ inputs.variant && format('-{0}', inputs.variant) || '' }}-h${{ inputs.hash }}
|
||||
path: packages/natives/native/pi_natives.${{ inputs.platform }}-${{ inputs.arch }}*.node
|
||||
if-no-files-found: error
|
||||
retention-days: 90
|
||||
@@ -0,0 +1,38 @@
|
||||
name: "ensure cargo tool"
|
||||
description: Ensure a cargo-installed CLI is present.
|
||||
|
||||
inputs:
|
||||
binary:
|
||||
required: true
|
||||
description: Binary name expected on PATH
|
||||
crate:
|
||||
required: false
|
||||
default: ""
|
||||
description: Crate name to cargo install; defaults to the binary name
|
||||
version:
|
||||
required: false
|
||||
default: ""
|
||||
description: Optional crate version
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- shell: bash
|
||||
env:
|
||||
BINARY: ${{ inputs.binary }}
|
||||
CRATE: ${{ inputs.crate }}
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if command -v "$BINARY" >/dev/null 2>&1; then
|
||||
echo "Using baked cargo tool: $BINARY"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
crate="${CRATE:-$BINARY}"
|
||||
install_args=(install --locked "$crate")
|
||||
if [ -n "$VERSION" ]; then
|
||||
install_args+=(--version "$VERSION")
|
||||
fi
|
||||
cargo "${install_args[@]}"
|
||||
echo "Installed cargo tool: $BINARY"
|
||||
@@ -0,0 +1,59 @@
|
||||
name: "ensure rust toolchain"
|
||||
description: >
|
||||
Ensure a pinned rustup toolchain, optional components, and an optional target
|
||||
are present, then prepend the real toolchain bin dir to PATH.
|
||||
|
||||
inputs:
|
||||
toolchain:
|
||||
required: true
|
||||
description: Rust toolchain name (for example nightly-2026-04-29)
|
||||
components:
|
||||
required: false
|
||||
default: ""
|
||||
description: Optional comma-separated rustup components
|
||||
target:
|
||||
required: false
|
||||
default: ""
|
||||
description: Optional rustup target triple
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- shell: bash
|
||||
env:
|
||||
TOOLCHAIN: ${{ inputs.toolchain }}
|
||||
COMPONENTS: ${{ inputs.components }}
|
||||
TARGET: ${{ inputs.target }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! command -v rustup >/dev/null 2>&1; then
|
||||
curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \
|
||||
| sh -s -- -y --default-toolchain "$TOOLCHAIN" --profile minimal
|
||||
fi
|
||||
|
||||
if ! rustc +"$TOOLCHAIN" --version >/dev/null 2>&1; then
|
||||
rustup toolchain install "$TOOLCHAIN" --profile minimal --no-self-update
|
||||
fi
|
||||
rustup default "$TOOLCHAIN"
|
||||
|
||||
missing_components=()
|
||||
if [ -n "$COMPONENTS" ]; then
|
||||
IFS=',' read -r -a wanted_components <<< "$COMPONENTS"
|
||||
for component in "${wanted_components[@]}"; do
|
||||
[ -z "$component" ] && continue
|
||||
if ! rustup component list --toolchain "$TOOLCHAIN" --installed | grep -qx "$component"; then
|
||||
missing_components+=("$component")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ ${#missing_components[@]} -gt 0 ]; then
|
||||
rustup component add --toolchain "$TOOLCHAIN" "${missing_components[@]}"
|
||||
fi
|
||||
|
||||
if [ -n "$TARGET" ] && ! rustup target list --toolchain "$TOOLCHAIN" --installed | grep -qx "$TARGET"; then
|
||||
rustup target add --toolchain "$TOOLCHAIN" "$TARGET"
|
||||
fi
|
||||
|
||||
toolchain_bin="$(dirname "$(rustup which cargo --toolchain "$TOOLCHAIN")")"
|
||||
echo "$toolchain_bin" >> "$GITHUB_PATH"
|
||||
echo "Using Rust toolchain: $(rustc +"$TOOLCHAIN" --version)"
|
||||
@@ -0,0 +1,47 @@
|
||||
name: "ensure sccache"
|
||||
description: Ensure a pinned sccache binary is on PATH.
|
||||
|
||||
inputs:
|
||||
version:
|
||||
required: false
|
||||
default: "0.15.0"
|
||||
description: sccache release version without the leading v
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- shell: bash
|
||||
env:
|
||||
SCCACHE_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if command -v sccache >/dev/null 2>&1; then
|
||||
current="$(sccache --version | awk '{print $2}')"
|
||||
if [ "$current" = "$SCCACHE_VERSION" ]; then
|
||||
echo "Using baked sccache $current"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
case "$(uname -s)-$(uname -m)" in
|
||||
Linux-x86_64) triple=x86_64-unknown-linux-musl ;;
|
||||
Darwin-arm64) triple=aarch64-apple-darwin ;;
|
||||
Darwin-x86_64) triple=x86_64-apple-darwin ;;
|
||||
*) triple="" ;;
|
||||
esac
|
||||
|
||||
if [ -n "$triple" ]; then
|
||||
url="https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-${triple}.tar.gz"
|
||||
tmpdir="${RUNNER_TEMP:-/tmp}/sccache-${SCCACHE_VERSION}"
|
||||
bindir="${HOME}/.local/bin"
|
||||
rm -rf "$tmpdir"
|
||||
mkdir -p "$tmpdir" "$bindir"
|
||||
curl -fsSL "$url" | tar -xz -C "$tmpdir"
|
||||
install -m755 "$tmpdir"/sccache-v${SCCACHE_VERSION}-${triple}/sccache "$bindir/sccache"
|
||||
echo "$bindir" >> "$GITHUB_PATH"
|
||||
echo "Installed sccache $("$bindir/sccache" --version)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cargo install --locked sccache --version "$SCCACHE_VERSION"
|
||||
echo "Installed sccache $(sccache --version)"
|
||||
@@ -0,0 +1,40 @@
|
||||
name: "ensure zig"
|
||||
description: Ensure a pinned Zig binary is on PATH.
|
||||
|
||||
inputs:
|
||||
version:
|
||||
required: true
|
||||
description: Zig release version
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- shell: bash
|
||||
env:
|
||||
ZIG_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if command -v zig >/dev/null 2>&1 && [ "$(zig version)" = "$ZIG_VERSION" ]; then
|
||||
echo "Using baked zig $ZIG_VERSION"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "$(uname -s)-$(uname -m)" in
|
||||
Linux-x86_64) archive="zig-x86_64-linux-${ZIG_VERSION}" ;;
|
||||
Darwin-arm64) archive="zig-aarch64-macos-${ZIG_VERSION}" ;;
|
||||
Darwin-x86_64) archive="zig-x86_64-macos-${ZIG_VERSION}" ;;
|
||||
*)
|
||||
echo "Unsupported zig host: $(uname -s)-$(uname -m)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
tmpdir="${RUNNER_TEMP:-/tmp}/zig-${ZIG_VERSION}"
|
||||
bindir="${HOME}/.local/bin"
|
||||
rm -rf "$tmpdir"
|
||||
mkdir -p "$tmpdir" "$bindir"
|
||||
curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/${archive}.tar.xz" -o "$tmpdir/zig.tar.xz"
|
||||
tar -xJf "$tmpdir/zig.tar.xz" -C "$tmpdir"
|
||||
install -m755 "$tmpdir/${archive}/zig" "$bindir/zig"
|
||||
echo "$bindir" >> "$GITHUB_PATH"
|
||||
echo "Installed zig $("$bindir/zig" version)"
|
||||
+30
-29
@@ -212,7 +212,7 @@ jobs:
|
||||
- { variant: modern }
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/build-native
|
||||
- uses: ./.github/actions/build-native-kata
|
||||
with:
|
||||
hash: ${{ needs.native_artifact_lookup.outputs.source-hash }}
|
||||
platform: linux
|
||||
@@ -224,7 +224,7 @@ jobs:
|
||||
# Pre-warm the cross-platform native build cache on `main`, in addition to
|
||||
# building the artifacts that ship in releases. Skipped on main when
|
||||
# native_artifact_lookup already found a recent run with all artifacts intact.
|
||||
native_cross_platform:
|
||||
native_cross_platform_kata:
|
||||
name: "Native: ${{ matrix.platform }} ${{ matrix.arch }}"
|
||||
needs: [release_metadata, native_artifact_lookup]
|
||||
if: ${{ needs.release_metadata.outputs.is-release == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.native_artifact_lookup.outputs.cross-platform-run-id == '') }}
|
||||
@@ -233,9 +233,29 @@ jobs:
|
||||
matrix:
|
||||
include:
|
||||
- { os: omp-kata, platform: linux, arch: arm64, target: aarch64-unknown-linux-gnu }
|
||||
- { os: omp-kata, platform: win32, arch: x64, target: x86_64-pc-windows-msvc, variant: baseline }
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/build-native-kata
|
||||
with:
|
||||
hash: ${{ needs.native_artifact_lookup.outputs.source-hash }}
|
||||
platform: ${{ matrix.platform }}
|
||||
arch: ${{ matrix.arch }}
|
||||
variant: ${{ matrix.variant }}
|
||||
target: ${{ matrix.target }}
|
||||
save_cache: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
||||
|
||||
native_cross_platform_macos:
|
||||
name: "Native: ${{ matrix.platform }} ${{ matrix.arch }}"
|
||||
needs: [release_metadata, native_artifact_lookup]
|
||||
if: ${{ needs.release_metadata.outputs.is-release == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.native_artifact_lookup.outputs.cross-platform-run-id == '') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { os: macos-15-intel, platform: darwin, arch: x64, variant: baseline }
|
||||
- { os: macos-14, platform: darwin, arch: arm64 }
|
||||
- { os: omp-kata, platform: win32, arch: x64, target: x86_64-pc-windows-msvc, variant: baseline }
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -247,7 +267,6 @@ jobs:
|
||||
variant: ${{ matrix.variant }}
|
||||
target: ${{ matrix.target }}
|
||||
save_cache: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
||||
|
||||
test_workspace:
|
||||
name: Test TS workspace fast
|
||||
runs-on: omp-kata
|
||||
@@ -464,31 +483,12 @@ jobs:
|
||||
runs-on: omp-kata
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
- uses: ./.github/actions/ensure-rust-toolchain
|
||||
with:
|
||||
toolchain: nightly-2026-04-29
|
||||
- name: Decide Rust artifact cache
|
||||
id: rust_artifact_cache
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${SCCACHE_BUCKET:-}" ]; then
|
||||
echo "use_rust_cache=false" >> "$GITHUB_OUTPUT"
|
||||
echo "Rust target cache: skipped on shared-sccache runners"
|
||||
else
|
||||
echo "use_rust_cache=true" >> "$GITHUB_OUTPUT"
|
||||
echo "Rust target cache: GitHub Actions"
|
||||
fi
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
if: steps.rust_artifact_cache.outputs.use_rust_cache == 'true'
|
||||
- uses: ./.github/actions/ensure-sccache
|
||||
with:
|
||||
shared-key: install-methods-linux-x64
|
||||
cache-on-failure: true
|
||||
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
||||
cache-workspace-crates: true
|
||||
# On omp-kata, shared S3-backed sccache already carries the compile reuse
|
||||
# and avoids a second GitHub-cache handshake for target/.
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.10
|
||||
version: "0.15.0"
|
||||
- name: Enable sccache for cargo
|
||||
# Conditional backend: self-hosted omp-kata injects a shared S3
|
||||
# (RustFS) sccache via pod env; GitHub-hosted runners keep the GHA
|
||||
@@ -513,8 +513,9 @@ jobs:
|
||||
release_binary:
|
||||
name: "Release binary: ${{ matrix.target_id }}"
|
||||
if: ${{ needs.release_metadata.outputs.is-release == 'true' && !cancelled() &&
|
||||
needs.native_linux_x64.result == 'success' && needs.native_cross_platform.result ==
|
||||
'success' && needs.test_workspace.result == 'success' &&
|
||||
needs.native_linux_x64.result == 'success' && needs.native_cross_platform_kata.result ==
|
||||
'success' && needs.native_cross_platform_macos.result == 'success' &&
|
||||
needs.test_workspace.result == 'success' &&
|
||||
needs.test_coding_agent_singleton.result == 'success' &&
|
||||
needs.test_ts_native.result == 'success' &&
|
||||
needs.test_coding_agent_ui.result == 'success' &&
|
||||
@@ -522,7 +523,7 @@ jobs:
|
||||
needs.test_coding_agent_native.result == 'success' &&
|
||||
needs.test_smoke.result == 'success' && needs.check.result == 'success' &&
|
||||
needs.install_methods.result == 'success' }}
|
||||
needs: [release_metadata, check, native_linux_x64, native_cross_platform, test_workspace, test_coding_agent_singleton, test_ts_native, test_coding_agent_ui, test_coding_agent_runtime, test_coding_agent_native, test_smoke, install_methods, native_artifact_lookup]
|
||||
needs: [release_metadata, check, native_linux_x64, native_cross_platform_kata, native_cross_platform_macos, test_workspace, test_coding_agent_singleton, test_ts_native, test_coding_agent_ui, test_coding_agent_runtime, test_coding_agent_native, test_smoke, install_methods, native_artifact_lookup]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
||||
@@ -26,15 +26,17 @@ All host commands below run on `<CI_HOST>` (the single k3s node) as root.
|
||||
The base `ghcr.io/actions/actions-runner:latest` is a clean Ubuntu 24.04 runner.
|
||||
On a normal (GitHub-hosted-style) runner, the CI workflow installs its system
|
||||
dependencies at the start of every job: the cairo/pango native stack for canvas
|
||||
builds, `fd`/`ripgrep`/`imagemagick`, `bun`, and a pinned Rust nightly with the
|
||||
cross targets. Inside a Kata microVM that is destroyed after a single job, paying
|
||||
that apt/bun/rustup cost on **every** job is pure latency - the microVM starts
|
||||
cold each time.
|
||||
builds, `fd`/`ripgrep`/`imagemagick`, `bun`, `sccache`, Zig, the cargo-native
|
||||
helper CLIs (`cargo-nextest`, `cargo-zigbuild`, `cargo-xwin`), and a pinned Rust
|
||||
nightly with the cross targets/components. Inside a Kata microVM that is
|
||||
destroyed after a single job, paying that apt/bun/rustup/tool-download cost on
|
||||
**every** job is pure latency - the microVM starts cold each time.
|
||||
|
||||
The preloaded image moves that work to build time. Every ephemeral runner then
|
||||
starts with the toolchain already present: apt deps are not re-fetched, `bun` and
|
||||
`cargo`/`rustc` are on `PATH`, and the pinned Rust toolchain is already the
|
||||
default so target/component installs in CI become no-ops.
|
||||
starts with the toolchain already present: apt deps are not re-fetched, `bun`,
|
||||
`cargo`/`rustc`, `sccache`, `zig`, and the cargo helper CLIs are on `PATH`, and
|
||||
the pinned Rust toolchain is already the default so target/component installs in
|
||||
CI become no-ops.
|
||||
|
||||
### Stay in sync with `setup-system-deps`
|
||||
|
||||
@@ -79,7 +81,8 @@ reproduced verbatim (it contains no secrets or redactable host identifiers; the
|
||||
# tool and release workflows expect it
|
||||
# - C/build toolchain the native + canvas builds need
|
||||
# - bun (system-wide, on PATH)
|
||||
# - rust nightly toolchain (pinned) + clippy/rustfmt + linux-arm64/windows-msvc targets
|
||||
# - sccache + Zig + cargo-nextest/cargo-zigbuild/cargo-xwin for native builds
|
||||
# - rust nightly (pinned) + clippy/rustfmt/rust-analyzer + linux-arm64/windows-msvc targets
|
||||
#
|
||||
# Rebuild + reimport (see /root/omp-kata-runner.md) after bumping the ARGs below
|
||||
# or the apt set. Keep the apt set in sync with .github/actions/setup-system-deps.
|
||||
@@ -87,19 +90,23 @@ FROM ghcr.io/actions/actions-runner:latest
|
||||
|
||||
ARG RUST_NIGHTLY=nightly-2026-04-29
|
||||
ARG BUN_VERSION=1.3.14
|
||||
ARG SCCACHE_VERSION=0.15.0
|
||||
ARG ZIG_VERSION=0.16.0
|
||||
|
||||
USER root
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Mirrors the "Install system deps" block in .github/workflows/ci.yml plus the
|
||||
# C/build toolchain (native + canvas builds) and the GitHub CLI. The gh apt repo
|
||||
# is added first so `gh` installs in the same apt transaction.
|
||||
# native/cross toolchain (clang/lld/llvm), the baked cache/tooling binaries, and
|
||||
# the GitHub CLI. The gh apt repo is added first so `gh` installs in the same apt
|
||||
# transaction.
|
||||
RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg -o /usr/share/keyrings/githubcli-archive-keyring.gpg \
|
||||
&& chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
|
||||
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y \
|
||||
build-essential pkg-config curl ca-certificates git unzip xz-utils zstd gh \
|
||||
clang lld llvm \
|
||||
libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \
|
||||
fd-find ripgrep imagemagick \
|
||||
&& ln -sf "$(command -v fdfind)" /usr/local/bin/fd \
|
||||
@@ -111,17 +118,34 @@ ENV BUN_INSTALL=/usr/local
|
||||
RUN curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}" \
|
||||
&& bun --version
|
||||
|
||||
# rust toolchain for the runner user; rustup default == pinned nightly so
|
||||
# dtolnay/rust-toolchain@nightly and target/component adds are no-ops in CI.
|
||||
# Pinned native-build helpers, system-wide.
|
||||
RUN curl -fsSL "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \
|
||||
| tar -xz -C /tmp \
|
||||
&& install -m755 "/tmp/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" /usr/local/bin/sccache \
|
||||
&& rm -rf "/tmp/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl"
|
||||
RUN curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz \
|
||||
&& tar -xJf /tmp/zig.tar.xz -C /opt \
|
||||
&& ln -sf "/opt/zig-x86_64-linux-${ZIG_VERSION}/zig" /usr/local/bin/zig \
|
||||
&& rm -f /tmp/zig.tar.xz
|
||||
|
||||
# rust toolchain + cargo helpers for the runner user; rustup default == pinned
|
||||
# nightly so Rust setup becomes a no-op on the preloaded image.
|
||||
USER runner
|
||||
ENV RUSTUP_HOME=/home/runner/.rustup \
|
||||
CARGO_HOME=/home/runner/.cargo \
|
||||
PATH=/home/runner/.cargo/bin:/usr/local/bin:${PATH}
|
||||
RUN curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \
|
||||
| sh -s -- -y --default-toolchain "${RUST_NIGHTLY}" --profile minimal \
|
||||
&& rustup component add clippy rustfmt \
|
||||
&& rustup component add clippy rustfmt rust-analyzer \
|
||||
&& rustup target add aarch64-unknown-linux-gnu x86_64-pc-windows-msvc \
|
||||
&& cargo --version && rustc --version
|
||||
&& cargo install --locked cargo-nextest cargo-zigbuild cargo-xwin \
|
||||
&& cargo --version \
|
||||
&& rustc --version \
|
||||
&& sccache --version \
|
||||
&& zig version \
|
||||
&& cargo nextest --version \
|
||||
&& cargo zigbuild --version \
|
||||
&& cargo xwin --version
|
||||
```
|
||||
|
||||
### Stage-by-stage annotation
|
||||
@@ -152,9 +176,11 @@ In order:
|
||||
tool.
|
||||
- `apt-get install` pulls three groups:
|
||||
- **build toolchain / utilities:** `build-essential pkg-config curl
|
||||
ca-certificates git unzip xz-utils zstd gh`. `build-essential` + `pkg-config`
|
||||
are needed by the native and canvas builds; `zstd` is the codec the bun and
|
||||
sccache cache tarballs use (see [04-arc-and-caching.md](./04-arc-and-caching.md)).
|
||||
ca-certificates git unzip xz-utils zstd gh clang lld llvm`.
|
||||
`build-essential` + `pkg-config` are needed by the native and canvas builds;
|
||||
`zstd` is the codec the bun and sccache cache tarballs use (see
|
||||
[04-arc-and-caching.md](./04-arc-and-caching.md)); `clang lld llvm` are the
|
||||
MSVC-cross prerequisites that used to be apt-installed per job.
|
||||
- **canvas / cairo native stack:** `libcairo2-dev libpango1.0-dev libjpeg-dev
|
||||
libgif-dev librsvg2-dev` - the `-dev` headers the canvas/rsvg native modules
|
||||
compile against.
|
||||
@@ -174,16 +200,25 @@ In order:
|
||||
`bun-v${BUN_VERSION}`, and `bun --version` fails the build if the install is
|
||||
broken.
|
||||
|
||||
**Pinned native-build helpers (two root `RUN`s).** `sccache` is downloaded as a
|
||||
version-pinned GitHub release tarball and installed to `/usr/local/bin`; Zig is
|
||||
downloaded as the pinned release archive, unpacked under `/opt`, and symlinked
|
||||
into `/usr/local/bin/zig`. Baking these two removes the per-job
|
||||
`mozilla-actions/sccache-action` and `mlugg/setup-zig` downloads from the
|
||||
self-hosted path.
|
||||
|
||||
**Rust toolchain (`USER runner` + rustup `RUN`).** The toolchain is installed as
|
||||
the **`runner` user** - the UID jobs execute as - so cargo/rustc are owned by and
|
||||
visible to the job without sudo. `RUSTUP_HOME`/`CARGO_HOME` are pinned under
|
||||
`/home/runner`, and `~/.cargo/bin` is prepended to `PATH`. rustup installs the
|
||||
pinned nightly as the **default toolchain** (`--profile minimal`), then adds the
|
||||
`clippy` and `rustfmt` components and the `aarch64-unknown-linux-gnu`
|
||||
(Linux arm64) and `x86_64-pc-windows-msvc` (Windows cross) targets. Because the
|
||||
default toolchain already *is* the pinned nightly with these components/targets,
|
||||
the corresponding `rustup` steps in CI become no-ops - the warm-start payoff.
|
||||
`cargo --version && rustc --version` is the final build-time sanity check.
|
||||
`clippy`, `rustfmt`, and `rust-analyzer` components plus the
|
||||
`aarch64-unknown-linux-gnu` (Linux arm64) and `x86_64-pc-windows-msvc` (Windows
|
||||
cross) targets. The same layer also `cargo install`s the Rust-native helper CLIs
|
||||
`cargo-nextest`, `cargo-zigbuild`, and `cargo-xwin`, so the self-hosted native
|
||||
build path no longer fetches those tools job-by-job. Because the default toolchain
|
||||
already *is* the pinned nightly with these components/targets, the corresponding
|
||||
Rust setup steps in CI become no-ops - the warm-start payoff.
|
||||
|
||||
---
|
||||
|
||||
@@ -217,10 +252,10 @@ DOCKER_BUILDKIT=1 docker build -t "$IMAGE" -t omp-kata-runner:preloaded .
|
||||
echo "==> [2/5] verifying baked tools"
|
||||
docker run --rm --entrypoint bash "$IMAGE" -lc '
|
||||
set -e
|
||||
for b in gh fd rg magick bun cargo rustc pkg-config zstd; do
|
||||
for b in gh fd rg magick bun cargo rustc pkg-config zstd clang lld sccache zig cargo-nextest cargo-zigbuild cargo-xwin; do
|
||||
command -v "$b" >/dev/null || { echo "MISSING: $b"; exit 1; }
|
||||
done
|
||||
echo "tools OK | $(bun --version) | $(rustc --version) | gh $(gh --version | head -1 | cut -d" " -f3)"
|
||||
echo "tools OK | bun $(bun --version) | rust $(rustc --version) | sccache $(sccache --version | awk '\''{print $2}'\'') | zig $(zig version) | gh $(gh --version | head -1 | cut -d\" \" -f3)"
|
||||
'
|
||||
|
||||
echo "==> [3/5] importing into k3s containerd (k8s.io namespace)"
|
||||
@@ -259,10 +294,10 @@ BuildKit + the docker layer cache make an unchanged rebuild near-instant.
|
||||
|
||||
**[2/5] verify baked tools.** Runs the freshly built image with a bash entrypoint
|
||||
and asserts every expected binary is on `PATH`
|
||||
(`gh fd rg magick bun cargo rustc pkg-config zstd`), failing the whole script if
|
||||
any is missing, then prints the bun / rustc / gh versions. This catches a broken
|
||||
apt set, missing shim, or bad toolchain pin **before** anything touches the
|
||||
cluster.
|
||||
(`gh fd rg magick bun cargo rustc pkg-config zstd clang lld sccache zig cargo-nextest cargo-zigbuild cargo-xwin`),
|
||||
failing the whole script if any is missing, then prints the key version tuple
|
||||
(bun / rust / sccache / zig / gh). This catches a broken apt set, missing shim,
|
||||
or bad toolchain pin **before** anything touches the cluster.
|
||||
|
||||
**[3/5] import into k3s containerd.**
|
||||
`docker save "$IMAGE" | k3s ctr -n k8s.io images import --platform linux/amd64 -`
|
||||
|
||||
@@ -56,10 +56,10 @@ DOCKER_BUILDKIT=1 docker build -t "$IMAGE" -t omp-kata-runner:preloaded .
|
||||
echo "==> [2/5] verifying baked tools"
|
||||
docker run --rm --entrypoint bash "$IMAGE" -lc '
|
||||
set -e
|
||||
for b in gh fd rg magick bun cargo rustc pkg-config zstd; do
|
||||
for b in gh fd rg magick bun cargo rustc pkg-config zstd clang lld sccache zig cargo-nextest cargo-zigbuild cargo-xwin; do
|
||||
command -v "$b" >/dev/null || { echo "MISSING: $b"; exit 1; }
|
||||
done
|
||||
echo "tools OK | $(bun --version) | $(rustc --version) | gh $(gh --version | head -1 | cut -d" " -f3)"
|
||||
echo "tools OK | bun $(bun --version) | rust $(rustc --version) | sccache $(sccache --version | awk '\''{print $2}'\'') | zig $(zig version) | gh $(gh --version | head -1 | cut -d\" \" -f3)"
|
||||
'
|
||||
|
||||
echo "==> [3/5] importing into k3s containerd (k8s.io namespace)"
|
||||
|
||||
+29
-7
@@ -9,7 +9,8 @@
|
||||
# tool and release workflows expect it
|
||||
# - C/build toolchain the native + canvas builds need
|
||||
# - bun (system-wide, on PATH)
|
||||
# - rust nightly toolchain (pinned) + clippy/rustfmt + linux-arm64/windows-msvc targets
|
||||
# - sccache + Zig + cargo-nextest/cargo-zigbuild/cargo-xwin for native builds
|
||||
# - rust nightly (pinned) + clippy/rustfmt/rust-analyzer + linux-arm64/windows-msvc targets
|
||||
#
|
||||
# Rebuild + reimport (see /root/omp-kata-runner.md) after bumping the ARGs below
|
||||
# or the apt set. Keep the apt set in sync with .github/actions/setup-system-deps.
|
||||
@@ -17,19 +18,23 @@ FROM ghcr.io/actions/actions-runner:latest
|
||||
|
||||
ARG RUST_NIGHTLY=nightly-2026-04-29
|
||||
ARG BUN_VERSION=1.3.14
|
||||
ARG SCCACHE_VERSION=0.15.0
|
||||
ARG ZIG_VERSION=0.16.0
|
||||
|
||||
USER root
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Mirrors the "Install system deps" block in .github/workflows/ci.yml plus the
|
||||
# C/build toolchain (native + canvas builds) and the GitHub CLI. The gh apt repo
|
||||
# is added first so `gh` installs in the same apt transaction.
|
||||
# native/cross toolchain (clang/lld/llvm), the baked cache/tooling binaries, and
|
||||
# the GitHub CLI. The gh apt repo is added first so `gh` installs in the same apt
|
||||
# transaction.
|
||||
RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg -o /usr/share/keyrings/githubcli-archive-keyring.gpg \
|
||||
&& chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
|
||||
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y \
|
||||
build-essential pkg-config curl ca-certificates git unzip xz-utils zstd gh \
|
||||
clang lld llvm \
|
||||
libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \
|
||||
fd-find ripgrep imagemagick \
|
||||
&& ln -sf "$(command -v fdfind)" /usr/local/bin/fd \
|
||||
@@ -41,14 +46,31 @@ ENV BUN_INSTALL=/usr/local
|
||||
RUN curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}" \
|
||||
&& bun --version
|
||||
|
||||
# rust toolchain for the runner user; rustup default == pinned nightly so
|
||||
# dtolnay/rust-toolchain@nightly and target/component adds are no-ops in CI.
|
||||
# Pinned native-build helpers, system-wide.
|
||||
RUN curl -fsSL "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \
|
||||
| tar -xz -C /tmp \
|
||||
&& install -m755 "/tmp/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" /usr/local/bin/sccache \
|
||||
&& rm -rf "/tmp/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl"
|
||||
RUN curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz \
|
||||
&& tar -xJf /tmp/zig.tar.xz -C /opt \
|
||||
&& ln -sf "/opt/zig-x86_64-linux-${ZIG_VERSION}/zig" /usr/local/bin/zig \
|
||||
&& rm -f /tmp/zig.tar.xz
|
||||
|
||||
# rust toolchain + cargo helpers for the runner user; rustup default == pinned
|
||||
# nightly so Rust setup becomes a no-op on the preloaded image.
|
||||
USER runner
|
||||
ENV RUSTUP_HOME=/home/runner/.rustup \
|
||||
CARGO_HOME=/home/runner/.cargo \
|
||||
PATH=/home/runner/.cargo/bin:/usr/local/bin:${PATH}
|
||||
RUN curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \
|
||||
| sh -s -- -y --default-toolchain "${RUST_NIGHTLY}" --profile minimal \
|
||||
&& rustup component add clippy rustfmt \
|
||||
&& rustup component add clippy rustfmt rust-analyzer \
|
||||
&& rustup target add aarch64-unknown-linux-gnu x86_64-pc-windows-msvc \
|
||||
&& cargo --version && rustc --version
|
||||
&& cargo install --locked cargo-nextest cargo-zigbuild cargo-xwin \
|
||||
&& cargo --version \
|
||||
&& rustc --version \
|
||||
&& sccache --version \
|
||||
&& zig version \
|
||||
&& cargo nextest --version \
|
||||
&& cargo zigbuild --version \
|
||||
&& cargo xwin --version
|
||||
|
||||
@@ -8,9 +8,9 @@
|
||||
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { RUN_EMBEDDINGS } from "./setup";
|
||||
import { initBeam } from "@oh-my-pi/pi-mnemopi/core/beam";
|
||||
import { Mnemopi } from "@oh-my-pi/pi-mnemopi/core/memory";
|
||||
import { RUN_EMBEDDINGS } from "./setup";
|
||||
|
||||
const OLD_MODEL = "BAAI/bge-small-en-v1.5";
|
||||
const NEW_MODEL = "intfloat/multilingual-e5-large";
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { RUN_EMBEDDINGS } from "./setup";
|
||||
import {
|
||||
cosineSimilarity,
|
||||
embed,
|
||||
@@ -8,6 +7,7 @@ import {
|
||||
resetEmbeddingProviderForTests,
|
||||
setEmbeddingProviderForTests,
|
||||
} from "@oh-my-pi/pi-mnemopi/core/embeddings";
|
||||
import { RUN_EMBEDDINGS } from "./setup";
|
||||
|
||||
function withEnvValue<T>(key: string, value: string | undefined, fn: () => T): T {
|
||||
const previous = process.env[key];
|
||||
|
||||
@@ -16,7 +16,6 @@ import { describe, expect, it } from "bun:test";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { RUN_EMBEDDINGS } from "./setup";
|
||||
import { cmdRemember } from "@oh-my-pi/pi-mnemopi/cli";
|
||||
import { BeamMemory } from "@oh-my-pi/pi-mnemopi/core/beam";
|
||||
import { Mnemopi } from "@oh-my-pi/pi-mnemopi/core/memory";
|
||||
@@ -24,6 +23,7 @@ import {
|
||||
type ResolvedMnemopiRuntimeOptions,
|
||||
withMnemopiRuntimeOptions,
|
||||
} from "@oh-my-pi/pi-mnemopi/core/runtime-options";
|
||||
import { RUN_EMBEDDINGS } from "./setup";
|
||||
|
||||
interface EmbeddingRow {
|
||||
readonly memory_id: string;
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { getFastembedCacheDir } from "@oh-my-pi/pi-utils";
|
||||
import { RUN_EMBEDDINGS } from "./setup";
|
||||
import {
|
||||
available,
|
||||
embed,
|
||||
@@ -12,7 +10,9 @@ import {
|
||||
} from "@oh-my-pi/pi-mnemopi/core/embeddings";
|
||||
import { Mnemopi } from "@oh-my-pi/pi-mnemopi/core/memory";
|
||||
import { withMnemopiRuntimeOptions } from "@oh-my-pi/pi-mnemopi/core/runtime-options";
|
||||
import { getFastembedCacheDir } from "@oh-my-pi/pi-utils";
|
||||
import packageJson from "../package.json" with { type: "json" };
|
||||
import { RUN_EMBEDDINGS } from "./setup";
|
||||
|
||||
const ENV_KEYS = [
|
||||
"NODE_ENV",
|
||||
|
||||
@@ -64,7 +64,6 @@ class FakeLocalLlmBackend implements LlmBackend {
|
||||
}
|
||||
export const RUN_EMBEDDINGS = Bun.env.EMBEDDINGS === "1";
|
||||
|
||||
|
||||
beforeEach(() => {
|
||||
// Real embeddings (fastembed + onnxruntime-node, ~270MB peers) install on
|
||||
// demand via `bun install` on first use. Default the suite to the lightweight
|
||||
|
||||
Reference in New Issue
Block a user