From 5f988a82701bbb90053882452fc6417d9dd20931 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 27 Jul 2026 07:53:28 +0200 Subject: [PATCH] ci: configured native artifact caching and parallel execution in ci workflows - Enhanced CI workflows and GitHub actions to support native artifact caching and parallel builds. - Added composite actions and scripts for computing sources, finding artifacts, and managing caches. - Updated infrastructure documentation and runner deployment scripts with revised resource limits. --- .github/actions/build-native/action.yml | 132 ++++-- .../actions/find-native-artifacts/action.yml | 99 ++++ .github/actions/native-source-hash/action.yml | 30 ++ .../actions/restore-linux-native/action.yml | 54 +++ .github/workflows/ci.yml | 427 ++++++------------ .github/workflows/native-prewarm.yml | 90 ++++ infra/docs/04-arc-and-caching.md | 86 ++-- infra/docs/README.md | 10 +- infra/reload-runner.sh | 44 +- infra/runner.Dockerfile | 2 +- scripts/ci-native-artifact-cache.test.ts | 83 ++++ scripts/ci-native-artifact-cache.ts | 129 ++++++ scripts/ci-target-cache.ts | 11 +- scripts/ci-test-ts.ts | 12 +- scripts/install-tests/run-ci.sh | 4 +- 15 files changed, 824 insertions(+), 389 deletions(-) create mode 100644 .github/actions/find-native-artifacts/action.yml create mode 100644 .github/actions/native-source-hash/action.yml create mode 100644 .github/actions/restore-linux-native/action.yml create mode 100644 .github/workflows/native-prewarm.yml create mode 100644 scripts/ci-native-artifact-cache.test.ts create mode 100755 scripts/ci-native-artifact-cache.ts diff --git a/.github/actions/build-native/action.yml b/.github/actions/build-native/action.yml index cfeb6d52a..cc548e2d1 100644 --- a/.github/actions/build-native/action.yml +++ b/.github/actions/build-native/action.yml @@ -28,10 +28,9 @@ inputs: default: "" glibc: description: > - Optional glibc floor (e.g. "2.17") for linux-gnu builds. Routes the build - through cargo-zigbuild against that floor without affecting the rustup - target or the host-arch native test steps. Combine with `target` to pin a - cross-arch linux build, or set alone for a host-arch (x64) linux build. + Optional glibc floor override for linux-gnu builds (defaults to "2.17"). + Routes the build through cargo-zigbuild against that floor without + affecting the rustup target or host-arch native test steps. required: false default: "" libc: @@ -43,16 +42,19 @@ inputs: required: false default: "false" skip_validation: - description: > - Skip clippy/rustfmt and the Rust test suite. Set on release runs: the - version-bump commit only changes version strings, and the tagged - content's Rust code already passed validation on its main-push run. + description: Skip clippy/rustfmt and the Rust test suite for build-only matrix entries. required: false default: "false" + skip_build: + description: Run validation and cache population without building or uploading a native addon. + required: false + default: "false" + cache_scope: + description: Separates target snapshots whose Cargo work differs (for example, build and validation). + required: false + default: "build" save_cache: - description: > - Whether to write build caches: Swatinem/rust-cache on GitHub-hosted - runners, the RustFS target/ snapshot on omp-kata. + description: Whether to persist the GitHub-hosted or RustFS target snapshot. required: false default: "false" @@ -82,9 +84,20 @@ runs: env: TARGET: ${{ inputs.target }} GLIBC: ${{ inputs.glibc }} + PLATFORM: ${{ inputs.platform }} + LIBC: ${{ inputs.libc }} ARCH: ${{ inputs.arch }} run: | set -euo pipefail + # Keep the portability floor here: this action is included in the + # native source hash, and every workflow then consumes one value. + if [ "$PLATFORM" = linux ] && [ "$LIBC" != musl ]; then + GLIBC="${GLIBC:-2.17}" + elif [ -n "$GLIBC" ]; then + echo "::error::glibc floor '$GLIBC' is only valid for linux-gnu builds" + exit 1 + fi + # `cross_target` is what the napi build feeds cargo: cargo-zigbuild reads # the glibc floor as a `..` triple suffix. `bare_target` is # what rustup needs — never glibc-suffixed (rustup rejects the suffix) @@ -98,11 +111,13 @@ runs: fi cross_target="" if [ -n "$GLIBC" ]; then - if [ -z "$base" ]; then - echo "::error::glibc floor '$GLIBC' set but no linux-gnu base triple for arch '$ARCH'" - exit 1 - fi - cross_target="${base}.${GLIBC}" + case "$base" in + *-linux-gnu) cross_target="${base}.${GLIBC}" ;; + *) + echo "::error::glibc floor '$GLIBC' requires a linux-gnu target, got '$base'" + exit 1 + ;; + esac elif [ -n "$TARGET" ]; then cross_target="$TARGET" fi @@ -187,25 +202,28 @@ runs: echo "RUSTFLAGS=$rustflags" >> "$GITHUB_ENV" echo "Configured RUSTFLAGS=$rustflags" - # --- target/ cache (GitHub-hosted only) --------------------------------- - # Swatinem keys target/ off its restore-time environment, so it must run - # after RUSTFLAGS is set: if x64 target-cpu were only selected later, cargo - # would invalidate the restored target/ while rust-cache saw an exact key - # and refused to save the rebuilt artifacts (macOS x64 baseline rebuilds - # forever). The native source hash is in the shared key too: rust-cache's - # lockfile scan misses the workspace-root Cargo.toml version Cargo - # fingerprints, so a version bump could otherwise get an exact hit for - # artifacts Cargo must rebuild. On omp-kata the reuse layers are the - # mounted Cargo registry, RustFS sccache, and the RustFS target/ snapshot - # (see "Restore target/ cache" below), so Swatinem stays GitHub-only. - - name: Cache Rust target/ (GitHub-hosted) + # --- Cargo + rolling target cache (GitHub-hosted only) ------------------ + # Swatinem keeps the registry/tool cache. target/ uses an explicit rolling + # key: a new source hash restores the latest compatible snapshot through + # restore-keys, then saves the rebuilt state under a fresh immutable key. + # This is especially important for the three-core Intel macOS runner. + - name: Cache Cargo dependencies (GitHub-hosted) if: steps.detect.outputs.on_infra == 'false' uses: Swatinem/rust-cache@v2 with: - shared-key: native-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }}-h${{ inputs.hash }} + shared-key: native-deps-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }} cache-on-failure: true save-if: ${{ inputs.save_cache == 'true' }} - cache-workspace-crates: true + cache-targets: false + - name: Restore rolling Rust target/ (GitHub-hosted) + id: gha-target + if: steps.detect.outputs.on_infra == 'false' + uses: actions/cache/restore@v4 + with: + path: target + key: native-target-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }}-${{ inputs.cache_scope }}-${{ hashFiles('rust-toolchain.toml') }}-h${{ inputs.hash }} + restore-keys: | + native-target-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }}-${{ inputs.cache_scope }}-${{ hashFiles('rust-toolchain.toml') }}- # --- sccache ------------------------------------------------------------ - name: Ensure baked sccache (omp-kata) @@ -225,10 +243,24 @@ runs: env: CROSS_TARGET: ${{ steps.resolve.outputs.cross_target }} run: | + jobs="${OMP_CI_CPU_COUNT:-$(getconf _NPROCESSORS_ONLN)}" + if [ -z "${OMP_CI_CPU_COUNT:-}" ] && [ -r /sys/fs/cgroup/cpu.max ]; then + read -r quota period < /sys/fs/cgroup/cpu.max + if [ "$quota" != "max" ]; then + quota_jobs=$((quota / period)) + [ "$quota_jobs" -ge 1 ] || quota_jobs=1 + [ "$quota_jobs" -ge "$jobs" ] || jobs="$quota_jobs" + fi + fi { + echo "OMP_CI_CPU_COUNT=$jobs" echo "RUSTC_WRAPPER=sccache" echo "CARGO_INCREMENTAL=0" + echo "CARGO_BUILD_JOBS=$jobs" + echo "CMAKE_BUILD_PARALLEL_LEVEL=$jobs" + echo "NEXTEST_TEST_THREADS=$jobs" } >> "$GITHUB_ENV" + echo "Native build parallelism: $jobs" # Route CMake-built C (audiopus_sys' bundled opus) through sccache # too — build scripts bypass RUSTC_WRAPPER. Non-cross builds only: # cross builds compile C with zig cc / clang-cl wrapper scripts that @@ -248,13 +280,13 @@ runs: # --- cargo-nextest (native test runner; non-cross builds only) ---------- - name: Ensure baked cargo-nextest (omp-kata) - if: steps.detect.outputs.on_infra == 'true' && inputs.target == '' + if: steps.detect.outputs.on_infra == 'true' && inputs.target == '' && inputs.skip_validation != 'true' uses: ./.github/actions/ensure-cargo-tool with: binary: cargo-nextest crate: cargo-nextest - name: Install cargo-nextest (GitHub-hosted) - if: steps.detect.outputs.on_infra == 'false' && inputs.target == '' + if: steps.detect.outputs.on_infra == 'false' && inputs.target == '' && inputs.skip_validation != 'true' uses: taiki-e/install-action@v2 with: tool: nextest @@ -323,12 +355,17 @@ runs: # cargo's fingerprint/link work bypass it. Snapshot target/ to the same # RustFS S3 bucket, keyed per platform/libc/arch/variant + toolchain and # overwritten on each save so storage stays bounded at one snapshot per - # key. GitHub-hosted runners get the same effect from Swatinem above. + # key. GitHub-hosted runners get the same effect from the rolling cache above. + - name: Verify target cache compressor (omp-kata) + if: steps.detect.outputs.on_infra == 'true' + shell: bash + run: zstd --version + - name: Restore target/ cache (omp-kata) if: steps.detect.outputs.on_infra == 'true' shell: bash env: - TARGET_CACHE_KEY: native-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }} + TARGET_CACHE_KEY: native-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }}-${{ inputs.cache_scope }} run: bun scripts/ci-target-cache.ts restore "$TARGET_CACHE_KEY" # --- Checks, build, upload (shared) ------------------------------------- @@ -345,29 +382,46 @@ runs: shell: bash run: bun run test:rs - name: Build native addon(s) + if: inputs.skip_build != 'true' shell: bash env: CROSS_TARGET: ${{ steps.resolve.outputs.cross_target }} TARGET_PLATFORM: ${{ inputs.platform }} TARGET_ARCH: ${{ inputs.arch }} TARGET_VARIANTS: ${{ inputs.variant }} - run: bun run ci:build:native + run: | + if [ "$CROSS_TARGET" = x86_64-apple-darwin ]; then + # Do not accept Homebrew's arm64 libopus through pkg-config; + # build audiopus_sys's bundled x64 archive. + export OPUS_NO_PKG_CONFIG=1 + fi + bun run ci:build:native - name: sccache stats shell: bash run: sccache --show-stats || true + - name: Save native addon(s) to in-cluster cache + if: inputs.skip_build != 'true' && steps.detect.outputs.on_infra == 'true' + shell: bash + env: + ARTIFACT_NAME: pi-natives-${{ inputs.platform }}-${{ inputs.libc && format('{0}-', inputs.libc) || '' }}${{ inputs.arch }}${{ inputs.variant && format('-{0}', inputs.variant) || '' }}-h${{ inputs.hash }} + run: bun scripts/ci-native-artifact-cache.ts save "${{ inputs.hash }}" "$ARTIFACT_NAME" - name: Upload native addon(s) + if: inputs.skip_build != 'true' uses: actions/upload-artifact@v4 with: name: pi-natives-${{ inputs.platform }}-${{ inputs.libc && format('{0}-', inputs.libc) || '' }}${{ inputs.arch }}${{ inputs.variant && format('-{0}', inputs.variant) || '' }}-h${{ inputs.hash }} path: packages/natives/native/pi_natives.${{ inputs.platform }}-${{ inputs.arch }}*.node if-no-files-found: error - # Explicit so the native_artifact_lookup canary keeps working even if - # org defaults shift; bump if Rust source ever stays stable for >90 days - # of main pushes and you want to avoid rebuilds. retention-days: 90 - name: Save target/ cache (omp-kata) if: steps.detect.outputs.on_infra == 'true' && inputs.save_cache == 'true' shell: bash env: - TARGET_CACHE_KEY: native-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }} + TARGET_CACHE_KEY: native-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }}-${{ inputs.cache_scope }} run: bun scripts/ci-target-cache.ts save "$TARGET_CACHE_KEY" + - name: Save rolling Rust target/ (GitHub-hosted) + if: steps.detect.outputs.on_infra == 'false' && inputs.save_cache == 'true' && steps.gha-target.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: target + key: native-target-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }}-${{ inputs.cache_scope }}-${{ hashFiles('rust-toolchain.toml') }}-h${{ inputs.hash }} diff --git a/.github/actions/find-native-artifacts/action.yml b/.github/actions/find-native-artifacts/action.yml new file mode 100644 index 000000000..0a5de5bec --- /dev/null +++ b/.github/actions/find-native-artifacts/action.yml @@ -0,0 +1,99 @@ +name: Find reusable native artifacts +description: Find complete trusted native artifact sets for one source hash, including canceled main runs. + +inputs: + hash: + description: Native source hash embedded in artifact names + required: true + +outputs: + linux-x64-run-id: + description: Run containing both Linux x64 variants + value: ${{ steps.find.outputs.linux-x64-run-id }} + cross-platform-run-id: + description: Run containing every cross-platform artifact + value: ${{ steps.find.outputs.cross-platform-run-id }} + validation-run-id: + description: Run containing the successful Rust validation marker + value: ${{ steps.find.outputs.validation-run-id }} + +runs: + using: composite + steps: + - name: Find complete artifact sets + id: find + shell: bash + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + REPOSITORY_ID: ${{ github.repository_id }} + SOURCE_HASH: ${{ inputs.hash }} + run: | + set -euo pipefail + + artifact_run_ids() { + local artifact_name="$1" + gh api --paginate "/repos/${REPOSITORY}/actions/artifacts?name=${artifact_name}&per_page=100" \ + --jq ".artifacts[] | select(.expired == false and .workflow_run.head_branch == \"main\" and .workflow_run.head_repository_id == ${REPOSITORY_ID}) | .workflow_run.id" \ + | sort -rn | uniq + } + + find_complete_run() { + local canary="$1" + shift + local candidate names required complete + while read -r candidate; do + [ -n "$candidate" ] || continue + names="$(gh api "/repos/${REPOSITORY}/actions/runs/${candidate}/artifacts?per_page=100" \ + --jq '.artifacts[] | select(.expired == false) | .name')" + complete=true + for required in "$@"; do + if ! grep -qFx "$required" <<<"$names"; then + complete=false + break + fi + done + if $complete; then + echo "$candidate" + return 0 + fi + done < <(artifact_run_ids "$canary") + } + + linux_baseline="pi-natives-linux-x64-baseline-h${SOURCE_HASH}" + linux_modern="pi-natives-linux-x64-modern-h${SOURCE_HASH}" + cross_required=( + "pi-natives-linux-arm64-h${SOURCE_HASH}" + "pi-natives-linux-musl-x64-baseline-h${SOURCE_HASH}" + "pi-natives-linux-musl-arm64-h${SOURCE_HASH}" + "pi-natives-darwin-x64-baseline-h${SOURCE_HASH}" + "pi-natives-darwin-arm64-h${SOURCE_HASH}" + "pi-natives-win32-x64-baseline-h${SOURCE_HASH}" + ) + validation_marker="pi-natives-rust-validation-h${SOURCE_HASH}" + + linux_x64_run_id="$(find_complete_run "$linux_modern" "$linux_baseline" "$linux_modern")" + cross_platform_run_id="$(find_complete_run "${cross_required[3]}" "${cross_required[@]}")" + validation_run_id="$(find_complete_run "$validation_marker" "$validation_marker")" + + if [ -n "$linux_x64_run_id" ]; then + echo "Reusing Linux x64 native artifacts from run $linux_x64_run_id" + else + echo "No complete Linux x64 artifact set for hash $SOURCE_HASH" + fi + if [ -n "$cross_platform_run_id" ]; then + echo "Reusing cross-platform native artifacts from run $cross_platform_run_id" + else + echo "No complete cross-platform artifact set for hash $SOURCE_HASH" + fi + if [ -n "$validation_run_id" ]; then + echo "Reusing Rust validation from run $validation_run_id" + else + echo "No Rust validation marker for hash $SOURCE_HASH" + fi + + { + echo "linux-x64-run-id=$linux_x64_run_id" + echo "cross-platform-run-id=$cross_platform_run_id" + echo "validation-run-id=$validation_run_id" + } >> "$GITHUB_OUTPUT" diff --git a/.github/actions/native-source-hash/action.yml b/.github/actions/native-source-hash/action.yml new file mode 100644 index 000000000..4b95925cf --- /dev/null +++ b/.github/actions/native-source-hash/action.yml @@ -0,0 +1,30 @@ +name: Compute native source hash +description: Hash every source, toolchain, and build-or-validation input that governs reusable native artifacts. + +outputs: + source-hash: + description: Stable 16-hex native source fingerprint + value: ${{ steps.compute.outputs.source-hash }} + +runs: + using: composite + steps: + - name: Compute native source hash + id: compute + shell: bash + run: | + set -euo pipefail + source_hash=$(find \ + crates \ + packages/natives/scripts \ + Cargo.toml Cargo.lock rust-toolchain.toml rustfmt.toml \ + packages/natives/package.json \ + scripts/ci-build-native.ts scripts/ci-target-cache.ts scripts/host-detect.ts scripts/run-rs-task.ts \ + .github/actions/build-native/action.yml .github/actions/native-source-hash/action.yml \ + -type f -print0 \ + | sort -z \ + | xargs -0 sha256sum \ + | sha256sum \ + | cut -c1-16) + echo "source-hash=$source_hash" >> "$GITHUB_OUTPUT" + echo "Native source hash: $source_hash" diff --git a/.github/actions/restore-linux-native/action.yml b/.github/actions/restore-linux-native/action.yml new file mode 100644 index 000000000..240faebb0 --- /dev/null +++ b/.github/actions/restore-linux-native/action.yml @@ -0,0 +1,54 @@ +name: Restore Linux x64 native addons +description: Restore both Linux x64 variants from the in-cluster cache or a trusted GitHub artifact run. + +inputs: + hash: + description: Native source hash embedded in artifact names + required: true + native-job-result: + description: Result of the current run's Linux x64 native matrix + required: true + cached-run-id: + description: Prior run containing both Linux x64 variants + required: false + default: "" + +runs: + using: composite + steps: + - name: Restore native addons from in-cluster cache + id: local + shell: bash + run: | + bun scripts/ci-native-artifact-cache.ts restore \ + "${{ inputs.hash }}" \ + packages/natives/native \ + "pi-natives-linux-x64-baseline-h${{ inputs.hash }}" \ + "pi-natives-linux-x64-modern-h${{ inputs.hash }}" + + - name: Resolve GitHub artifact run + if: steps.local.outputs.hit != 'true' + id: source + shell: bash + run: | + set -euo pipefail + if [ "${{ inputs.native-job-result }}" = "success" ]; then + run_id="${{ github.run_id }}" + else + run_id="${{ inputs.cached-run-id }}" + fi + if [ -z "$run_id" ]; then + echo "No Linux x64 native artifact source is available" >&2 + exit 1 + fi + echo "run-id=$run_id" >> "$GITHUB_OUTPUT" + + - name: Download native addons from GitHub + if: steps.local.outputs.hit != 'true' + uses: actions/download-artifact@v4 + with: + pattern: pi-natives-linux-x64-*-h${{ inputs.hash }} + path: packages/natives/native + merge-multiple: true + run-id: ${{ steps.source.outputs.run-id }} + github-token: ${{ github.token }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a19eae985..b50509fa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,12 +30,6 @@ concurrency: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - # glibc floor for shipped linux native addons. The omp-kata runner image is - # Ubuntu 24.04 (glibc 2.39); a plain native build links 2.39 symbol versions - # and fails to dlopen on older distros. Building the linux-gnu addons through - # cargo-zigbuild against this floor keeps them portable. Bump to raise the - # minimum supported glibc. - GLIBC_FLOOR: "2.17" # audiopus_sys bundles an opus tree whose CMakeLists declares a # cmake_minimum_required below 3.5; CMake 4.x refuses to configure it # without this override (macOS runner images ship CMake 4). @@ -96,18 +90,14 @@ jobs: echo "release-tag=$release_tag" } >> "$GITHUB_OUTPUT" - # Compute a stable hash of every input that affects the native cdylib output, - # then look for any prior successful main run that already uploaded the - # native artifacts for this hash. Two independent outputs: - # * `linux-x64-run-id` — set when the linux x64 canary - # (`pi-natives-linux-x64-modern-h`) is present on a prior main run, - # so native-dependent TS test jobs and `native_linux_x64` can reuse it. - # * `cross-platform-run-id` — set when ALL cross-platform native artifacts - # also have non-expired artifacts on that same prior run, so - # `native_cross_platform` can skip the cold rebuild on main pushes after - # dep changes have already warmed sccache there. - # Non-release native jobs are skipped when their canary hits; the canary - # retention window (see build-native action) is the effective TTL. + # Compute one native source hash, then validate complete artifact sets from + # any trusted main-branch run. Run conclusion is deliberately irrelevant: + # an upload proves that build step completed before a later job failed or a + # newer push canceled the workflow. + # + # Linux x64, the full cross-platform matrix, and Rust validation are tracked + # independently. Consumers either use a complete prior set or build the + # missing set in this run; no single canary can hide a partial matrix. native_artifact_lookup: name: Look up cached native artifacts runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-22.04' || 'omp-kata' }} @@ -115,92 +105,18 @@ jobs: source-hash: ${{ steps.compute.outputs.source-hash }} linux-x64-run-id: ${{ steps.find.outputs.linux-x64-run-id }} cross-platform-run-id: ${{ steps.find.outputs.cross-platform-run-id }} + validation-run-id: ${{ steps.find.outputs.validation-run-id }} steps: - uses: actions/checkout@v4 - name: Compute native source hash id: compute - shell: bash - run: | - source_hash=$(find crates Cargo.toml Cargo.lock rust-toolchain.toml \ - packages/natives/scripts packages/natives/package.json \ - scripts/ci-build-native.ts scripts/host-detect.ts \ - -type f -print0 \ - | sort -z \ - | xargs -0 sha256sum \ - | sha256sum \ - | cut -c1-16) - echo "source-hash=$source_hash" >> "$GITHUB_OUTPUT" - echo "Native source hash: $source_hash" - - name: Find prior main build with matching native artifacts + uses: ./.github/actions/native-source-hash + - name: Find trusted reusable artifacts id: find - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - hash="${{ steps.compute.outputs.source-hash }}" - # Canary for native_linux_x64: presence of the modern artifact - # implies the baseline sibling is also there (they upload from the - # same job). - linux_canary="pi-natives-linux-x64-modern-h${hash}" - # Required set for cross-platform reuse — names must match the - # `actions/upload-artifact` `name:` template in build-native action. - cross_platform_required=( - "pi-natives-linux-arm64-h${hash}" - "pi-natives-linux-musl-x64-baseline-h${hash}" - "pi-natives-linux-musl-arm64-h${hash}" - "pi-natives-darwin-x64-baseline-h${hash}" - "pi-natives-darwin-arm64-h${hash}" - "pi-natives-win32-x64-baseline-h${hash}" - ) - linux_x64_run_id="" - cross_platform_run_id="" - for candidate in $(gh run list \ - --workflow=ci.yml --branch=main --status=success --event=push \ - --limit=20 --json databaseId --jq='.[].databaseId'); do - names=$(gh api "/repos/${{ github.repository }}/actions/runs/$candidate/artifacts?per_page=100" \ - --jq '.artifacts[] | select(.expired == false) | .name') - if [ -z "$linux_x64_run_id" ] && echo "$names" | grep -qFx "$linux_canary"; then - linux_x64_run_id="$candidate" - fi - if [ -z "$cross_platform_run_id" ]; then - all_found=true - # Cross-platform reuse requires the linux canary AND every - # cross-platform artifact, since release_binary downloads them - # from the same run. - if ! echo "$names" | grep -qFx "$linux_canary"; then - all_found=false - else - for req in "${cross_platform_required[@]}"; do - if ! echo "$names" | grep -qFx "$req"; then - all_found=false - break - fi - done - fi - if $all_found; then - cross_platform_run_id="$candidate" - fi - fi - if [ -n "$linux_x64_run_id" ] && [ -n "$cross_platform_run_id" ]; then - break - fi - done - if [ -n "$linux_x64_run_id" ]; then - echo "Reusing Linux x64 native artifacts from run $linux_x64_run_id" - else - echo "No cached Linux x64 native artifacts for hash $hash; native_linux_x64 will rebuild." - fi - if [ -n "$cross_platform_run_id" ]; then - echo "Reusing cross-platform native artifacts from run $cross_platform_run_id" - else - echo "No cached cross-platform native artifacts for hash $hash; native_cross_platform will rebuild on main." - fi - { - echo "linux-x64-run-id=$linux_x64_run_id" - echo "cross-platform-run-id=$cross_platform_run_id" - } >> "$GITHUB_OUTPUT" + uses: ./.github/actions/find-native-artifacts + with: + hash: ${{ steps.compute.outputs.source-hash }} - # Fast lint, type check, and browser bundle build (no Rust, no native build needed) check: name: Lint, type check & web build runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-22.04' || 'omp-kata' }} @@ -212,22 +128,44 @@ jobs: - name: Build collab web run: bun run collab:web:build - # Linux x64 baseline + modern: required by `test`, so it runs on every PR - # unless native_artifact_lookup found a cached run. Release runs always - # rebuild for fresh artifacts but skip clippy + Rust tests (skip_validation): - # the bump commit only changes version strings over content that already - # passed validation on its main-push run. + rust_validation: + name: Validate Rust workspace + needs: [native_artifact_lookup] + if: ${{ needs.native_artifact_lookup.outputs.validation-run-id == '' }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-22.04' || 'omp-kata' }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-native + with: + hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} + platform: linux + arch: x64 + variant: baseline + rust_checks: "true" + skip_build: "true" + cache_scope: validation + save_cache: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + - name: Create validation marker + shell: bash + run: echo "${{ needs.native_artifact_lookup.outputs.source-hash }}" > "$RUNNER_TEMP/rust-validation" + - name: Upload validation marker + uses: actions/upload-artifact@v4 + with: + name: pi-natives-rust-validation-h${{ needs.native_artifact_lookup.outputs.source-hash }} + path: ${{ runner.temp }}/rust-validation + retention-days: 90 + + # Linux x64 baseline + modern supply the TS and install jobs. Rust validation + # is a separate parallel job, so both matrix entries are build-only. native_linux_x64: name: "Native: Linux x64 (${{ matrix.variant }})" - needs: [release_metadata, native_artifact_lookup] - if: ${{ needs.release_metadata.outputs.is-release == 'true' || needs.native_artifact_lookup.outputs.linux-x64-run-id == '' }} + needs: [native_artifact_lookup] + if: ${{ needs.native_artifact_lookup.outputs.linux-x64-run-id == '' }} runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-22.04' || 'omp-kata' }} strategy: fail-fast: false matrix: - include: - - { variant: baseline, rust_checks: true } - - { variant: modern } + variant: [baseline, modern] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/build-native @@ -236,18 +174,15 @@ jobs: platform: linux arch: x64 variant: ${{ matrix.variant }} - glibc: ${{ env.GLIBC_FLOOR }} - rust_checks: ${{ matrix.rust_checks && 'true' || 'false' }} - skip_validation: ${{ needs.release_metadata.outputs.is-release }} + skip_validation: "true" save_cache: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - # Pre-warm the cross-platform native build cache on `main`, in addition to - # building the artifacts that ship in releases. Skipped on main when - # native_artifact_lookup already found a recent run with all artifacts intact. + # Cross-platform builds stay in this workflow only as a release fallback. + # Successful main CI runs launch the non-blocking native-prewarm workflow. native_cross_platform_kata: name: "Native: ${{ matrix.platform }} ${{ matrix.libc || '' }} ${{ matrix.arch }}" needs: [release_metadata, native_artifact_lookup] - if: ${{ needs.release_metadata.outputs.is-release == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.native_artifact_lookup.outputs.cross-platform-run-id == '') }} + if: ${{ needs.release_metadata.outputs.is-release == 'true' && needs.native_artifact_lookup.outputs.cross-platform-run-id == '' }} strategy: fail-fast: false matrix: @@ -267,18 +202,18 @@ jobs: libc: ${{ matrix.libc }} variant: ${{ matrix.variant }} target: ${{ matrix.target }} - glibc: ${{ matrix.platform == 'linux' && matrix.libc != 'musl' && env.GLIBC_FLOOR || '' }} + skip_validation: "true" save_cache: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} native_cross_platform_macos: name: "Native: ${{ matrix.platform }} ${{ matrix.arch }}" needs: [release_metadata, native_artifact_lookup] - if: ${{ needs.release_metadata.outputs.is-release == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.native_artifact_lookup.outputs.cross-platform-run-id == '') }} + if: ${{ needs.release_metadata.outputs.is-release == 'true' && needs.native_artifact_lookup.outputs.cross-platform-run-id == '' }} strategy: fail-fast: false matrix: include: - - { os: macos-15-intel, platform: darwin, arch: x64, variant: baseline } + - { os: macos-14, platform: darwin, arch: x64, target: x86_64-apple-darwin, variant: baseline } - { os: macos-14, platform: darwin, arch: arm64 } runs-on: ${{ matrix.os }} steps: @@ -290,7 +225,9 @@ jobs: arch: ${{ matrix.arch }} variant: ${{ matrix.variant }} target: ${{ matrix.target }} + skip_validation: "true" save_cache: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + test_workspace: name: Test TS workspace fast runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-22.04' || 'omp-kata' }} @@ -301,24 +238,14 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-system-deps - uses: ./.github/actions/bun-install - - name: Resolve Linux x64 native artifact run - id: source - shell: bash - run: | - if [ "${{ needs.native_linux_x64.result }}" = "success" ]; then - echo "artifact-run-id=${{ github.run_id }}" >> "$GITHUB_OUTPUT" - else - echo "artifact-run-id=${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }}" >> "$GITHUB_OUTPUT" - fi - - name: Download native addons - uses: actions/download-artifact@v4 + - uses: ./.github/actions/restore-linux-native with: - pattern: pi-natives-linux-x64-*-h${{ needs.native_artifact_lookup.outputs.source-hash }} - path: packages/natives/native - merge-multiple: true - run-id: ${{ steps.source.outputs.artifact-run-id }} - github-token: ${{ secrets.GITHUB_TOKEN }} + hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} + native-job-result: ${{ needs.native_linux_x64.result }} + cached-run-id: ${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }} - name: Test workspace packages and repo scripts (TS) + env: + OMP_TEST_CONCURRENCY: "4" run: bun run ci:test:ts:workspace test_coding_agent_singleton: @@ -331,23 +258,11 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-system-deps - uses: ./.github/actions/bun-install - - name: Resolve Linux x64 native artifact run - id: source - shell: bash - run: | - if [ "${{ needs.native_linux_x64.result }}" = "success" ]; then - echo "artifact-run-id=${{ github.run_id }}" >> "$GITHUB_OUTPUT" - else - echo "artifact-run-id=${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }}" >> "$GITHUB_OUTPUT" - fi - - name: Download native addons - uses: actions/download-artifact@v4 + - uses: ./.github/actions/restore-linux-native with: - pattern: pi-natives-linux-x64-*-h${{ needs.native_artifact_lookup.outputs.source-hash }} - path: packages/natives/native - merge-multiple: true - run-id: ${{ steps.source.outputs.artifact-run-id }} - github-token: ${{ secrets.GITHUB_TOKEN }} + hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} + native-job-result: ${{ needs.native_linux_x64.result }} + cached-run-id: ${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }} - name: Test coding-agent singleton/global-state bucket # Keep global Settings/env/fake-timer tests serial; native addon # artifacts are still available like every other coding-agent bucket. @@ -364,24 +279,14 @@ jobs: - uses: ./.github/actions/setup-system-deps - uses: ./.github/actions/bun-install - - name: Resolve Linux x64 native artifact run - id: source - shell: bash - run: | - if [ "${{ needs.native_linux_x64.result }}" = "success" ]; then - echo "artifact-run-id=${{ github.run_id }}" >> "$GITHUB_OUTPUT" - else - echo "artifact-run-id=${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }}" >> "$GITHUB_OUTPUT" - fi - - name: Download native addons - uses: actions/download-artifact@v4 + - uses: ./.github/actions/restore-linux-native with: - pattern: pi-natives-linux-x64-*-h${{ needs.native_artifact_lookup.outputs.source-hash }} - path: packages/natives/native - merge-multiple: true - run-id: ${{ steps.source.outputs.artifact-run-id }} - github-token: ${{ secrets.GITHUB_TOKEN }} + hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} + native-job-result: ${{ needs.native_linux_x64.result }} + cached-run-id: ${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }} - name: Test native/TUI/browser-ish packages (TS) + env: + OMP_TEST_CONCURRENCY: "4" run: bun run ci:test:ts:native test_coding_agent_ui: @@ -395,24 +300,14 @@ jobs: - uses: ./.github/actions/setup-system-deps - uses: ./.github/actions/bun-install - - name: Resolve Linux x64 native artifact run - id: source - shell: bash - run: | - if [ "${{ needs.native_linux_x64.result }}" = "success" ]; then - echo "artifact-run-id=${{ github.run_id }}" >> "$GITHUB_OUTPUT" - else - echo "artifact-run-id=${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }}" >> "$GITHUB_OUTPUT" - fi - - name: Download native addons - uses: actions/download-artifact@v4 + - uses: ./.github/actions/restore-linux-native with: - pattern: pi-natives-linux-x64-*-h${{ needs.native_artifact_lookup.outputs.source-hash }} - path: packages/natives/native - merge-multiple: true - run-id: ${{ steps.source.outputs.artifact-run-id }} - github-token: ${{ secrets.GITHUB_TOKEN }} + hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} + native-job-result: ${{ needs.native_linux_x64.result }} + cached-run-id: ${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }} - name: Test coding-agent UI/TUI bucket + env: + OMP_TEST_CONCURRENCY: "2" run: bun run ci:test:coding-agent:ui test_coding_agent_runtime: @@ -425,26 +320,16 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-system-deps - uses: ./.github/actions/bun-install - - name: Resolve Linux x64 native artifact run - id: source - shell: bash - run: | - if [ "${{ needs.native_linux_x64.result }}" = "success" ]; then - echo "artifact-run-id=${{ github.run_id }}" >> "$GITHUB_OUTPUT" - else - echo "artifact-run-id=${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }}" >> "$GITHUB_OUTPUT" - fi - - name: Download native addons - uses: actions/download-artifact@v4 + - uses: ./.github/actions/restore-linux-native with: - pattern: pi-natives-linux-x64-*-h${{ needs.native_artifact_lookup.outputs.source-hash }} - path: packages/natives/native - merge-multiple: true - run-id: ${{ steps.source.outputs.artifact-run-id }} - github-token: ${{ secrets.GITHUB_TOKEN }} + hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} + native-job-result: ${{ needs.native_linux_x64.result }} + cached-run-id: ${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }} - name: Test coding-agent runtime bucket # Runtime/session tests import native-backed barrels too; keep this # separate for concurrency, not as a native-free guardrail. + env: + OMP_TEST_CONCURRENCY: "4" run: bun run ci:test:coding-agent:runtime test_coding_agent_native: @@ -458,24 +343,14 @@ jobs: - uses: ./.github/actions/setup-system-deps - uses: ./.github/actions/bun-install - - name: Resolve Linux x64 native artifact run - id: source - shell: bash - run: | - if [ "${{ needs.native_linux_x64.result }}" = "success" ]; then - echo "artifact-run-id=${{ github.run_id }}" >> "$GITHUB_OUTPUT" - else - echo "artifact-run-id=${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }}" >> "$GITHUB_OUTPUT" - fi - - name: Download native addons - uses: actions/download-artifact@v4 + - uses: ./.github/actions/restore-linux-native with: - pattern: pi-natives-linux-x64-*-h${{ needs.native_artifact_lookup.outputs.source-hash }} - path: packages/natives/native - merge-multiple: true - run-id: ${{ steps.source.outputs.artifact-run-id }} - github-token: ${{ secrets.GITHUB_TOKEN }} + hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} + native-job-result: ${{ needs.native_linux_x64.result }} + cached-run-id: ${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }} - name: Test coding-agent native/unit bucket + env: + OMP_TEST_CONCURRENCY: "4" run: bun run ci:test:coding-agent:native test_smoke: @@ -489,84 +364,41 @@ jobs: - uses: ./.github/actions/setup-system-deps - uses: ./.github/actions/bun-install - - name: Resolve Linux x64 native artifact run - id: source - shell: bash - run: | - if [ "${{ needs.native_linux_x64.result }}" = "success" ]; then - echo "artifact-run-id=${{ github.run_id }}" >> "$GITHUB_OUTPUT" - else - echo "artifact-run-id=${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }}" >> "$GITHUB_OUTPUT" - fi - - name: Download native addons - uses: actions/download-artifact@v4 + - uses: ./.github/actions/restore-linux-native with: - pattern: pi-natives-linux-x64-*-h${{ needs.native_artifact_lookup.outputs.source-hash }} - path: packages/natives/native - merge-multiple: true - run-id: ${{ steps.source.outputs.artifact-run-id }} - github-token: ${{ secrets.GITHUB_TOKEN }} + hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} + native-job-result: ${{ needs.native_linux_x64.result }} + cached-run-id: ${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }} - name: CLI smoke test run: bun run ci:test:smoke install_methods: name: Install method smoke tests runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-22.04' || 'omp-kata' }} + needs: [native_linux_x64, native_artifact_lookup] + if: ${{ !cancelled() && needs.native_linux_x64.result != 'failure' }} steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/ensure-rust-toolchain - with: - toolchain: nightly-2026-04-29 - - uses: ./.github/actions/ensure-cmake - - name: Detect runner environment - id: detect - shell: bash - run: | - # $SCCACHE_BUCKET is injected only on self-hosted omp-kata pods; its - # presence selects baked sccache + shared S3. GitHub-hosted runners - # (PRs) need sccache-action to export the GHA cache URL/token into the - # step env — a bare binary install leaves SCCACHE_GHA_ENABLED set with - # no cache URL, so sccache server startup fails ("cache url for ghac - # not found"). Mirrors the build-native action's sccache wiring. - if [ -n "${SCCACHE_BUCKET:-}" ]; then - echo "on_infra=true" >> "$GITHUB_OUTPUT" - else - echo "on_infra=false" >> "$GITHUB_OUTPUT" - fi - - name: Ensure baked sccache (omp-kata) - if: steps.detect.outputs.on_infra == 'true' - uses: ./.github/actions/ensure-sccache - with: - version: "0.15.0" - - name: Setup sccache (GitHub-hosted) - if: steps.detect.outputs.on_infra == 'false' - uses: mozilla-actions/sccache-action@v0.0.10 - - name: Enable sccache for cargo - # Conditional backend: self-hosted omp-kata injects a shared S3 - # (RustFS) sccache via pod env; GitHub-hosted runners keep the GHA - # cache. CARGO_INCREMENTAL=0 keeps sccache from silently no-oping. - shell: bash - run: | - { - echo "RUSTC_WRAPPER=sccache" - echo "CARGO_INCREMENTAL=0" - } >> "$GITHUB_ENV" - if [ -n "${SCCACHE_BUCKET:-}" ]; then - echo "sccache backend: shared S3 ($SCCACHE_BUCKET @ $SCCACHE_ENDPOINT)" - else - echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" - echo "sccache backend: GitHub Actions cache" - fi - uses: ./.github/actions/setup-system-deps - uses: ./.github/actions/bun-install + - uses: ./.github/actions/restore-linux-native + with: + hash: ${{ needs.native_artifact_lookup.outputs.source-hash }} + native-job-result: ${{ needs.native_linux_x64.result }} + cached-run-id: ${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }} - name: Install method smoke tests + env: + OMP_INSTALL_TEST_SKIP_NATIVE_BUILD: "1" run: bun run ci:test:install-methods + release_binary: name: "Release binary: ${{ matrix.target_id }}" if: ${{ needs.release_metadata.outputs.is-release == 'true' && !cancelled() && - needs.native_linux_x64.result == 'success' && needs.native_cross_platform_kata.result == - 'success' && needs.native_cross_platform_macos.result == 'success' && + needs.rust_validation.result != 'failure' && + needs.native_linux_x64.result != 'failure' && + needs.native_cross_platform_kata.result != 'failure' && + needs.native_cross_platform_macos.result != 'failure' && needs.test_workspace.result == 'success' && needs.test_coding_agent_singleton.result == 'success' && needs.test_ts_native.result == 'success' && @@ -575,7 +407,7 @@ jobs: needs.test_coding_agent_native.result == 'success' && needs.test_smoke.result == 'success' && needs.check.result == 'success' && needs.install_methods.result == 'success' }} - needs: [release_metadata, check, native_linux_x64, native_cross_platform_kata, native_cross_platform_macos, test_workspace, test_coding_agent_singleton, test_ts_native, test_coding_agent_ui, test_coding_agent_runtime, test_coding_agent_native, test_smoke, install_methods, native_artifact_lookup] + needs: [release_metadata, check, rust_validation, native_linux_x64, native_cross_platform_kata, native_cross_platform_macos, test_workspace, test_coding_agent_singleton, test_ts_native, test_coding_agent_ui, test_coding_agent_runtime, test_coding_agent_native, test_smoke, install_methods, native_artifact_lookup] strategy: fail-fast: false matrix: @@ -641,6 +473,7 @@ jobs: runs-on: ${{ matrix.os }} permissions: contents: read + actions: read id-token: write env: MACOS_SIGNING: ${{ secrets.APPLE_CERTIFICATE_P12 != '' && secrets.APPLE_CERTIFICATE_PASSWORD != '' && secrets.APPLE_API_KEY_ID != '' && secrets.APPLE_API_ISSUER_ID != '' && secrets.APPLE_API_KEY != '' }} @@ -667,12 +500,36 @@ jobs: path: ~/.bun/install/cache key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }} - run: bun install --frozen-lockfile + - name: Resolve native artifact run + id: native-source + shell: bash + run: | + set -euo pipefail + if [ "${{ matrix.target_id }}" = "linux-x64" ]; then + if [ "${{ needs.native_linux_x64.result }}" = "success" ]; then + run_id="${{ github.run_id }}" + else + run_id="${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }}" + fi + elif [ "${{ needs.native_cross_platform_kata.result }}" = "success" ] || \ + [ "${{ needs.native_cross_platform_macos.result }}" = "success" ]; then + run_id="${{ github.run_id }}" + else + run_id="${{ needs.native_artifact_lookup.outputs.cross-platform-run-id }}" + fi + if [ -z "$run_id" ]; then + echo "No native artifact run for ${{ matrix.target_id }}" >&2 + exit 1 + fi + echo "run-id=$run_id" >> "$GITHUB_OUTPUT" - name: Download native addon(s) uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: ${{ matrix.native_artifact_pattern }}-h${{ needs.native_artifact_lookup.outputs.source-hash }} path: packages/natives/native merge-multiple: true + run-id: ${{ steps.native-source.outputs.run-id }} + github-token: ${{ github.token }} - name: Build release binary env: RELEASE_TARGETS: ${{ matrix.target_id }} @@ -806,7 +663,7 @@ jobs: needs.release_binary.result == 'success' && needs.release_github_verify.result == 'success' && !inputs.skip_npm }} - needs: [release_metadata, release_binary, release_github_verify, native_artifact_lookup] + needs: [release_metadata, release_binary, release_github_verify, native_linux_x64, native_artifact_lookup] runs-on: ubuntu-22.04 # `id-token: write` lets npm mint the GitHub OIDC token it exchanges for a # short-lived publish token (trusted publishing + provenance). When a @@ -815,6 +672,7 @@ jobs: permissions: id-token: write contents: read + actions: read steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 @@ -836,15 +694,24 @@ jobs: - run: bun install --frozen-lockfile # The pi-coding-agent prepack executes workspace code (bundle-dist # imports the pi-utils barrel, which loads the pi-natives addon), so - # this job needs the linux x64 native addons just like `test` does. - # Release runs always rebuild natives in this same run, so the - # default run-id resolves the artifacts. + # this job needs the Linux x64 native addons just like TS tests do. + - name: Resolve Linux x64 native artifact run + id: native-source + shell: bash + run: | + if [ "${{ needs.native_linux_x64.result }}" = "success" ]; then + echo "run-id=${{ github.run_id }}" >> "$GITHUB_OUTPUT" + else + echo "run-id=${{ needs.native_artifact_lookup.outputs.linux-x64-run-id }}" >> "$GITHUB_OUTPUT" + fi - name: Download native addons uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: pi-natives-linux-x64-*-h${{ needs.native_artifact_lookup.outputs.source-hash }} path: packages/natives/native merge-multiple: true + run-id: ${{ steps.native-source.outputs.run-id }} + github-token: ${{ github.token }} - name: Publish to npm env: # Fallback auth: setup-node wrote an .npmrc referencing diff --git a/.github/workflows/native-prewarm.yml b/.github/workflows/native-prewarm.yml new file mode 100644 index 000000000..0e20c7a33 --- /dev/null +++ b/.github/workflows/native-prewarm.yml @@ -0,0 +1,90 @@ +name: Native prewarm + +on: + workflow_run: + workflows: [CI] + types: [completed] + workflow_dispatch: +# Prewarming must never extend the required CI workflow. Keep one warmup running +# to completion so it publishes the target snapshot; GitHub coalesces newer +# pending runs in this concurrency group. +concurrency: + group: native-prewarm-main + cancel-in-progress: false + +permissions: + actions: read + contents: read + +jobs: + lookup: + name: Look up cross-platform artifacts + if: ${{ github.event_name == 'workflow_dispatch' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main') }} + runs-on: omp-kata + outputs: + source-hash: ${{ steps.compute.outputs.source-hash }} + cross-platform-run-id: ${{ steps.find.outputs.cross-platform-run-id }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + - id: compute + uses: ./.github/actions/native-source-hash + - id: find + uses: ./.github/actions/find-native-artifacts + with: + hash: ${{ steps.compute.outputs.source-hash }} + + cross_platform_kata: + name: "Prewarm native: ${{ matrix.platform }} ${{ matrix.libc || '' }} ${{ matrix.arch }}" + needs: [lookup] + if: ${{ needs.lookup.outputs.cross-platform-run-id == '' }} + strategy: + fail-fast: false + max-parallel: 2 + matrix: + include: + - { platform: linux, arch: arm64, target: aarch64-unknown-linux-gnu } + - { platform: linux, libc: musl, arch: x64, target: x86_64-unknown-linux-musl, variant: baseline } + - { platform: linux, libc: musl, arch: arm64, target: aarch64-unknown-linux-musl } + - { platform: win32, arch: x64, target: x86_64-pc-windows-msvc, variant: baseline } + runs-on: omp-kata + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + - uses: ./.github/actions/build-native + with: + hash: ${{ needs.lookup.outputs.source-hash }} + platform: ${{ matrix.platform }} + libc: ${{ matrix.libc }} + arch: ${{ matrix.arch }} + variant: ${{ matrix.variant }} + target: ${{ matrix.target }} + skip_validation: "true" + save_cache: "true" + + cross_platform_macos: + name: "Prewarm native: darwin ${{ matrix.arch }}" + needs: [lookup] + if: ${{ needs.lookup.outputs.cross-platform-run-id == '' }} + strategy: + fail-fast: false + matrix: + include: + - { os: macos-14, arch: x64, target: x86_64-apple-darwin, variant: baseline } + - { os: macos-14, arch: arm64 } + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + - uses: ./.github/actions/build-native + with: + hash: ${{ needs.lookup.outputs.source-hash }} + platform: darwin + arch: ${{ matrix.arch }} + variant: ${{ matrix.variant }} + target: ${{ matrix.target }} + skip_validation: "true" + save_cache: "true" diff --git a/infra/docs/04-arc-and-caching.md b/infra/docs/04-arc-and-caching.md index 8fc52bd5a..fc84ae179 100644 --- a/infra/docs/04-arc-and-caching.md +++ b/infra/docs/04-arc-and-caching.md @@ -161,63 +161,46 @@ githubConfigUrl: "https://github.com//" githubConfigSecret: arc-github runnerScaleSetName: omp-kata minRunners: 0 -maxRunners: 10 +maxRunners: 4 # none: each job runs inside the runner container, which itself lives in a Kata microVM containerMode: type: "" template: spec: - runtimeClassName: kata-qemu # <-- every runner pod boots its own KVM microVM + runtimeClassName: kata-qemu securityContext: - # ghcr.io/actions/actions-runner runs jobs as uid/gid 1001 ("runner"). - # Let kubelet make the PVC writable by that user without changing image-owned - # ~/.cargo/bin or ~/.rustup. fsGroup: 1001 fsGroupChangePolicy: OnRootMismatch - initContainers: - - name: prepare-runner-cache - image: omp-kata-runner:2026-06-15-002621 - imagePullPolicy: IfNotPresent - command: - - bash - - -lc - - install -d -o 1001 -g 1001 -m 2775 /cache/bun-store /cache/cargo-registry - securityContext: - runAsUser: 0 - volumeMounts: - - name: runner-cache - mountPath: /cache containers: - name: runner - # Preloaded image: stock ghcr.io/actions/actions-runner + CI deps baked in - # (apt cairo/pango/jpeg/gif/rsvg stack, fd/ripgrep/imagemagick, bun, rust - # nightly + clippy/rustfmt + arm64/msvc targets). Built + imported locally; - # see /root/omp-kata-runner-image/. IfNotPresent uses the local image. - image: omp-kata-runner:2026-06-15-002621 + image: omp-kata-runner:2026-07-27-072222 imagePullPolicy: IfNotPresent command: ["/home/runner/run.sh"] - # Shared sccache backend (in-cluster RustFS S3). Exposes SCCACHE_BUCKET/ - # ENDPOINT/REGION/USE_SSL + AWS creds to every job; CI flips RUSTC_WRAPPER - # on for rust builds only. GitHub-hosted runners lack this env and keep the - # GHA cache backend. See /root/sccache-rustfs/. envFrom: - secretRef: name: sccache-s3 + env: + - name: OMP_NATIVE_CACHE_DIR + value: /home/runner/.cache/omp-native-artifacts volumeMounts: - # Shared stores only. Keep node_modules, Cargo target/, and Cargo git - # checkouts per-job to avoid mutable build-output or checkout poisoning. - name: runner-cache mountPath: /home/runner/.bun/install/cache subPath: bun-store - name: runner-cache - mountPath: /home/runner/.cargo/registry - subPath: cargo-registry + mountPath: /home/runner/.cargo/registry/cache + subPath: cargo-registry/cache + - name: runner-cache + mountPath: /home/runner/.cargo/registry/index + subPath: cargo-registry/index + - name: runner-cache + mountPath: /home/runner/.cache/omp-native-artifacts + subPath: native-artifacts resources: requests: - cpu: "2" - memory: "4Gi" + cpu: "8" + memory: "12Gi" limits: - cpu: "16" + cpu: "8" memory: "12Gi" volumes: - name: runner-cache @@ -232,10 +215,10 @@ Field by field: - **`githubConfigSecret: arc-github`** - the auth secret from [step 1](#1-github-app-and-the-arc-github-secret). - **`runnerScaleSetName: omp-kata`** - the runner label. This is the string that goes in a workflow's `runs-on:`. -- **`minRunners: 0` / `maxRunners: 10`** - **scale-to-zero**. With no queued jobs - there are zero runner pods (and zero microVMs) consuming the node; the listener - scales up to ten concurrent runners on demand. (The older ops notes capped this - at 3; the live value is 10.) +- **`minRunners: 0` / `maxRunners: 4`** - **scale-to-zero**. With no queued jobs + there are zero runner microVMs. Each admitted runner gets an honest 8-vCPU, + 12-GiB request and limit; excess jobs queue instead of ten 16-vCPU guests + fighting over the reference host's 32 physical CPUs. - **`containerMode.type: ""`** - **none**. The default chart offers `dind` (Docker-in-Docker sidecar) or `kubernetes` mode for job-container isolation; both are unnecessary here because the *whole runner pod* is already isolated in @@ -509,20 +492,25 @@ incremental enabled). The sccache backend is conditional: - otherwise (GitHub-hosted) - it installs the toolchains, exports `SCCACHE_GHA_ENABLED=true`, and uses the **GitHub Actions cache**. -`Swatinem/rust-cache` runs only on GitHub-hosted runners (it caches Cargo -`target/`). On omp-kata the mounted Cargo registry handles crate downloads and -sccache fills the compile-output gap when `target/` is cold. +`Swatinem/rust-cache` keeps Cargo registry/tool data on GitHub-hosted runners. +An explicit rolling `actions/cache` entry restores and saves `target/` across +source hashes, which is essential for the slower Intel macOS build. On omp-kata, +[`scripts/ci-target-cache.ts`](../../scripts/ci-target-cache.ts) stores the +rolling `target/` snapshot in RustFS while sccache covers individual compiler +outputs. -**(b) Cargo registry cache** - the scale-set pod template mounts -`runner-cache:/cargo-registry` at `/home/runner/.cargo/registry`. Cargo uses it -automatically because the image keeps `CARGO_HOME=/home/runner/.cargo`. +**(b) Cargo registry cache** - the scale-set pod template mounts only the +immutable download cache and sparse index at +`/home/runner/.cargo/registry/cache` and `/home/runner/.cargo/registry/index`. +Source extraction, lock files, Cargo git checkouts, and `target/` remain +job-local; virtio-fs does not propagate Cargo's file locks safely across VMs. -Only the registry cache is shared. Cargo `target/` stays per-job, and -`/home/runner/.cargo/git` stays per-job too; this repo has no git dependencies, -and git checkouts are a worse shared mutable-cache boundary than crates.io -archives with lockfile checksums. +**(c) Native addon artifacts** - completed `.node` outputs are copied atomically +to the runner-cache PVC under their native source hash. Native-dependent jobs on +omp-kata restore both Linux x64 variants locally; GitHub-hosted jobs fall back to +the trusted Actions artifact run. -**(c) Bun package store** - +**(d) Bun package store** - [`.github/actions/bun-install`](../../.github/actions/bun-install/action.yml) wraps `bun install --frozen-lockfile`. On omp-kata, the pod template mounts `runner-cache:/bun-store` at Bun's default store path diff --git a/infra/docs/README.md b/infra/docs/README.md index b7994f729..98eaf30a6 100644 --- a/infra/docs/README.md +++ b/infra/docs/README.md @@ -39,7 +39,7 @@ flowchart LR SEC -.->|"envFrom"| POD NP -.->|"filters egress"| POD JOB -->|"sccache (S3 SigV4)"| RUSTFS - JOB -->|"Bun store + Cargo registry mounts"| PVC + JOB -->|"Bun/Cargo stores + native artifacts"| PVC POD -->|"allowed egress"| NAT NAT -->|"checkout / API / internet"| GH ``` @@ -47,10 +47,10 @@ flowchart LR Key properties baked into this design: - **One job = one VM.** Runner pods are ephemeral and JIT-registered; there is no VM templating or pooling, so a job never inherits state from a previous job. -- **Scale-to-zero.** `minRunners: 0` / `maxRunners: 10` — when no jobs are queued, zero runner pods (and zero microVMs) exist. +- **Scale-to-zero.** `minRunners: 0` / `maxRunners: 4` — when no jobs are queued, zero runner pods (and zero microVMs) exist. - **Host-kernel isolation.** Jobs see the microVM's guest kernel, not the host kernel, so a kernel exploit in a job does not reach the host. - **No external registry.** The runner image is built on the host and imported straight into k3s' containerd. -- **Shared, in-cluster cache.** `sccache` targets RustFS over the cluster network; Bun's package store and Cargo's registry cache are mounted from the runner cache PVC. Cache traffic stays on the host. +- **Shared, in-cluster cache.** RustFS stores sccache outputs and rolling Cargo target snapshots; the runner-cache PVC stores Bun/Cargo downloads and source-hash-addressed native addons. Cache traffic stays on the host. ## End-to-end job lifecycle @@ -59,7 +59,7 @@ Key properties baked into this design: 3. The listener signals demand to the **ARC controller**, which scales the **EphemeralRunnerSet** up by one. 4. The controller creates a single **JIT-registered ephemeral runner pod** in `arc-runners`, with `runtimeClassName: kata-qemu` and the `sccache-s3` secret injected via `envFrom`. 5. containerd hands the pod to the Kata shim, which **boots a fresh QEMU/KVM microVM** (own guest kernel; the container rootfs is shared in over virtio-fs). No templating — every job gets a clean VM. -6. The runner agent inside the microVM **registers just-in-time and picks up exactly one job**. Steps run isolated from the host, using RustFS over S3 for `sccache`, mounted PVC paths for Bun/Cargo package caches, and NAT egress for the public internet, all constrained by the `runner-egress-lockdown` NetworkPolicy. +6. The runner agent inside the microVM **registers just-in-time and picks up exactly one job**. Steps run isolated from the host, using RustFS for Rust compilation caches, mounted PVC paths for package/native-artifact caches, and NAT egress for the public internet, all constrained by the `runner-egress-lockdown` NetworkPolicy. 7. The job finishes; the ephemeral runner **deregisters and the pod (and its microVM) is destroyed** — never reused. 8. When no jobs remain queued, the EphemeralRunnerSet **scales back to zero**, leaving no idle runners or VMs. @@ -71,7 +71,7 @@ Key properties baked into this design: | Kata Containers runtime | QEMU/KVM microVM runtime: containerd drop-in registering `kata-qemu` + the `kata-qemu` RuntimeClass | Kata `3.31.0` | [02-kata-runtime.md](02-kata-runtime.md) | | Preloaded runner image | Custom `actions/runner` image (build toolchain, Bun, Rust nightly + cross targets, native-build deps) built on the host and imported into k3s containerd — no registry | local dated tag | [03-runner-image.md](03-runner-image.md) | | ARC (runner scale set) | actions-runner-controller, `gha-runner-scale-set` flavor: controller in `arc-systems`, one scale set + listener, GitHub App auth | ARC `0.14.2` | [04-arc-and-caching.md](04-arc-and-caching.md) | -| Shared caches | RustFS S3 (`svc rustfs:9000`, 100Gi PVC) backs `sccache`; `arc-runners/runner-cache` (100Gi PVC) mounts Bun's package store and Cargo's registry cache into runner pods; the `sccache-s3` secret and egress NetworkPolicy wire access | in-cluster services/storage | [04-arc-and-caching.md](04-arc-and-caching.md) | +| Shared caches | RustFS S3 (`svc rustfs:9000`, 100Gi PVC) backs sccache and rolling Cargo target snapshots; `arc-runners/runner-cache` (100Gi PVC) holds Bun/Cargo downloads and immutable native addons; the `sccache-s3` secret and egress NetworkPolicy wire access | in-cluster services/storage | [04-arc-and-caching.md](04-arc-and-caching.md) | ## Prerequisites diff --git a/infra/reload-runner.sh b/infra/reload-runner.sh index 3b53733b6..c84756bdc 100755 --- a/infra/reload-runner.sh +++ b/infra/reload-runner.sh @@ -32,6 +32,9 @@ # CONTAINERD_SOCKET_REMOTE remote containerd socket [/run/k3s/containerd/containerd.sock] # NERDCTL_VERSION nerdctl release to bootstrap on demand [2.1.6] # BUILDKIT_VERSION BuildKit release to bootstrap on demand [0.25.1] +# RUNNER_MAX_RUNNERS maximum concurrent Kata runner pods [4] +# RUNNER_CPU requested and limited CPU cores per runner [8] +# RUNNER_MEMORY requested and limited memory per runner [12Gi] set -euo pipefail : "${CI_HOST:?set CI_HOST to the ssh target of your CI host, e.g. CI_HOST=my-ci-host}" @@ -45,6 +48,9 @@ BUILD_BACKEND="${BUILD_BACKEND:-auto}" CONTAINERD_SOCKET_REMOTE="${CONTAINERD_SOCKET_REMOTE:-/run/k3s/containerd/containerd.sock}" NERDCTL_VERSION="${NERDCTL_VERSION:-2.1.6}" BUILDKIT_VERSION="${BUILDKIT_VERSION:-0.25.1}" +RUNNER_MAX_RUNNERS="${RUNNER_MAX_RUNNERS:-4}" +RUNNER_CPU="${RUNNER_CPU:-8}" +RUNNER_MEMORY="${RUNNER_MEMORY:-12Gi}" arg="${1:-$(date +%Y-%m-%d-%H%M%S)}" case "$arg" in *:*) IMAGE="$arg";; *) IMAGE="omp-kata-runner:$arg";; esac @@ -61,11 +67,13 @@ scp -q "$here/runner.Dockerfile" "${CI_HOST}:${REMOTE_CTX}/Dockerfile" # regardless of the host's login shell. ssh "$CI_HOST" bash -s -- \ "$IMAGE" "$REMOTE_CTX" "$ARC_VALUES" "$ARC_RELEASE" "$ARC_NAMESPACE" "$ARC_CHART_VERSION" \ - "$KUBECONFIG_REMOTE" "$BUILD_BACKEND" "$CONTAINERD_SOCKET_REMOTE" "$NERDCTL_VERSION" "$BUILDKIT_VERSION" <<'REMOTE' + "$KUBECONFIG_REMOTE" "$BUILD_BACKEND" "$CONTAINERD_SOCKET_REMOTE" "$NERDCTL_VERSION" "$BUILDKIT_VERSION" \ + "$RUNNER_MAX_RUNNERS" "$RUNNER_CPU" "$RUNNER_MEMORY" <<'REMOTE' set -euo pipefail IMAGE="$1"; REMOTE_CTX="$2"; ARC_VALUES="$3"; ARC_RELEASE="$4"; ARC_NAMESPACE="$5"; ARC_CHART_VERSION="$6" export KUBECONFIG="$7" BUILD_BACKEND="$8"; CONTAINERD_SOCKET="$9"; NERDCTL_VERSION="${10}"; BUILDKIT_VERSION="${11}" +RUNNER_MAX_RUNNERS="${12}"; RUNNER_CPU="${13}"; RUNNER_MEMORY="${14}" cd "$REMOTE_CTX" TOOLS_DIR="$REMOTE_CTX/.containerd-build-tools" @@ -82,6 +90,8 @@ BUILDKITD_PID="" cleanup_buildkitd() { if [ -n "${BUILDKITD_PID:-}" ]; then kill "$BUILDKITD_PID" >/dev/null 2>&1 || true + wait "$BUILDKITD_PID" >/dev/null 2>&1 || true + rm -f "$RUN_DIR/buildkitd.sock" fi } trap cleanup_buildkitd EXIT @@ -116,7 +126,13 @@ bootstrap_containerd_tools() { } start_buildkitd() { - rm -f "$RUN_DIR/buildkitd.sock" + if [ -S "$RUN_DIR/buildkitd.sock" ]; then + if "$BUILDKITCTL_BIN" --addr "$BUILDKIT_ADDR" debug workers >/dev/null 2>&1; then + echo "==> reusing running BuildKit daemon" + return 0 + fi + rm -f "$RUN_DIR/buildkitd.sock" + fi "$BUILDKITD_BIN" \ --addr "$BUILDKIT_ADDR" \ --root "$ROOT_DIR" \ @@ -138,10 +154,10 @@ verify_baked_tools() { local runner="$1" "$runner" --namespace k8s.io run --rm --entrypoint bash "$IMAGE" -lc ' set -e - for b in gh fd rg magick bun cargo rustc pkg-config clang lld sccache zig cmake ninja cargo-nextest cargo-zigbuild cargo-xwin; do + for b in gh fd rg magick bun cargo rustc pkg-config clang lld sccache zstd zig cmake ninja cargo-nextest cargo-zigbuild cargo-xwin; do command -v "$b" >/dev/null || { echo "MISSING: $b"; exit 1; } done - echo "tools OK | bun $(bun --version) | rust $(rustc --version) | sccache $(set -- $(sccache --version); echo "$2") | zig $(zig version) | cmake $(set -- $(cmake --version | head -1); echo "$3") | ninja $(ninja --version) | gh $(set -- $(gh --version | head -1); echo "$3")" + echo "tools OK | bun $(bun --version) | rust $(rustc --version) | sccache $(set -- $(sccache --version); echo "$2") | zstd $(zstd --version) | zig $(zig version) | cmake $(set -- $(cmake --version | head -1); echo "$3") | ninja $(ninja --version) | gh $(set -- $(gh --version | head -1); echo "$3")" ' } @@ -170,10 +186,10 @@ build_with_docker() { echo "==> [2/5] verifying baked tools" docker run --rm --entrypoint bash "$IMAGE" -lc ' set -e - for b in gh fd rg magick bun cargo rustc pkg-config clang lld sccache zig cmake ninja cargo-nextest cargo-zigbuild cargo-xwin; do + for b in gh fd rg magick bun cargo rustc pkg-config clang lld sccache zstd zig cmake ninja cargo-nextest cargo-zigbuild cargo-xwin; do command -v "$b" >/dev/null || { echo "MISSING: $b"; exit 1; } done - echo "tools OK | bun $(bun --version) | rust $(rustc --version) | sccache $(set -- $(sccache --version); echo "$2") | zig $(zig version) | cmake $(set -- $(cmake --version | head -1); echo "$3") | ninja $(ninja --version) | gh $(set -- $(gh --version | head -1); echo "$3")" + echo "tools OK | bun $(bun --version) | rust $(rustc --version) | sccache $(set -- $(sccache --version); echo "$2") | zstd $(zstd --version) | zig $(zig version) | cmake $(set -- $(cmake --version | head -1); echo "$3") | ninja $(ninja --version) | gh $(set -- $(gh --version | head -1); echo "$3")" ' echo "==> [3/5] importing into k3s containerd (k8s.io namespace)" @@ -216,15 +232,27 @@ esac echo "==> [4/5] pointing ARC runner scale set at $IMAGE" sed -i "s#image: omp-kata-runner:.*#image: $IMAGE#" "$ARC_VALUES" +sed -i -E "s/^maxRunners:.*/maxRunners: $RUNNER_MAX_RUNNERS/" "$ARC_VALUES" helm upgrade "$ARC_RELEASE" --namespace "$ARC_NAMESPACE" --version "$ARC_CHART_VERSION" \ -f "$ARC_VALUES" \ + --set-string "template.spec.containers[0].resources.requests.cpu=$RUNNER_CPU" \ + --set-string "template.spec.containers[0].resources.limits.cpu=$RUNNER_CPU" \ + --set-string "template.spec.containers[0].resources.requests.memory=$RUNNER_MEMORY" \ + --set-string "template.spec.containers[0].resources.limits.memory=$RUNNER_MEMORY" \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set >/dev/null echo "==> [5/5] verifying rollout" live="$(kubectl get autoscalingrunnerset "$ARC_RELEASE" -n "$ARC_NAMESPACE" \ -o jsonpath='{.spec.template.spec.containers[0].image}')" -echo "ARC runner image is now: $live" -[ "$live" = "$IMAGE" ] && echo "OK: reloaded $IMAGE" || { echo "MISMATCH: expected $IMAGE"; exit 1; } +live_max="$(kubectl get autoscalingrunnerset "$ARC_RELEASE" -n "$ARC_NAMESPACE" -o jsonpath='{.spec.maxRunners}')" +live_resources="$(kubectl get autoscalingrunnerset "$ARC_RELEASE" -n "$ARC_NAMESPACE" \ + -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}/{.spec.template.spec.containers[0].resources.limits.cpu} {.spec.template.spec.containers[0].resources.requests.memory}/{.spec.template.spec.containers[0].resources.limits.memory}')" +expected_resources="$RUNNER_CPU/$RUNNER_CPU $RUNNER_MEMORY/$RUNNER_MEMORY" +echo "ARC runner image/resources: $live | max=$live_max | $live_resources" +[ "$live" = "$IMAGE" ] || { echo "MISMATCH: expected image $IMAGE"; exit 1; } +[ "$live_max" = "$RUNNER_MAX_RUNNERS" ] || { echo "MISMATCH: expected maxRunners $RUNNER_MAX_RUNNERS"; exit 1; } +[ "$live_resources" = "$expected_resources" ] || { echo "MISMATCH: expected resources $expected_resources"; exit 1; } +echo "OK: reloaded $IMAGE" REMOTE echo "OK: $IMAGE built on ${CI_HOST}, stored in k3s containerd, and rolled out to ARC." diff --git a/infra/runner.Dockerfile b/infra/runner.Dockerfile index 81b36a5ca..e09b4302f 100644 --- a/infra/runner.Dockerfile +++ b/infra/runner.Dockerfile @@ -35,7 +35,7 @@ RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg -o && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list \ && apt-get update \ && apt-get install -y \ - build-essential pkg-config curl ca-certificates git unzip xz-utils gh \ + build-essential pkg-config curl ca-certificates git unzip xz-utils zstd gh \ clang lld llvm \ libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \ fd-find ripgrep imagemagick \ diff --git a/scripts/ci-native-artifact-cache.test.ts b/scripts/ci-native-artifact-cache.test.ts new file mode 100644 index 000000000..a0fbf2732 --- /dev/null +++ b/scripts/ci-native-artifact-cache.test.ts @@ -0,0 +1,83 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; + +const script = path.join(import.meta.dir, "ci-native-artifact-cache.ts"); +const tempRoots: string[] = []; + +async function tempDir(): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "omp-native-cache-test-")); + tempRoots.push(dir); + return dir; +} + +async function run(args: string[], cacheDir: string, outputPath?: string): Promise { + const proc = Bun.spawn([process.execPath, script, ...args], { + cwd: path.join(import.meta.dir, ".."), + env: { + ...process.env, + OMP_NATIVE_CACHE_DIR: cacheDir, + GITHUB_OUTPUT: outputPath, + }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (exitCode !== 0) throw new Error(`cache command failed (${exitCode}): ${stderr}`); + return stdout; +} + +afterEach(async () => { + await Promise.all(tempRoots.splice(0).map(dir => fs.rm(dir, { recursive: true, force: true }))); +}); + +describe("CI native artifact cache", () => { + it("restores a complete artifact set without unrelated build output", async () => { + const root = await tempDir(); + const source = path.join(root, "source"); + const destination = path.join(root, "destination"); + const output = path.join(root, "github-output"); + await fs.mkdir(source); + await Promise.all([ + Bun.write(path.join(source, "pi_natives.linux-x64.node"), "baseline"), + Bun.write(path.join(source, "build.log"), "not an artifact"), + ]); + + await run(["save", "abcdef12", "pi-natives-linux-x64-baseline-habcdef12", source], root); + await run(["restore", "abcdef12", destination, "pi-natives-linux-x64-baseline-habcdef12"], root, output); + + expect(await Bun.file(output).text()).toBe("hit=true\n"); + expect(await Bun.file(path.join(destination, "pi_natives.linux-x64.node")).text()).toBe("baseline"); + expect(await Bun.file(path.join(destination, "build.log")).exists()).toBe(false); + }); + + it("reports a miss and copies nothing unless every requested artifact is complete", async () => { + const root = await tempDir(); + const source = path.join(root, "source"); + const destination = path.join(root, "destination"); + const output = path.join(root, "github-output"); + await fs.mkdir(source); + await Bun.write(path.join(source, "pi_natives.linux-x64.node"), "baseline"); + await run(["save", "abcdef12", "pi-natives-linux-x64-baseline-habcdef12", source], root); + + await run( + [ + "restore", + "abcdef12", + destination, + "pi-natives-linux-x64-baseline-habcdef12", + "pi-natives-linux-x64-modern-habcdef12", + ], + root, + output, + ); + + expect(await Bun.file(output).text()).toBe("hit=false\n"); + expect(await Bun.file(destination).exists()).toBe(false); + }); +}); diff --git a/scripts/ci-native-artifact-cache.ts b/scripts/ci-native-artifact-cache.ts new file mode 100755 index 000000000..473e75f39 --- /dev/null +++ b/scripts/ci-native-artifact-cache.ts @@ -0,0 +1,129 @@ +#!/usr/bin/env bun + +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +const CACHE_ENV = "OMP_NATIVE_CACHE_DIR"; +const COMPLETE_FILE = ".complete"; + +function validateSegment(value: string, label: string): void { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value)) { + throw new Error(`Invalid ${label} ${JSON.stringify(value)}`); + } +} + +async function writeOutput(name: string, value: string): Promise { + const outputPath = Bun.env.GITHUB_OUTPUT; + if (outputPath) { + await fs.appendFile(outputPath, `${name}=${value}\n`); + } +} + +async function completedFiles(artifactDir: string): Promise { + try { + const text = await Bun.file(path.join(artifactDir, COMPLETE_FILE)).text(); + const files = text.split("\n").filter(Boolean); + if (files.length === 0 || files.some(file => path.basename(file) !== file || !file.endsWith(".node"))) { + return null; + } + for (const file of files) { + if (!(await Bun.file(path.join(artifactDir, file)).exists())) return null; + } + return files; + } catch { + return null; + } +} + +async function save(cacheRoot: string, hash: string, artifactName: string, sourceDir: string): Promise { + validateSegment(hash, "source hash"); + validateSegment(artifactName, "artifact name"); + const entries = await fs.readdir(sourceDir, { withFileTypes: true }); + const files = entries + .filter(entry => entry.isFile() && entry.name.endsWith(".node")) + .map(entry => entry.name) + .sort(); + if (files.length === 0) throw new Error(`No native addons found in ${sourceDir}`); + + const hashDir = path.join(cacheRoot, hash); + const artifactDir = path.join(hashDir, artifactName); + if (await completedFiles(artifactDir)) { + console.log(`Native artifact cache already populated: ${artifactName}`); + return; + } + + await fs.mkdir(hashDir, { recursive: true }); + const stagingDir = path.join(hashDir, `${artifactName}.tmp-${process.pid}-${crypto.randomUUID()}`); + await fs.mkdir(stagingDir); + try { + for (const file of files) { + await fs.copyFile(path.join(sourceDir, file), path.join(stagingDir, file)); + } + await Bun.write(path.join(stagingDir, COMPLETE_FILE), `${files.join("\n")}\n`); + try { + await fs.rename(stagingDir, artifactDir); + } catch (error) { + if (!(await completedFiles(artifactDir))) throw error; + await fs.rm(stagingDir, { recursive: true, force: true }); + } + } catch (error) { + await fs.rm(stagingDir, { recursive: true, force: true }); + throw error; + } + console.log(`Saved native artifact cache: ${artifactName} (${files.join(", ")})`); +} + +async function restore( + cacheRoot: string, + hash: string, + destination: string, + artifactNames: string[], +): Promise { + validateSegment(hash, "source hash"); + if (artifactNames.length === 0) throw new Error("At least one artifact name is required"); + + const sources: Array<{ dir: string; files: string[] }> = []; + for (const artifactName of artifactNames) { + validateSegment(artifactName, "artifact name"); + const dir = path.join(cacheRoot, hash, artifactName); + const files = await completedFiles(dir); + if (!files) return false; + sources.push({ dir, files }); + } + + await fs.mkdir(destination, { recursive: true }); + for (const source of sources) { + for (const file of source.files) { + await fs.copyFile(path.join(source.dir, file), path.join(destination, file)); + } + } + console.log(`Restored native artifacts from local cache: ${artifactNames.join(", ")}`); + return true; +} + +async function main(): Promise { + const [mode, hash, first, ...rest] = process.argv.slice(2); + if ((mode !== "save" && mode !== "restore") || !hash || !first) { + throw new Error( + "Usage: ci-native-artifact-cache.ts save [source-dir] | restore ...", + ); + } + + const cacheRoot = Bun.env[CACHE_ENV]?.trim(); + if (!cacheRoot) { + if (mode === "restore") await writeOutput("hit", "false"); + console.log(`Native artifact cache disabled: ${CACHE_ENV} is unset`); + return; + } + + if (mode === "save") { + await save(cacheRoot, hash, first, rest[0] ?? "packages/natives/native"); + return; + } + + const hit = await restore(cacheRoot, hash, first, rest); + await writeOutput("hit", String(hit)); + if (!hit) console.log(`Native artifact cache miss: ${rest.join(", ")}`); +} + +if (import.meta.main) await main(); diff --git a/scripts/ci-target-cache.ts b/scripts/ci-target-cache.ts index da7e2efc1..7ac39c23b 100755 --- a/scripts/ci-target-cache.ts +++ b/scripts/ci-target-cache.ts @@ -36,6 +36,11 @@ const MAX_SNAPSHOT_BYTES = 4 * 1024 ** 3; const EXISTS_TIMEOUT_MS = 30_000; const DOWNLOAD_TIMEOUT_MS = 180_000; const UPLOAD_TIMEOUT_MS = 300_000; +const configuredCompressionThreads = Number(Bun.env.OMP_CI_CPU_COUNT); +const COMPRESSION_THREADS = + Number.isInteger(configuredCompressionThreads) && configuredCompressionThreads > 0 + ? configuredCompressionThreads + : 2; /** * Resolve the S3 endpoint URL from the sccache pod env. `SCCACHE_ENDPOINT` is @@ -140,9 +145,11 @@ async function save(s3: S3Client, objectKey: string, targetDir: string): Promise // CARGO_INCREMENTAL=0 in CI, so incremental/ only exists from stray // local state; exclude it regardless — it is the one cargo dir that is // pure dead weight for a cold consumer. Explicit compress pipe for the - // same tar-flavor reason as in restore(); -T0 uses all cores. + // same tar-flavor reason as in restore(). Compression is capped to the + // runner's admitted CPU allocation; `-T0` multiplied host contention when + // several native matrix jobs saved snapshots together. const create = - await $`tar -cf - --exclude=${"*/incremental"} -C ${path.dirname(targetDir)} ${path.basename(targetDir)} | zstd -q -T0 -3 -f -o ${tmpTar}` + await $`tar -cf - --exclude=${"*/incremental"} -C ${path.dirname(targetDir)} ${path.basename(targetDir)} | zstd -q -T${COMPRESSION_THREADS} -3 -f -o ${tmpTar}` .quiet() .nothrow(); if (create.exitCode !== 0) { diff --git a/scripts/ci-test-ts.ts b/scripts/ci-test-ts.ts index 5fceccf97..5f3f3486f 100755 --- a/scripts/ci-test-ts.ts +++ b/scripts/ci-test-ts.ts @@ -120,6 +120,7 @@ const localOnlyWorkspacePackages = ["packages/mnemopi", "python/robomp/web"]; const repoScriptTests = [ "scripts/ci-concurrency.test.ts", "scripts/ci-build-native.test.ts", + "scripts/ci-native-artifact-cache.test.ts", "scripts/ci-release-notes.test.ts", "scripts/ci-release-publish.test.ts", "scripts/fix-dts-extensions.test.ts", @@ -560,6 +561,7 @@ function isCI(): boolean { // memory-constrained laptop), or `all`/`max` to launch every chunk at once. function testConcurrency(total: number): number { const raw = Bun.env.OMP_TEST_CONCURRENCY?.trim().toLowerCase(); + if (!raw) return Math.min(Math.max(1, os.availableParallelism()), total); if (raw === "all" || raw === "max") { return total; } @@ -567,7 +569,7 @@ function testConcurrency(total: number): number { if (Number.isFinite(override) && override >= 1) { return Math.min(Math.floor(override), total); } - return Math.min(Math.max(1, os.availableParallelism()), total); + throw new Error(`Invalid OMP_TEST_CONCURRENCY=${JSON.stringify(raw)}; expected a positive integer, all, or max`); } // ANSI styling for interactive runs only; disabled when stdout is not a TTY or @@ -890,9 +892,11 @@ if (import.meta.main) { } const testCommands = await commandsForMode(requestedMode as Mode); - // Outside CI, fan the independent chunk processes out across cores; CI keeps the - // sequential, fail-fast path so each memory-capped runner job stays bounded. - if (!isDryRun && !isCI() && testCommands.length > 1) { + const explicitConcurrency = Boolean(Bun.env.OMP_TEST_CONCURRENCY?.trim()); + // CI defaults to one process at a time, but memory-sized workflow buckets + // explicitly opt into bounded process concurrency. Local runs fan out by + // default and may use the same override. + if (!isDryRun && testCommands.length > 1 && (!isCI() || explicitConcurrency)) { await runTestCommandsInParallel(testCommands, testConcurrency(testCommands.length)); } else { for (const testCommand of testCommands) { diff --git a/scripts/install-tests/run-ci.sh b/scripts/install-tests/run-ci.sh index 247904412..d6dc7eac7 100755 --- a/scripts/install-tests/run-ci.sh +++ b/scripts/install-tests/run-ci.sh @@ -43,7 +43,9 @@ find_tarball() { } section "Binary install smoke" -bun --cwd=packages/natives run build +if [ "${OMP_INSTALL_TEST_SKIP_NATIVE_BUILD:-0}" != "1" ]; then + bun --cwd=packages/natives run build +fi bun --cwd=packages/coding-agent run build BINARY_DIR="$WORK_DIR/binary-bin"