diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..73bd0da5a --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,6 @@ +# actionlint only knows GitHub-hosted labels; register the self-hosted ARC +# runner scale set (infra/docs/04-arc-and-caching.md) so `runs-on: omp-kata` +# lints clean. +self-hosted-runner: + labels: + - omp-kata diff --git a/.github/actions/bazel-cache/action.yml b/.github/actions/bazel-cache/action.yml index 6d3239fcc..0d40424b0 100644 --- a/.github/actions/bazel-cache/action.yml +++ b/.github/actions/bazel-cache/action.yml @@ -12,11 +12,17 @@ description: > The v3 key is -: config covers toolchain and build settings, source covers crates/** + BUILD.bazel. Restores fall - back to the config-scoped prefix, so a source generation the exact key - has never seen still seeds from the previous generation's archive, and - the refreshed archive is saved after the build (an exact hit suppresses + back to the config-scoped prefix, then to a bare scope+os+arch prefix: + the config hash covers Cargo.toml/Cargo.lock, which every release + version bump rewrites, so without the bare fallback each release built + the darwin addons fully cold (~40-50 min on macos-15-intel; CI run + 30357804722). A stale-config archive is safe — bazel keys every + action by content digest, so mismatched entries just miss — and the + refreshed archive is saved after the build (an exact hit suppresses the save — without the source component the first archive for a config - generation would permanently shadow newer source states). + generation would permanently shadow newer source states). Restored + entries untouched for 14 days are pruned before the build so + ever-seeding archives don't grow without bound. inputs: scope: @@ -58,10 +64,12 @@ runs: fi key="bazel-disk-v3-${{ inputs.scope }}-${{ runner.os }}-${{ runner.arch }}-$CONFIG_HASH-$SOURCE_HASH" prefix="bazel-disk-v3-${{ inputs.scope }}-${{ runner.os }}-${{ runner.arch }}-$CONFIG_HASH-" + bare_prefix="bazel-disk-v3-${{ inputs.scope }}-${{ runner.os }}-${{ runner.arch }}-" { echo "remote=$remote" echo "cache-key=$key" echo "cache-prefix=$prefix" + echo "cache-bare-prefix=$bare_prefix" } >> "$GITHUB_OUTPUT" - name: Restore bazel disk cache @@ -71,8 +79,14 @@ runs: with: path: ~/.cache/omp-bazel-disk key: ${{ steps.backend.outputs.cache-key }} + # Order matters: exact key, same-config prefix, then any archive for + # this scope+os+arch. The bare fallback is what keeps release version + # bumps (Cargo.toml/Cargo.lock churn -> new CONFIG_HASH) from going + # fully cold; bazel's content-addressed action keys make a + # stale-config archive a partial hit, never a wrong output. restore-keys: | ${{ steps.backend.outputs.cache-prefix }} + ${{ steps.backend.outputs.cache-bare-prefix }} - name: Compose cache config id: compose @@ -112,6 +126,11 @@ runs: } > "$rc" else mkdir -p "$HOME/.cache/omp-bazel-disk" "$HOME/.cache/omp-bazel-repo" + # Seeding from stale archives means entries dead source + # generations wrote would otherwise ride along forever; drop + # anything untouched for 14 days (tar preserves mtimes across + # the actions/cache round trip, so age survives restores). + find "$HOME/.cache/omp-bazel-disk" -type f -mtime +14 -delete 2>/dev/null || true { echo "common --config=ci" echo "common --disk_cache=$HOME/.cache/omp-bazel-disk" diff --git a/.github/actions/bun-install/action.yml b/.github/actions/bun-install/action.yml index 54f2ddeab..752907a76 100644 --- a/.github/actions/bun-install/action.yml +++ b/.github/actions/bun-install/action.yml @@ -42,13 +42,25 @@ runs: curl -fsSL https://bun.sh/install | bash -s "bun-v1.3.14" echo "${BUN_INSTALL}/bin" >> "$GITHUB_PATH" - # Off-infra (GitHub-hosted): stock actions/cache for the bun store. - - name: Cache bun store (GitHub cache) + # Off-infra (GitHub-hosted): actions/cache for the bun store, split into + # restore + save. actions/cache entries are scoped per ref but count + # against the shared 10 GB repo quota — letting every PR branch save its + # own copy of the ~500 MB store duplicated it 19x, evicting the far more + # valuable bazel disk-cache archives (which is why release darwin builds + # kept going cold). PRs only restore; non-PR runs (main pushes, releases, + # dispatches) produce the shared entry every PR can see. + - name: Restore bun store (GitHub cache) + id: bun-cache if: steps.env.outputs.cache == 'gha' - uses: actions/cache@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ~/.bun/install/cache key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }} + # A lockfile change seeds from the previous store: entries are + # content-hash-named tarballs, so stale extras waste only space and + # bun downloads just the delta. + restore-keys: | + bun-${{ runner.os }}- # On-infra (omp-kata): the pod mounts a shared PVC at bun's store path. - name: Prepare mounted bun store @@ -75,3 +87,12 @@ runs: echo "::warning title=bun install retry::shared bun store install failed; retrying with a clean job-local cache" retry_cache="$(mktemp -d "${RUNNER_TEMP:-/tmp}/bun-cache-retry.XXXXXX")" bun install --frozen-lockfile --cache-dir="$retry_cache" + + # Producer half of the split cache: only non-PR runs save, so the store + # exists once per lockfile generation instead of once per PR ref. + - name: Save bun store (GitHub cache) + if: steps.env.outputs.cache == 'gha' && github.event_name != 'pull_request' && steps.bun-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.bun/install/cache + key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }} diff --git a/.github/actions/native-inputs/action.yml b/.github/actions/native-inputs/action.yml deleted file mode 100644 index 52d65ffe8..000000000 --- a/.github/actions/native-inputs/action.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: "Native inputs: change detection + artifact cache key" -description: > - Single source of truth for what counts as a native-affecting change. - - `rust` gates Rust validation (tests, clippy, rustfmt); `cache-key` - addresses the prebuilt Linux x64 addon pair published by trusted main - builds. The detector pathspec and the hashed file set MUST cover the same - inputs — drift means a native change could ship without validation or be - tested against stale addons. The key embeds a schema version, OS, arch, - target pair, and build profile so a future target/profile change can - never resolve valid-but-wrong .node files under the same source hash. - -outputs: - rust: - description: Whether the event touches native inputs (always true off pull_request) - value: ${{ steps.changes.outputs.rust }} - source-hash: - description: 16-hex fingerprint over every native build input - value: ${{ steps.hash.outputs.source-hash }} - cache-key: - description: Exact actions/cache key for the prebuilt Linux x64 addon pair - value: ${{ steps.hash.outputs.cache-key }} - -runs: - using: composite - steps: - - name: Detect native-affecting changes - id: changes - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - if [ "${{ github.event_name }}" != "pull_request" ]; then - echo "rust=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - # Write the diff to a file before matching: in a pipeline, an API - # failure is indistinguishable from "no native changes" (skips - # validation — fail-open), and grep -q can close the pipe early so - # gh dies on SIGPIPE and pipefail flips a MATCH to rust=false. - changed_files="$RUNNER_TEMP/native-changed-files" - gh pr diff ${{ github.event.pull_request.number }} --name-only > "$changed_files" - if grep -qE '^(crates/|bazel/|Cargo\.(toml|lock)|MODULE\.bazel(\.lock)?|BUILD\.bazel|\.bazelrc|\.bazelignore|\.bazelversion|rust-toolchain\.toml|rustfmt\.toml|scripts/bazel-natives\.ts|\.github/actions/(bazel-cache|bazel-natives|native-artifacts|native-inputs)/|\.github/workflows/ci\.yml)' "$changed_files"; then - echo "rust=true" >> "$GITHUB_OUTPUT" - else - echo "No native-affecting changes; skipping Rust validation." - echo "rust=false" >> "$GITHUB_OUTPUT" - fi - - # Content-addresses the addon bytes from git index entries (mode, - # blob identity, path) — `git ls-files -s` covers the executable bit - # and never follows symlinks into worktree bytes. `--error-unmatch` - # fails the step loudly when a listed path disappears instead of - # silently narrowing the key. - - name: Compute native source hash - id: hash - shell: bash - run: | - set -euo pipefail - source_hash=$(git ls-files -s -z --error-unmatch -- \ - crates bazel \ - Cargo.toml Cargo.lock \ - MODULE.bazel MODULE.bazel.lock BUILD.bazel \ - .bazelrc .bazelignore .bazelversion \ - rust-toolchain.toml rustfmt.toml \ - scripts/bazel-natives.ts \ - .github/actions/bazel-cache .github/actions/bazel-natives \ - .github/actions/native-artifacts .github/actions/native-inputs \ - .github/workflows/ci.yml \ - | sort -z \ - | sha256sum \ - | cut -c1-16) - { - echo "source-hash=$source_hash" - echo "cache-key=native-addons-v1-linux-x64-baseline+modern-opt-$source_hash" - } >> "$GITHUB_OUTPUT" - echo "Native source hash: $source_hash" diff --git a/.github/workflows/bazel-cache-warm.yml b/.github/workflows/bazel-cache-warm.yml index adc722996..3d61a3cb9 100644 --- a/.github/workflows/bazel-cache-warm.yml +++ b/.github/workflows/bazel-cache-warm.yml @@ -1,24 +1,38 @@ name: Warm bazel disk cache -# GitHub-hosted PR runners cannot use the cluster remote cache and only see -# actions/cache entries created on the default branch. Bazel action keys do -# not transfer across runner environments (a kata-produced disk cache misses -# every action on ubuntu-22.04), so seed the disk cache from the same image -# PR jobs run on. Runs the full hosted action set — Rust validation (test, -# clippy, rustfmt) plus the native addons — so one exact-key archive covers -# both rust_validate and native_addons for this source generation. The v3 -# key embeds a crates/** source fingerprint, so a native change on main -# means an exact miss: the build seeds from the previous generation via the -# config-scoped prefix and the refreshed archive is saved. An exact hit -# makes every invocation a cache replay and saves nothing. +# GitHub-hosted runners cannot use the cluster remote cache and only see +# actions/cache entries created on the default branch, and bazel action keys +# do not transfer across runner environments (a kata-produced disk cache +# misses every action on a hosted image) — so seed each hosted consumer from +# the same image it runs on: +# +# warm_darwin (macos-*) -> scopes `release-darwin-*`, consumed by the +# release_binary darwin jobs. Releases are the +# only other producers for these scopes and a +# release always changes CONFIG_HASH (version +# bump rewrites Cargo.toml/Cargo.lock), so +# without warming here every release rebuilt the +# darwin addons cold (~40-50 min on +# macos-15-intel — run 30357804722). +# warm_bun (ubuntu-22.04)-> the shared `bun--` store entry. +# PR jobs restore it but never save (per-PR-ref +# copies of the ~500 MB store once ate ~9.4 GB of +# the 10 GB repo cache quota and evicted every +# bazel archive). +# +# The v3 key embeds a crates/** source fingerprint, so a native change on +# main means an exact miss: the build seeds from a previous generation via +# the prefix/bare restore keys and the refreshed archive is saved. An exact +# hit makes every invocation a cache replay and saves nothing. +# +# Paths mirror the cache-key inputs (bazel-cache action) plus bun.lock — a +# packages/** push cannot change any archive, so it does not trigger a warm. on: push: branches: [main] paths: - - "packages/**" - "crates/**" - - "scripts/**" - "bazel/**" - "MODULE.bazel" - "MODULE.bazel.lock" @@ -30,50 +44,54 @@ on: - "Cargo.lock" - "rust-toolchain.toml" - "rustfmt.toml" + - "bun.lock" - ".github/**" workflow_dispatch: permissions: contents: read +# No cancel-in-progress: a cancelled warm saves nothing, and the first darwin +# warm on a cold runner takes long enough that back-to-back native pushes +# would cancel it forever. Superseded queued runs still collapse to the +# newest one, and staleness is harmless — consumers seed via prefix keys. concurrency: group: bazel-cache-warm - cancel-in-progress: true + cancel-in-progress: false jobs: - warm: - name: Seed hosted bazel disk cache + # Keeps the release_binary darwin scopes seeded near HEAD so a release's + # bazel build is the version-bump delta, not a cold graph. Scope, runner + # image, and target must stay in lockstep with the release_binary matrix + # in ci.yml — a mismatched scope warms an archive nobody restores. + warm_darwin: + name: "Seed darwin release bazel cache: ${{ matrix.target }}" + strategy: + fail-fast: false + matrix: + include: + - { os: macos-15-intel, target: darwin-x64-baseline, scope: release-darwin-x64 } + - { os: macos-14, target: darwin-arm64, scope: release-darwin-arm64 } + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.3" + # scripts/bazel-natives.ts is dependency-free (node builtins only), so + # no `bun install` is needed; the bazel-natives action restores, + # builds, and saves the scope's disk cache. + - uses: ./.github/actions/bazel-natives + with: + targets: ${{ matrix.target }} + cache-scope: ${{ matrix.scope }} + + # Produces the shared bun store entry PR jobs restore. bun-install's save + # half only runs on non-PR events, and main CI jobs run on omp-kata (PVC + # store, no GitHub cache), so this is the sole hosted-runner producer. + warm_bun: + name: Seed bun store cache runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - - id: cache - uses: ./.github/actions/bazel-cache - with: - scope: linux - # Invocation set mirrors the hosted CI jobs (rust_validate + - # native_addons) so the saved archive serves both. - - name: Rust tests - run: | - set -euo pipefail - if [ "$(ulimit -Sn)" != unlimited ] && [ "$(ulimit -Sn)" -lt 65536 ]; then ulimit -Sn 65536 || true; fi - bazelisk --bazelrc="${{ steps.cache.outputs.rc }}" test //crates/... - - name: Clippy (workspace lint policy on opted-in crates) - run: | - bazelisk query "kind('rust_library|rust_shared_library', //crates/pi-ast/... + //crates/pi-iso/... + //crates/pi-natives/... + //crates/pi-shell/... + //crates/pi-walker/...)" \ - | xargs bazelisk --bazelrc="${{ steps.cache.outputs.rc }}" build --config=clippy-strict -- - - name: Clippy (default lints elsewhere) - run: | - bazelisk query "kind('rust_library|rust_shared_library', //crates/... - (//crates/pi-ast/... + //crates/pi-iso/... + //crates/pi-natives/... + //crates/pi-shell/... + //crates/pi-walker/...) - //crates/vendor/brush-core/... - //crates/vendor/brush-builtins/...)" \ - | xargs bazelisk --bazelrc="${{ steps.cache.outputs.rc }}" build --config=clippy -- - - name: Rustfmt - run: bazelisk --bazelrc="${{ steps.cache.outputs.rc }}" build --config=rustfmt //crates/... - - name: Build native addons - run: | - set -euo pipefail - bazelisk --bazelrc="${{ steps.cache.outputs.rc }}" build //:natives-linux-x64-baseline //:natives-linux-x64-modern - - name: Save bazel disk cache - if: steps.cache.outputs.save-needed == 'true' - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ~/.cache/omp-bazel-disk - key: ${{ steps.cache.outputs.cache-key }} + - uses: ./.github/actions/bun-install diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 910fec12b..c2880cd12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,26 +124,23 @@ jobs: - name: Build collab web run: bun run collab:web:build - # Rust validation (tests, clippy, rustfmt) and native addon production are - # separate jobs: TS test shards wait only on the addons, and on - # native-changing PRs validation no longer delays artifact production. + # Rust validation (tests, clippy, rustfmt) runs on non-PR events only. + # Native-changing PRs are rare enough that they do not warrant PR-side + # Rust validation or a PR-side addon build (see native_addons): Rust is + # validated post-merge on main (kata remote cache, warm) and again at + # release. A skipped required check still satisfies branch protection. rust_validate: name: Validate Rust workspace (bazel) - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-22.04' || 'omp-kata' }} + if: github.event_name != 'pull_request' + runs-on: omp-kata steps: - uses: actions/checkout@v4 - - id: inputs - uses: ./.github/actions/native-inputs - # TS-only PRs skip Rust validation entirely. - - if: steps.inputs.outputs.rust == 'true' - uses: ./.github/actions/bun-install + - uses: ./.github/actions/bun-install - id: cache - if: steps.inputs.outputs.rust == 'true' uses: ./.github/actions/bazel-cache with: scope: linux - name: Rust tests - if: steps.inputs.outputs.rust == 'true' # The ulimit guard runs in the step that launches the bazel server # (limits are per-process and the server persists across steps). run: | @@ -154,67 +151,49 @@ jobs: # `[lints] workspace = true` get the workspace policy, the vendored # brush fork is exempt (same as run-rs-task.ts's cargo excludes). - name: Clippy (workspace lint policy on opted-in crates) - if: steps.inputs.outputs.rust == 'true' run: | bazelisk query "kind('rust_library|rust_shared_library', //crates/pi-ast/... + //crates/pi-iso/... + //crates/pi-natives/... + //crates/pi-shell/... + //crates/pi-walker/...)" \ | xargs bazelisk --bazelrc="${{ steps.cache.outputs.rc }}" build --config=clippy-strict -- - name: Clippy (default lints elsewhere) - if: steps.inputs.outputs.rust == 'true' run: | bazelisk query "kind('rust_library|rust_shared_library', //crates/... - (//crates/pi-ast/... + //crates/pi-iso/... + //crates/pi-natives/... + //crates/pi-shell/... + //crates/pi-walker/...) - //crates/vendor/brush-core/... - //crates/vendor/brush-builtins/...)" \ | xargs bazelisk --bazelrc="${{ steps.cache.outputs.rc }}" build --config=clippy -- - name: Rustfmt - if: steps.inputs.outputs.rust == 'true' run: bazelisk --bazelrc="${{ steps.cache.outputs.rc }}" build --config=rustfmt //crates/... - # No disk-cache save here: native_addons saves this same key - # concurrently, and an immutable archive can have only one producer - # per key — a validation-only archive winning the race would - # suppress the addon half on every later exact hit. Validation - # actions are seeded by the main cache warmer's combined archive. - # Builds the native addons every downstream job installs. TS-only PRs - # restore the prebuilt Linux x64 pair published by trusted main builds - # (exact content-addressed key, smoke-loaded before use) and skip Bazel - # entirely; anything else builds with bazel. Main builds all Linux-hosted - # targets, pull requests only the x64 pair tests require. + # Provides the native addons every downstream TS job installs. PRs never + # build: they fetch the latest release's Linux x64 pair from the + # @oh-my-pi/pi-natives-linux-x64 npm leaf instead. The workspace loader + # skips its version sentinel for workspace loads, so release addons load + # fine under a newer checkout; a PR whose TS tests depend on changed + # native behavior fails visibly and the native side lands via main. + # Main (kata, cluster remote cache) builds all Linux-hosted targets with + # bazel. Job name says "bazel" for continuity with required checks. native_addons: name: Build native addons (bazel) runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-22.04' || 'omp-kata' }} steps: - uses: actions/checkout@v4 - - id: inputs - uses: ./.github/actions/native-inputs - # Trusted fast path. Exact-key restores only — a prefix fallback - # could resolve valid-but-wrong .node files under a changed - # target/profile (the key embeds schema + os/arch + target pair + - # build profile + input hash; see native-inputs action). - - name: Restore prebuilt native addons - id: prebuilt + + # ---- PR path: latest release addons from npm ---- + - name: Fetch release native addons (npm) if: github.event_name == 'pull_request' - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ~/.cache/omp-native-addons - key: ${{ steps.inputs.outputs.cache-key }} - - name: Stage cached addons - if: steps.prebuilt.outputs.cache-hit == 'true' shell: bash - # Canonical filenames come from the target map in - # scripts/bazel-natives.ts; staging reproduces the bazel-bin - # layout the artifact upload below globs. run: | set -euo pipefail + tarball="$(npm view @oh-my-pi/pi-natives-linux-x64@latest dist.tarball)" + echo "Fetching $tarball" + curl -fsSL --retry 3 "$tarball" | tar -xz -C "$RUNNER_TEMP" mkdir -p bazel-bin/natives-linux-x64-baseline bazel-bin/natives-linux-x64-modern - cp ~/.cache/omp-native-addons/pi_natives.linux-x64-baseline.node bazel-bin/natives-linux-x64-baseline/ - cp ~/.cache/omp-native-addons/pi_natives.linux-x64-modern.node bazel-bin/natives-linux-x64-modern/ - # A poisoned cache entry must not ship: load both addons before - # trusting them. Failure falls back to a full build (loudly) rather - # than failing the PR. - - name: Smoke cached addons - id: smoke - if: steps.prebuilt.outputs.cache-hit == 'true' + cp "$RUNNER_TEMP/package/pi_natives.linux-x64-baseline.node" bazel-bin/natives-linux-x64-baseline/ + cp "$RUNNER_TEMP/package/pi_natives.linux-x64-modern.node" bazel-bin/natives-linux-x64-modern/ + # A corrupt download must fail here, not as a confusing dlopen error + # in every downstream test shard. + - name: Smoke release addons + if: github.event_name == 'pull_request' shell: bash run: | - set -uo pipefail + set -euo pipefail # NOTE: `bun -e 'require("./x.node")'` swallows dlopen failures # (exit 0 on a bogus addon); a script file enforces them in # both bun and node. Verified against a corrupt .node fixture. @@ -229,50 +208,40 @@ jobs: EOF loader=node if command -v bun >/dev/null 2>&1; then loader=bun; fi - if "$loader" "$RUNNER_TEMP/smoke-addons.js" \ - "$PWD/bazel-bin/natives-linux-x64-baseline/pi_natives.linux-x64-baseline.node" \ - "$PWD/bazel-bin/natives-linux-x64-modern/pi_natives.linux-x64-modern.node"; then - echo "ok=true" >> "$GITHUB_OUTPUT" - else - echo "::warning::cached native addons failed to load; falling back to a full bazel build" - rm -rf bazel-bin - echo "ok=false" >> "$GITHUB_OUTPUT" - fi - - name: Decide build path - id: decide + "$loader" "$RUNNER_TEMP/smoke-addons.js" \ + "$PWD/bazel-bin/natives-linux-x64-baseline/pi_natives.linux-x64-baseline.node" \ + "$PWD/bazel-bin/natives-linux-x64-modern/pi_natives.linux-x64-modern.node" + # Make the policy visible on the rare native-touching PR instead of + # leaving a reviewer to wonder which addons the tests exercised. + # Best-effort: a diff-API failure must not fail the job over a notice. + - name: Note native changes tested against release addons + if: github.event_name == 'pull_request' shell: bash env: - HIT: ${{ steps.prebuilt.outputs.cache-hit }} - SMOKE: ${{ steps.smoke.outputs.ok }} + GH_TOKEN: ${{ github.token }} run: | - if [ "$HIT" = "true" ] && [ "$SMOKE" = "true" ]; then - echo "Prebuilt addons restored and verified; skipping bazel." - echo "needed=false" >> "$GITHUB_OUTPUT" - else - echo "needed=true" >> "$GITHUB_OUTPUT" + if gh pr diff ${{ github.event.pull_request.number }} --name-only 2>/dev/null \ + | grep -qE '^(crates/|bazel/|Cargo\.(toml|lock)|MODULE\.bazel(\.lock)?|BUILD\.bazel|\.bazelrc|\.bazelignore|\.bazelversion|rust-toolchain\.toml|rustfmt\.toml)'; then + echo "::notice title=Native sources changed::PR CI tests against the latest release addons by design; native changes are validated post-merge on main and at release." fi + + # ---- main path: bazel build of all Linux-hosted targets ---- - id: cache - if: steps.decide.outputs.needed == 'true' + if: github.event_name != 'pull_request' uses: ./.github/actions/bazel-cache with: scope: linux - name: Build native addons once - if: steps.decide.outputs.needed == 'true' - env: - EVENT_NAME: ${{ github.event_name }} + if: github.event_name != 'pull_request' run: | set -eo pipefail if [ "$(ulimit -Sn)" != unlimited ] && [ "$(ulimit -Sn)" -lt 65536 ]; then ulimit -Sn 65536 || true; fi - targets=(//:natives-linux-x64-baseline //:natives-linux-x64-modern) - if [ "$EVENT_NAME" != "pull_request" ]; then - targets=(//:natives-linux-all) - fi - bazelisk --bazelrc="${{ steps.cache.outputs.rc }}" build "${targets[@]}" 2>&1 | tee "$RUNNER_TEMP/bazel-build.log" + bazelisk --bazelrc="${{ steps.cache.outputs.rc }}" build //:natives-linux-all 2>&1 | tee "$RUNNER_TEMP/bazel-build.log" # Cache-hit visibility: a supposedly warm build that executes # thousands of actions is the failure mode that made CI slow — make # it visible in the run summary instead of discovering it weeks in. - name: Report bazel cache stats - if: steps.decide.outputs.needed == 'true' + if: github.event_name != 'pull_request' shell: bash run: | # Bazel runs with --color=yes (config=ci); strip ANSI before @@ -280,57 +249,7 @@ jobs: # the raw log. summary=$(sed -E 's/\x1b\[[0-9;]*m//g' "$RUNNER_TEMP/bazel-build.log" | grep -E '[0-9]+ processes:' | tail -1 || true) echo "::notice title=Bazel build summary::${summary:-no process summary found}" - - name: Save Bazel disk cache - if: steps.cache.outputs.save-needed == 'true' - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ~/.cache/omp-bazel-disk - key: ${{ steps.cache.outputs.cache-key }} - # Producer side of the fast path: trusted main builds publish the - # x64 pair under the content-addressed key. Smoke-load before save — - # an exact-key archive is never overwritten, so bad bytes would - # poison every TS-only PR until manual eviction. - - name: Stage built addons for cache - if: github.event_name != 'pull_request' - shell: bash - run: | - set -euo pipefail - mkdir -p ~/.cache/omp-native-addons - cp bazel-bin/natives-linux-x64-baseline/pi_natives.linux-x64-baseline.node \ - bazel-bin/natives-linux-x64-modern/pi_natives.linux-x64-modern.node \ - ~/.cache/omp-native-addons/ - - name: Smoke addons before caching - if: github.event_name != 'pull_request' - shell: bash - run: | - set -euo pipefail - cd ~/.cache/omp-native-addons - cat > "$RUNNER_TEMP/smoke-addons.js" <<'EOF' - for (const f of process.argv.slice(2)) { - const m = require(f); - if (!m || Object.keys(m).length === 0) { - console.error(`addon failed to load: ${f}`); - process.exit(1); - } - } - EOF - bun "$RUNNER_TEMP/smoke-addons.js" \ - "$PWD/pi_natives.linux-x64-baseline.node" \ - "$PWD/pi_natives.linux-x64-modern.node" - - name: Look up native addon cache - id: addon-cache - if: github.event_name != 'pull_request' - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ~/.cache/omp-native-addons - key: ${{ steps.inputs.outputs.cache-key }} - lookup-only: true - - name: Save native addon cache - if: github.event_name != 'pull_request' && steps.addon-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ~/.cache/omp-native-addons - key: ${{ steps.inputs.outputs.cache-key }} + - name: Upload native addon artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/docs/natives-build-release-debugging.md b/docs/natives-build-release-debugging.md index b2886e8fa..32ead96ab 100644 --- a/docs/natives-build-release-debugging.md +++ b/docs/natives-build-release-debugging.md @@ -136,9 +136,11 @@ build --tls_certificate=infra/bazel-remote/ca.crt ### Split Rust validation and addon production -`.github/workflows/ci.yml` separates `rust_validate` from `native_addons`. Both run on `omp-kata` pods for pushes and `ubuntu-22.04` for pull requests, but TypeScript jobs depend only on `native_addons`. +`.github/workflows/ci.yml` separates `rust_validate` from `native_addons`; TypeScript jobs depend only on `native_addons`. -`rust_validate` uses `.github/actions/native-inputs` to inspect the complete pull-request file list. TypeScript-only pull requests skip every Rust step; native-affecting changes and all non-PR events run: +**Pull requests never build or validate Rust.** Native-affecting PRs are rare enough that they don't warrant a PR-side bazel build: `rust_validate` is skipped entirely (`if: github.event_name != 'pull_request'`), and `native_addons` fetches the latest release's Linux x64 addon pair from the `@oh-my-pi/pi-natives-linux-x64` npm leaf, smoke-loads both, and uploads them as the `native-addons` workflow artifact. The loader skips its version sentinel for workspace loads, so release-versioned addons load fine under a newer checkout. A PR whose TypeScript tests depend on changed native behavior fails visibly (and CI emits a notice on any native-touching PR); the Rust side is validated post-merge on main and again at release. + +On non-PR events both jobs run on `omp-kata` pods against the cluster remote cache. `rust_validate` runs: ```bash bazelisk --bazelrc="$rc" test //crates/... # full Rust suite @@ -153,19 +155,14 @@ bazelisk --bazelrc="$rc" build --config=rustfmt //crates/... - `--config=clippy` = rules_rust clippy aspect + `-Dwarnings`; `--config=clippy-strict` layers the generated `bazel/clippy.bazelrc` for crates with `[lints] workspace = true`. - `--config=rustfmt` = rustfmt aspect against the workspace `rustfmt.toml`. -- `rust_validate` never saves a hosted disk-cache archive: `native_addons` may concurrently own the same immutable key, while the main-branch warmer publishes a combined validation/addon archive. -`native_addons` is the artifact producer for every downstream TypeScript and release job: - -- Pull requests first restore the exact `native-addons-v1-linux-x64-baseline+modern-opt-` cache entry published by trusted main builds. Both addons are loaded before use; a miss or failed smoke check falls back to building the Linux x64 pair. -- Main and other non-PR runs build `//:natives-linux-all`, smoke the x64 pair before publishing its exact addon cache, and upload every `.node` output as the `native-addons` workflow artifact. -- Downstream jobs use `.github/actions/native-artifacts` to download that workflow artifact and install the requested target set without invoking Bazel. +`native_addons` on main builds `//:natives-linux-all` and uploads every `.node` output as the `native-addons` workflow artifact. Downstream jobs use `.github/actions/native-artifacts` to download that artifact and install the requested target set without invoking Bazel. No toolchain setup steps are required for native jobs: bazelisk is on the GitHub images and baked into the kata runner image; Bazel fetches Rust/zig/LLVM/xwin hermetically. ### Hosted cache warmer -`.github/workflows/bazel-cache-warm.yml` runs the full hosted validation and Linux x64 addon invocation set on `ubuntu-22.04`. Main-branch input changes restore the previous config-compatible generation, rebuild incrementally, and publish one combined exact-key disk-cache archive visible to pull requests. +`.github/workflows/bazel-cache-warm.yml` seeds the GitHub-hosted caches that have no other reliable producer: the `release-darwin-*` bazel disk caches (built on the same macOS images as the `release_binary` darwin matrix, so a release's bazel build is the version-bump delta instead of a ~40-min cold graph) and the shared bun store entry PR jobs restore but never save. It triggers only on pushes that can change those archives (crate/bazel/lock inputs, `bun.lock`, `.github/**`). ### `bazel-cache` action (`.github/actions/bazel-cache`) @@ -176,7 +173,7 @@ Single source of truth for cache wiring, emitted as a bazelrc fragment (its `rc` | omp-kata pod | `--config=ci --config=cache-rw --remote_cache=grpcs://bazel-remote.bazel-cache.svc.cluster.local:9092 --tls_certificate=infra/bazel-remote/ca.crt --remote_header='authorization=Basic '` | | GitHub-hosted | `--config=ci --disk_cache=~/.cache/omp-bazel-disk --repository_cache=~/.cache/omp-bazel-repo` | -Hosted disk caches use `bazel-disk-v3-----`. The config hash covers Cargo/Bazel/toolchain settings; the source hash covers `crates/**` and root `BUILD.bazel`. An exact miss restores the newest config-compatible generation and permits one refreshed exact-key save. The remote endpoint resolves only inside the cluster. +Hosted disk caches use `bazel-disk-v3-----`. The config hash covers Cargo/Bazel/toolchain settings; the source hash covers `crates/**` and root `BUILD.bazel`. Restores fall back from the exact key to the config-scoped prefix, then to a bare `--` prefix — the bare fallback is what keeps release version bumps (which rewrite `Cargo.toml`/`Cargo.lock` and thus the config hash) from rebuilding cold; bazel's content-addressed action keys make a stale archive a partial hit, never a wrong output. An inexact restore permits one refreshed exact-key save, and restored entries untouched for 14 days are pruned so archives don't grow without bound. The remote endpoint resolves only inside the cluster. ### Native artifact actions @@ -228,7 +225,7 @@ bazelisk build --nobuild //:natives-win32-x64-baseline ### Cache behavior - **omp-kata:** read-write gRPC to the in-cluster bazel-remote (`grpcs://bazel-remote.bazel-cache.svc.cluster.local:9092`, TLS via the committed `infra/bazel-remote/ca.crt`, htpasswd user `ci`). `--remote_local_fallback` plus retries make an outage degrade to local execution rather than fail the build. -- **GitHub-hosted:** no cluster access. The v3 `actions/cache` disk key separates config and source generations; `.github/workflows/bazel-cache-warm.yml` publishes the combined default-branch archive from the same `ubuntu-22.04` image as pull-request consumers. The smaller final-addon cache is independent and is trusted only after both addons load successfully. +- **GitHub-hosted:** no cluster access; only the darwin release/warm jobs build with bazel here. The v3 `actions/cache` disk key separates config and source generations with prefix + bare fallbacks (see the `bazel-cache` action section above); `.github/workflows/bazel-cache-warm.yml` publishes the `release-darwin-*` archives from the same macOS images as the release consumers. - **msvc repos:** the ~2 GiB LLVM download is sha256-pinned and repository-cache backed; the ~1 GiB xwin CRT/SDK splat is fetched from the Microsoft CDN inside the repo rule and is **not** repo-cache backed — a cold output base re-downloads it. Microsoft advances the VS channel payload over time, so remote-cache hit rates for win32 actions degrade gracefully after an MS bump (same property the previous cross toolchain had). Win32 link actions also don't share cache entries across host OSes (linux vs mac clang binaries). - Server-side operations (deploy, TLS/auth, egress, poisoning boundary): `infra/docs/04-arc-and-caching.md` §5.