ci(workflows): shared bun caching in CI via backend-aware install action
- Updated CI dependency install flow to share bun cache orchestration across jobs. - Added RustFS-backed bun cache restore/save script keyed by bun.lock hash.
This commit is contained in:
@@ -129,8 +129,7 @@ runs:
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
- shell: bash
|
||||
run: bun install --frozen-lockfile
|
||||
- uses: ./.github/actions/bun-install
|
||||
# Cross-compile toolchain selection: non-MSVC targets (e.g.
|
||||
# `aarch64-unknown-linux-gnu`) build with `cargo-zigbuild`; MSVC targets
|
||||
# (e.g. `x86_64-pc-windows-msvc`) build with `cargo-xwin`. The napi CLI's
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
name: "bun install (shared cache)"
|
||||
description: >
|
||||
Run `bun install --frozen-lockfile` with a shared dependency cache. On
|
||||
self-hosted omp-kata runners (where the sccache S3 credentials are injected)
|
||||
the bun store and node_modules are cached in the in-cluster RustFS S3; every
|
||||
other runner falls back to the stock actions/cache backend.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
# The whole repo already keys "are we on hosted infra?" off
|
||||
# $SCCACHE_BUCKET (see actions/build-native). RUNNER_ENVIRONMENT is empty
|
||||
# on ARC pods, so it is not a usable signal here.
|
||||
- name: Detect cache backend
|
||||
id: backend
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${SCCACHE_BUCKET:-}" ] && [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then
|
||||
echo "kind=rustfs" >> "$GITHUB_OUTPUT"
|
||||
echo "bun cache backend: RustFS S3 ($SCCACHE_BUCKET @ $SCCACHE_ENDPOINT)"
|
||||
else
|
||||
echo "kind=gha" >> "$GITHUB_OUTPUT"
|
||||
echo "bun cache backend: GitHub Actions cache"
|
||||
fi
|
||||
|
||||
# Off-infra (GitHub-hosted): stock actions/cache for the bun store.
|
||||
- name: Cache bun store (GitHub cache)
|
||||
if: steps.backend.outputs.kind == 'gha'
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
|
||||
|
||||
# On-infra (omp-kata): RustFS store + node_modules over the LAN.
|
||||
- name: Restore bun caches (RustFS)
|
||||
if: steps.backend.outputs.kind == 'rustfs'
|
||||
shell: bash
|
||||
run: bash "$GITHUB_ACTION_PATH/rustfs-cache.sh" restore
|
||||
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
run: bun install --frozen-lockfile
|
||||
|
||||
- name: Save bun caches (RustFS)
|
||||
if: steps.backend.outputs.kind == 'rustfs'
|
||||
shell: bash
|
||||
run: bash "$GITHUB_ACTION_PATH/rustfs-cache.sh" save
|
||||
Executable
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared bun dependency cache backed by the in-cluster RustFS (S3) object store.
|
||||
#
|
||||
# Used by .github/actions/bun-install on the self-hosted omp-kata runners. There
|
||||
# the stock `actions/cache` restore of ~/.bun/install/cache costs 130-186s per
|
||||
# job because GitHub's cache backend is only reachable over the node's NAT
|
||||
# egress, and ~9 jobs contend on it at once. RustFS lives in the same k3s node
|
||||
# (svc :9000, already allowed by the runner egress NetworkPolicy), so the same
|
||||
# payload moves at LAN speed.
|
||||
#
|
||||
# Credentials are the ones sccache already gets via the `sccache-s3` secret
|
||||
# (envFrom on every runner pod): AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY /
|
||||
# SCCACHE_ENDPOINT / SCCACHE_BUCKET / SCCACHE_REGION / SCCACHE_S3_USE_SSL.
|
||||
#
|
||||
# Two objects per lockfile, under the bun-cache/ key prefix of the sccache
|
||||
# bucket:
|
||||
# store-<os>-<lockhash> the bun global package store (~/.bun/install/cache)
|
||||
# nm-<os>-<lockhash> the installed node_modules trees (root + workspaces)
|
||||
# The store additionally publishes a rolling store-<os>-latest alias, so a
|
||||
# changed lockfile still warm-starts from the previous store and `bun install`
|
||||
# only fetches the delta. A node_modules hit short-circuits everything: the
|
||||
# subsequent `bun install --frozen-lockfile` is a no-op, so the store is neither
|
||||
# fetched nor saved.
|
||||
set -euo pipefail
|
||||
|
||||
mode="${1:?usage: rustfs-cache.sh restore|save}"
|
||||
|
||||
: "${SCCACHE_BUCKET:?SCCACHE_BUCKET required}"
|
||||
: "${SCCACHE_ENDPOINT:?SCCACHE_ENDPOINT required}"
|
||||
: "${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID required}"
|
||||
: "${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY required}"
|
||||
|
||||
region="${SCCACHE_REGION:-us-east-1}"
|
||||
if [ "${SCCACHE_S3_USE_SSL:-false}" = "true" ]; then scheme=https; else scheme=http; fi
|
||||
base="${scheme}://${SCCACHE_ENDPOINT}/${SCCACHE_BUCKET}/bun-cache"
|
||||
os="${RUNNER_OS:-$(uname -s)}"
|
||||
store_dir="${BUN_INSTALL_CACHE_DIR:-${HOME}/.bun/install/cache}"
|
||||
work="${RUNNER_TEMP:-/tmp}/bun-rustfs-cache"
|
||||
mkdir -p "$work"
|
||||
|
||||
# Prefer multi-threaded zstd (baked into the omp-kata runner image); fall back to
|
||||
# gzip so the action still works on an image that predates the zstd addition. The
|
||||
# object suffix records the codec, and restore only inflates archives this host
|
||||
# can actually decompress.
|
||||
if command -v zstd >/dev/null 2>&1; then
|
||||
tar_c=(-I "zstd -3 -T0"); ext="tzst"; alt_ext="tgz"
|
||||
else
|
||||
tar_c=(-I "gzip -6"); ext="tgz"; alt_ext="tzst"
|
||||
fi
|
||||
|
||||
lock_hash="$(sha256sum bun.lock | cut -c1-32)"
|
||||
store_key="store-${os}-${lock_hash}"
|
||||
store_latest="store-${os}-latest"
|
||||
nm_key="nm-${os}-${lock_hash}"
|
||||
|
||||
auth=(--aws-sigv4 "aws:amz:${region}:s3" --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}")
|
||||
# 404 (-f) and connection errors are non-zero; transient errors retry, 4xx do not.
|
||||
s3_get() { curl -fsS --retry 3 --retry-connrefused "${auth[@]}" "${base}/$1" -o "$2"; }
|
||||
s3_exists() { curl -fsS -I --retry 3 --retry-connrefused "${auth[@]}" "${base}/$1" -o /dev/null >/dev/null 2>&1; }
|
||||
s3_put() { curl -fsS --retry 3 --retry-connrefused "${auth[@]}" -T "$2" "${base}/$1" -o /dev/null; }
|
||||
|
||||
# Download <name>.<ext> (then the alternate codec) and extract into dir $2.
|
||||
# tar auto-detects the codec from the archive; a present-but-uninflatable archive
|
||||
# (codec mismatch with this host) is treated as a miss.
|
||||
fetch_extract() { # name dest
|
||||
local name="$1" dest="$2" e f
|
||||
for e in "$ext" "$alt_ext"; do
|
||||
f="${work}/${name}.${e}"
|
||||
if s3_get "${name}.${e}" "$f" 2>/dev/null; then
|
||||
mkdir -p "$dest"
|
||||
if tar -xf "$f" -C "$dest" 2>/dev/null; then rm -f "$f"; return 0; fi
|
||||
rm -f "$f"
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
case "$mode" in
|
||||
restore)
|
||||
if fetch_extract "$nm_key" "$PWD"; then
|
||||
echo "bun cache: node_modules HIT ($nm_key) — install becomes a no-op"
|
||||
: > "${work}/nm_hit"
|
||||
exit 0
|
||||
fi
|
||||
echo "bun cache: node_modules miss ($nm_key)"
|
||||
if fetch_extract "$store_key" "$store_dir"; then
|
||||
echo "bun cache: store HIT ($store_key)"
|
||||
elif fetch_extract "$store_latest" "$store_dir"; then
|
||||
echo "bun cache: store warm-start ($store_latest)"
|
||||
else
|
||||
echo "bun cache: store miss — cold install"
|
||||
fi
|
||||
;;
|
||||
save)
|
||||
if [ -f "${work}/nm_hit" ]; then
|
||||
echo "bun cache: node_modules was a hit — nothing to save"
|
||||
exit 0
|
||||
fi
|
||||
# Store (+ rolling latest): save when this exact lockfile has none yet.
|
||||
if [ -d "$store_dir" ] && ! s3_exists "${store_key}.${ext}"; then
|
||||
tar "${tar_c[@]}" -cf "${work}/store.${ext}" -C "$store_dir" .
|
||||
s3_put "${store_key}.${ext}" "${work}/store.${ext}"
|
||||
s3_put "${store_latest}.${ext}" "${work}/store.${ext}"
|
||||
rm -f "${work}/store.${ext}"
|
||||
echo "bun cache: saved store ($store_key + $store_latest)"
|
||||
fi
|
||||
# node_modules: save the installed trees for this exact lockfile.
|
||||
if ! s3_exists "${nm_key}.${ext}"; then
|
||||
shopt -s nullglob
|
||||
nm_paths=(node_modules packages/*/node_modules python/robomp/web/node_modules)
|
||||
if [ ${#nm_paths[@]} -gt 0 ]; then
|
||||
tar "${tar_c[@]}" -cf "${work}/nm.${ext}" "${nm_paths[@]}"
|
||||
s3_put "${nm_key}.${ext}" "${work}/nm.${ext}"
|
||||
rm -f "${work}/nm.${ext}"
|
||||
echo "bun cache: saved node_modules ($nm_key, ${#nm_paths[@]} trees)"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "rustfs-cache.sh: unknown mode '$mode' (want restore|save)" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -193,12 +193,7 @@ jobs:
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
- name: Cache bun dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
|
||||
- run: bun install --frozen-lockfile
|
||||
- uses: ./.github/actions/bun-install
|
||||
- name: Type check workspace
|
||||
run: bun run ci:check:full
|
||||
- name: Build collab web
|
||||
@@ -267,12 +262,7 @@ jobs:
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
- name: Cache bun dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
|
||||
- run: bun install --frozen-lockfile
|
||||
- uses: ./.github/actions/bun-install
|
||||
- name: Resolve Linux x64 native artifact run
|
||||
id: source
|
||||
shell: bash
|
||||
@@ -304,12 +294,7 @@ jobs:
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
- name: Cache bun dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
|
||||
- run: bun install --frozen-lockfile
|
||||
- uses: ./.github/actions/bun-install
|
||||
- name: Resolve Linux x64 native artifact run
|
||||
id: source
|
||||
shell: bash
|
||||
@@ -343,13 +328,8 @@ jobs:
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
- name: Cache bun dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
|
||||
- uses: ./.github/actions/setup-system-deps
|
||||
- run: bun install --frozen-lockfile
|
||||
- uses: ./.github/actions/bun-install
|
||||
- name: Resolve Linux x64 native artifact run
|
||||
id: source
|
||||
shell: bash
|
||||
@@ -381,13 +361,8 @@ jobs:
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
- name: Cache bun dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
|
||||
- uses: ./.github/actions/setup-system-deps
|
||||
- run: bun install --frozen-lockfile
|
||||
- uses: ./.github/actions/bun-install
|
||||
- name: Resolve Linux x64 native artifact run
|
||||
id: source
|
||||
shell: bash
|
||||
@@ -419,12 +394,7 @@ jobs:
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
- name: Cache bun dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
|
||||
- run: bun install --frozen-lockfile
|
||||
- uses: ./.github/actions/bun-install
|
||||
- name: Resolve Linux x64 native artifact run
|
||||
id: source
|
||||
shell: bash
|
||||
@@ -458,13 +428,8 @@ jobs:
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
- name: Cache bun dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
|
||||
- uses: ./.github/actions/setup-system-deps
|
||||
- run: bun install --frozen-lockfile
|
||||
- uses: ./.github/actions/bun-install
|
||||
- name: Resolve Linux x64 native artifact run
|
||||
id: source
|
||||
shell: bash
|
||||
@@ -496,13 +461,8 @@ jobs:
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
- name: Cache bun dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
|
||||
- uses: ./.github/actions/setup-system-deps
|
||||
- run: bun install --frozen-lockfile
|
||||
- uses: ./.github/actions/bun-install
|
||||
- name: Resolve Linux x64 native artifact run
|
||||
id: source
|
||||
shell: bash
|
||||
@@ -561,13 +521,8 @@ jobs:
|
||||
echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
|
||||
echo "sccache backend: GitHub Actions cache"
|
||||
fi
|
||||
- name: Cache bun dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
|
||||
- uses: ./.github/actions/setup-system-deps
|
||||
- run: bun install --frozen-lockfile
|
||||
- uses: ./.github/actions/bun-install
|
||||
- name: Install method smoke tests
|
||||
run: bun run ci:test:install-methods
|
||||
|
||||
|
||||
@@ -679,13 +679,14 @@ describe("agent() through eval runtimes", () => {
|
||||
cost: 0,
|
||||
durationMs: i * 10,
|
||||
});
|
||||
await Bun.sleep(5);
|
||||
await Bun.sleep(40);
|
||||
}
|
||||
return singleResult(options, { output: "done" });
|
||||
});
|
||||
|
||||
const ops: string[] = [];
|
||||
using idle = new IdleTimeout(40);
|
||||
// Timing invariant (keep, do not re-tighten): total mock work (20*40ms = 800ms) > idle window (250ms) > scheduling jitter (~tens of ms).
|
||||
using idle = new IdleTimeout(250);
|
||||
const result = await runEvalAgent(
|
||||
{ prompt: "investigate" },
|
||||
{
|
||||
|
||||
@@ -281,9 +281,10 @@ describe("editToolRenderer", () => {
|
||||
const component = new ToolExecutionComponent("edit", { input }, { snapshots }, hashlineTool, uiStub, tmpDir);
|
||||
|
||||
component.setArgsComplete();
|
||||
await Bun.sleep(50);
|
||||
|
||||
const rendered = Bun.stripANSI(component.render(160).join("\n"));
|
||||
// The preview diff computes asynchronously after args complete; poll
|
||||
// instead of a fixed sleep so the slower CI VM has time to finish it.
|
||||
const rendered = await waitForRenderedText(component, 160, "export const b = 22;");
|
||||
expect(rendered).toContain("export const b = 22;");
|
||||
expect(rendered).not.toContain("No changes would be made");
|
||||
} finally {
|
||||
|
||||
Reference in New Issue
Block a user