diff --git a/.github/actions/build-native/action.yml b/.github/actions/build-native/action.yml index b9f202561..cfeb6d52a 100644 --- a/.github/actions/build-native/action.yml +++ b/.github/actions/build-native/action.yml @@ -42,8 +42,17 @@ inputs: description: Run clippy/rustfmt checks (only one matrix entry should set this) required: false default: "false" + skip_validation: + description: > + Skip clippy/rustfmt and the Rust test suite. Set on release runs: the + version-bump commit only changes version strings, and the tagged + content's Rust code already passed validation on its main-push run. + required: false + default: "false" save_cache: - description: Whether Swatinem/rust-cache should write a cache entry (GitHub-hosted only) + description: > + Whether to write build caches: Swatinem/rust-cache on GitHub-hosted + runners, the RustFS target/ snapshot on omp-kata. required: false default: "false" @@ -186,8 +195,9 @@ runs: # forever). The native source hash is in the shared key too: rust-cache's # lockfile scan misses the workspace-root Cargo.toml version Cargo # fingerprints, so a version bump could otherwise get an exact hit for - # artifacts Cargo must rebuild. On omp-kata the mounted Cargo registry + - # RustFS sccache are the reuse layers, so there is no second cache restore. + # artifacts Cargo must rebuild. On omp-kata the reuse layers are the + # mounted Cargo registry, RustFS sccache, and the RustFS target/ snapshot + # (see "Restore target/ cache" below), so Swatinem stays GitHub-only. - name: Cache Rust target/ (GitHub-hosted) if: steps.detect.outputs.on_infra == 'false' uses: Swatinem/rust-cache@v2 @@ -212,11 +222,23 @@ runs: # conditional: omp-kata reads the shared S3 (RustFS) config from the # inherited pod env; GitHub-hosted runners use the GHA cache backend. shell: bash + env: + CROSS_TARGET: ${{ steps.resolve.outputs.cross_target }} run: | { echo "RUSTC_WRAPPER=sccache" echo "CARGO_INCREMENTAL=0" } >> "$GITHUB_ENV" + # Route CMake-built C (audiopus_sys' bundled opus) through sccache + # too — build scripts bypass RUSTC_WRAPPER. Non-cross builds only: + # cross builds compile C with zig cc / clang-cl wrapper scripts that + # sccache may fail to classify, which would hard-fail the compile. + if [ -z "$CROSS_TARGET" ]; then + { + echo "CMAKE_C_COMPILER_LAUNCHER=sccache" + echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" + } >> "$GITHUB_ENV" + fi if [ -n "${SCCACHE_BUCKET:-}" ]; then echo "sccache backend: shared S3 ($SCCACHE_BUCKET @ $SCCACHE_ENDPOINT)" else @@ -295,9 +317,23 @@ runs: shell: bash run: echo "XWIN_ACCEPT_LICENSE=1" >> "$GITHUB_ENV" + # --- target/ cache (omp-kata) -------------------------------------------- + # sccache only covers rustc invocations; build-script outputs (57 + # tree-sitter grammar C compiles, bundled opus via CMake, ring asm) and + # cargo's fingerprint/link work bypass it. Snapshot target/ to the same + # RustFS S3 bucket, keyed per platform/libc/arch/variant + toolchain and + # overwritten on each save so storage stays bounded at one snapshot per + # key. GitHub-hosted runners get the same effect from Swatinem above. + - name: Restore target/ cache (omp-kata) + if: steps.detect.outputs.on_infra == 'true' + shell: bash + env: + TARGET_CACHE_KEY: native-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }} + run: bun scripts/ci-target-cache.ts restore "$TARGET_CACHE_KEY" + # --- Checks, build, upload (shared) ------------------------------------- - name: Rust checks - if: inputs.rust_checks == 'true' + if: inputs.rust_checks == 'true' && inputs.skip_validation != 'true' shell: bash run: bun run check:rs - name: Test workspace (Rust) @@ -305,7 +341,7 @@ runs: # Windows-only tests are no longer exercised in CI (win32-x64 cross-builds # on Linux). Skipping the duplicate Linux runs on macOS saves ~10 min of # parallel runner time. - if: inputs.target == '' && inputs.platform != 'darwin' + if: inputs.target == '' && inputs.platform != 'darwin' && inputs.skip_validation != 'true' shell: bash run: bun run test:rs - name: Build native addon(s) @@ -316,6 +352,9 @@ runs: TARGET_ARCH: ${{ inputs.arch }} TARGET_VARIANTS: ${{ inputs.variant }} run: bun run ci:build:native + - name: sccache stats + shell: bash + run: sccache --show-stats || true - name: Upload native addon(s) uses: actions/upload-artifact@v4 with: @@ -326,3 +365,9 @@ runs: # org defaults shift; bump if Rust source ever stays stable for >90 days # of main pushes and you want to avoid rebuilds. retention-days: 90 + - name: Save target/ cache (omp-kata) + if: steps.detect.outputs.on_infra == 'true' && inputs.save_cache == 'true' + shell: bash + env: + TARGET_CACHE_KEY: native-${{ inputs.platform }}-${{ inputs.libc || 'default' }}-${{ inputs.arch }}-${{ inputs.variant || 'default' }} + run: bun scripts/ci-target-cache.ts save "$TARGET_CACHE_KEY" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d0a69165..a19eae985 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -214,7 +214,9 @@ jobs: # Linux x64 baseline + modern: required by `test`, so it runs on every PR # unless native_artifact_lookup found a cached run. Release runs always - # rebuild for fresh artifacts. + # rebuild for fresh artifacts but skip clippy + Rust tests (skip_validation): + # the bump commit only changes version strings over content that already + # passed validation on its main-push run. native_linux_x64: name: "Native: Linux x64 (${{ matrix.variant }})" needs: [release_metadata, native_artifact_lookup] @@ -236,6 +238,7 @@ jobs: variant: ${{ matrix.variant }} glibc: ${{ env.GLIBC_FLOOR }} rust_checks: ${{ matrix.rust_checks && 'true' || 'false' }} + skip_validation: ${{ needs.release_metadata.outputs.is-release }} save_cache: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} # Pre-warm the cross-platform native build cache on `main`, in addition to diff --git a/package.json b/package.json index 90c02b3ca..98a1a7630 100644 --- a/package.json +++ b/package.json @@ -117,7 +117,7 @@ "build:native": "bun --cwd=packages/natives run build", "test": "bun scripts/ci-test-ts.ts local", "test:ts": "bun scripts/ci-test-ts.ts local-ts", - "test:scripts": "bun test scripts/ci-build-native.test.ts scripts/ci-concurrency.test.ts scripts/ci-release-build-binaries.test.ts scripts/ci-release-notes.test.ts scripts/ci-release-publish.test.ts scripts/fix-dts-extensions.test.ts scripts/link-omp.test.ts scripts/musl-release.test.ts", + "test:scripts": "bun test scripts/ci-build-native.test.ts scripts/ci-concurrency.test.ts scripts/ci-release-build-binaries.test.ts scripts/ci-release-notes.test.ts scripts/ci-release-publish.test.ts scripts/ci-target-cache.test.ts scripts/fix-dts-extensions.test.ts scripts/link-omp.test.ts scripts/musl-release.test.ts", "test:rs": "bun scripts/run-rs-task.ts test:rs", "check": "bun run --parallel check:ts check:rs", "check:ts": "bun run check:tools && bun run --workspaces --if-present check", diff --git a/scripts/ci-target-cache.test.ts b/scripts/ci-target-cache.test.ts new file mode 100644 index 000000000..3c4c9f9b4 --- /dev/null +++ b/scripts/ci-target-cache.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; +import { objectKeyFor, resolveEndpoint } from "./ci-target-cache"; + +describe("resolveEndpoint", () => { + test("selects scheme from SCCACHE_S3_USE_SSL, defaulting to http for in-cluster RustFS", () => { + expect(resolveEndpoint({ SCCACHE_ENDPOINT: "rustfs.sccache.svc.cluster.local:9000" })).toBe( + "http://rustfs.sccache.svc.cluster.local:9000", + ); + expect(resolveEndpoint({ SCCACHE_ENDPOINT: "rustfs:9000", SCCACHE_S3_USE_SSL: "true" })).toBe( + "https://rustfs:9000", + ); + expect(resolveEndpoint({ SCCACHE_ENDPOINT: "rustfs:9000", SCCACHE_S3_USE_SSL: "false" })).toBe( + "http://rustfs:9000", + ); + }); + + test("passes through endpoints that already carry a scheme and rejects missing config", () => { + expect(resolveEndpoint({ SCCACHE_ENDPOINT: "https://s3.example.com" })).toBe("https://s3.example.com"); + expect(resolveEndpoint({})).toBeNull(); + expect(resolveEndpoint({ SCCACHE_ENDPOINT: " " })).toBeNull(); + }); +}); + +describe("objectKeyFor", () => { + test("namespaces snapshots under target-cache/ and separates toolchains", () => { + const stable = objectKeyFor("native-linux-default-x64-baseline", "rustc 1.91.0-nightly (abc 2026-04-29)"); + expect(stable).toMatch(/^target-cache\/native-linux-default-x64-baseline-[0-9a-f]{12}\.tar\.zst$/); + // Same inputs must be deterministic; a toolchain bump must be a clean miss. + expect(objectKeyFor("native-linux-default-x64-baseline", "rustc 1.91.0-nightly (abc 2026-04-29)")).toBe(stable); + expect(objectKeyFor("native-linux-default-x64-baseline", "rustc 1.92.0-nightly (def 2026-06-01)")).not.toBe( + stable, + ); + }); + + test("rejects keys that could escape the target-cache/ prefix", () => { + expect(() => objectKeyFor("../sccache-poison", "rustc 1.91.0")).toThrow(/Invalid cache key/); + expect(() => objectKeyFor("a/b", "rustc 1.91.0")).toThrow(/Invalid cache key/); + expect(() => objectKeyFor("", "rustc 1.91.0")).toThrow(/Invalid cache key/); + }); +}); diff --git a/scripts/ci-target-cache.ts b/scripts/ci-target-cache.ts new file mode 100755 index 000000000..da7e2efc1 --- /dev/null +++ b/scripts/ci-target-cache.ts @@ -0,0 +1,207 @@ +#!/usr/bin/env bun + +/** + * Persist the cargo `target/` directory to the in-cluster RustFS S3 bucket + * between omp-kata CI runs. + * + * sccache only caches rustc invocations; build-script outputs (57 tree-sitter + * grammar C compiles, audiopus_sys' bundled opus via CMake, ring's asm) and + * cargo's fingerprint/link work bypass it entirely. Restoring `target/` reuses + * all of that, so a warm native job only recompiles workspace crates. + * + * Storage model: one object per cache key (`target-cache/-.tar.zst`), + * overwritten on every save — storage stays bounded at one snapshot per + * platform/libc/arch/variant/toolchain. Staleness is safe: cargo fingerprints + * invalidate anything that no longer matches, exactly like Swatinem/rust-cache + * on the GitHub-hosted runners. + * + * Credentials/config come from the pod-wide sccache env (`SCCACHE_BUCKET`, + * `SCCACHE_ENDPOINT`, `SCCACHE_S3_USE_SSL`, `AWS_*`); Bun's S3Client reads the + * AWS credentials from the environment. Off-infra (no `SCCACHE_BUCKET`) and + * every failure path degrade to a logged no-op — this script must never fail + * a CI job. + * + * Usage: `bun scripts/ci-target-cache.ts ` + */ + +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { $, S3Client } from "bun"; + +const repoRoot = path.join(import.meta.dir, ".."); + +/** Compressed snapshots above this size are not uploaded; the next full miss rebuilds a compact one. */ +const MAX_SNAPSHOT_BYTES = 4 * 1024 ** 3; +const EXISTS_TIMEOUT_MS = 30_000; +const DOWNLOAD_TIMEOUT_MS = 180_000; +const UPLOAD_TIMEOUT_MS = 300_000; + +/** + * Resolve the S3 endpoint URL from the sccache pod env. `SCCACHE_ENDPOINT` is + * host:port without a scheme; `SCCACHE_S3_USE_SSL=true` selects https, + * anything else http (in-cluster RustFS serves plain HTTP). A value that + * already carries a scheme is passed through untouched. + */ +export function resolveEndpoint(env: Record): string | null { + const endpoint = env.SCCACHE_ENDPOINT?.trim(); + if (!endpoint) return null; + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(endpoint)) return endpoint; + const scheme = env.SCCACHE_S3_USE_SSL === "true" ? "https" : "http"; + return `${scheme}://${endpoint}`; +} + +/** + * Object key for one snapshot. The toolchain fingerprint (`rustc -V`) is + * hashed in so a nightly bump becomes a clean miss instead of a useless + * multi-GB restore that cargo immediately invalidates. + */ +export function objectKeyFor(cacheKey: string, rustcVersion: string): string { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(cacheKey)) { + throw new Error(`Invalid cache key ${JSON.stringify(cacheKey)}; expected [A-Za-z0-9._-]+`); + } + const toolchain = new Bun.CryptoHasher("sha256").update(rustcVersion).digest("hex").slice(0, 12); + return `target-cache/${cacheKey}-${toolchain}.tar.zst`; +} + +function withTimeout(promise: Promise, ms: number, label: string): Promise { + const { promise: timeout, reject } = Promise.withResolvers(); + const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + +/** `target_directory` from cargo metadata (honors CARGO_TARGET_DIR/config), falling back to `/target`. */ +async function resolveTargetDir(): Promise { + const meta = await $`cargo metadata --no-deps --format-version 1`.cwd(repoRoot).quiet().nothrow(); + if (meta.exitCode === 0) { + try { + const parsed = meta.json() as { target_directory?: string }; + if (parsed.target_directory) return parsed.target_directory; + } catch { + // fall through to default + } + } + return path.join(repoRoot, "target"); +} + +async function restore(s3: S3Client, objectKey: string, targetDir: string): Promise { + const object = s3.file(objectKey); + if (!(await withTimeout(object.exists(), EXISTS_TIMEOUT_MS, "cache lookup"))) { + console.log(`target cache miss: ${objectKey}`); + return; + } + const tmpTar = path.join(Bun.env.RUNNER_TEMP ?? os.tmpdir(), `target-cache-${process.pid}.tar.zst`); + const started = Bun.nanoseconds(); + try { + // NB: `Bun.write(dest, new Response(s3file.stream()))` never resolves in + // Bun 1.3.x; iterating the stream into a FileSink works. + const download = async () => { + const sink = Bun.file(tmpTar).writer(); + for await (const chunk of object.stream()) sink.write(chunk); + await sink.end(); + }; + await withTimeout(download(), DOWNLOAD_TIMEOUT_MS, "cache download"); + const sizeMb = (Bun.file(tmpTar).size / 1024 ** 2).toFixed(0); + // Explicit decompress pipe: GNU tar passes -d to a --use-compress-program + // filter on extract but bsdtar does not, so filter flags are a trap. + // The pipe reports only tar's exit code, which reads a truncated zstd + // stream as a short-but-valid archive — so verify the zstd layer first. + const verify = await $`zstd -tq ${tmpTar}`.quiet().nothrow(); + const extract = + verify.exitCode === 0 + ? await $`zstd -dcq ${tmpTar} | tar -xf - -C ${path.dirname(targetDir)}`.quiet().nothrow() + : verify; + if (extract.exitCode !== 0) { + // A torn/corrupt snapshot must not leave a half-extracted target/ + // behind: cargo would trust whatever fingerprints survived. + await fs.rm(targetDir, { recursive: true, force: true }); + console.warn( + `target cache extract failed (exit ${extract.exitCode}); removed ${targetDir} and continuing cold`, + ); + return; + } + const secs = ((Bun.nanoseconds() - started) / 1e9).toFixed(1); + console.log(`target cache restored: ${objectKey} (${sizeMb} MiB in ${secs}s)`); + } finally { + await fs.rm(tmpTar, { force: true }); + } +} + +async function save(s3: S3Client, objectKey: string, targetDir: string): Promise { + try { + await fs.stat(targetDir); + } catch { + console.log(`target cache save skipped: ${targetDir} does not exist`); + return; + } + const tmpTar = path.join(Bun.env.RUNNER_TEMP ?? os.tmpdir(), `target-cache-${process.pid}.tar.zst`); + const started = Bun.nanoseconds(); + try { + // CARGO_INCREMENTAL=0 in CI, so incremental/ only exists from stray + // local state; exclude it regardless — it is the one cargo dir that is + // pure dead weight for a cold consumer. Explicit compress pipe for the + // same tar-flavor reason as in restore(); -T0 uses all cores. + const create = + await $`tar -cf - --exclude=${"*/incremental"} -C ${path.dirname(targetDir)} ${path.basename(targetDir)} | zstd -q -T0 -3 -f -o ${tmpTar}` + .quiet() + .nothrow(); + if (create.exitCode !== 0) { + console.warn(`target cache save skipped: tar failed (exit ${create.exitCode})`); + return; + } + const size = Bun.file(tmpTar).size; + if (size > MAX_SNAPSHOT_BYTES) { + // Orphaned artifacts accumulate across restore→build→save cycles; + // refusing oversized uploads bounds the object. The stale snapshot + // keeps serving restores until a full-miss rebuild saves a compact one. + console.warn( + `target cache save skipped: snapshot ${(size / 1024 ** 3).toFixed(1)} GiB exceeds ${MAX_SNAPSHOT_BYTES / 1024 ** 3} GiB cap`, + ); + return; + } + await withTimeout(s3.write(objectKey, Bun.file(tmpTar)), UPLOAD_TIMEOUT_MS, "cache upload"); + const secs = ((Bun.nanoseconds() - started) / 1e9).toFixed(1); + console.log(`target cache saved: ${objectKey} (${(size / 1024 ** 2).toFixed(0)} MiB in ${secs}s)`); + } finally { + await fs.rm(tmpTar, { force: true }); + } +} + +async function main(): Promise { + const [mode, cacheKey] = [process.argv[2], process.argv[3]]; + if ((mode !== "restore" && mode !== "save") || !cacheKey) { + console.error("Usage: bun scripts/ci-target-cache.ts "); + process.exit(1); + } + + const bucket = Bun.env.SCCACHE_BUCKET; + const endpoint = resolveEndpoint(Bun.env); + if (!bucket || !endpoint) { + console.log("target cache skipped: no SCCACHE_BUCKET/SCCACHE_ENDPOINT in env (not on omp-kata infra)"); + return; + } + if (!Bun.which("zstd")) { + console.warn("target cache skipped: zstd not on PATH"); + return; + } + const rustc = await $`rustc -V`.quiet().nothrow(); + if (rustc.exitCode !== 0) { + console.warn("target cache skipped: rustc not on PATH"); + return; + } + + const objectKey = objectKeyFor(cacheKey, rustc.text().trim()); + const s3 = new S3Client({ bucket, endpoint, region: Bun.env.SCCACHE_REGION ?? Bun.env.AWS_REGION ?? "us-east-1" }); + const targetDir = await resolveTargetDir(); + if (mode === "restore") await restore(s3, objectKey, targetDir); + else await save(s3, objectKey, targetDir); +} + +if (import.meta.main) { + try { + await main(); + } catch (err) { + // Cache trouble must never fail a build; cold compile is the fallback. + console.warn(`target cache ${process.argv[2] ?? ""} failed non-fatally:`, err); + } +}