feat(build): migrated native pipeline to bazel with remote caching
- Replaced the napi-cli/cargo-zigbuild/cargo-xwin/sccache build path with Bazel: rules_rust + crate_universe over Cargo.lock, hermetic zig cc toolchains (linux-gnu pinned to glibc 2.17, linux-musl), host Xcode for darwin, and a repo-local hermetic clang-cl + llvm-ml + xwin toolchain for windows-msvc (bazel/toolchains/msvc). - All eight shipped addons build as //:natives-<target> via the release transition in bazel/defs.bzl (opt, thin LTO, cgu=16, stripped, canonical .node naming); scripts/bazel-natives.ts is the single driver for local dev and CI. - Rust validation moved to bazel test + clippy aspects (strict workspace policy for opted-in crates, default lints elsewhere, mirroring cargo semantics) and the rustfmt aspect; cargo stays as the dev-iteration surface, with brush-core/brush-builtins promoted to workspace members and excluded from cargo dev tasks to keep their historical scope. - CI caches through an in-cluster bazel-remote action cache (TLS + basic auth, cluster-internal only); GitHub-hosted runners never touch the infrastructure and use an actions/cache-backed disk cache instead. - Deleted the hand-rolled caching machinery: ci-target-cache, ci-native-artifact-cache, ci-build-native, native-source-hash, find-native-artifacts, restore-linux-native, native-prewarm workflow, ensure-* toolchain actions, and all sccache/Swatinem wiring. - Warm native rebuilds drop from ~20 minutes to seconds; a cold client with a warm remote cache rebuilds the linux x64 pair in ~2.5 minutes.
This commit is contained in:
can1357
@@ -0,0 +1,115 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
conventionOutputPaths,
|
||||
type HostInfo,
|
||||
hostTargetName,
|
||||
parseBazelFilesOutput,
|
||||
parseCliArgs,
|
||||
resolveTargetLabels,
|
||||
} from "./bazel-natives";
|
||||
|
||||
const linuxModern: HostInfo = { platform: "linux", arch: "x64", avx2: true };
|
||||
const linuxBaseline: HostInfo = { platform: "linux", arch: "x64", avx2: false };
|
||||
const macArm: HostInfo = { platform: "darwin", arch: "arm64", avx2: false };
|
||||
|
||||
describe("hostTargetName", () => {
|
||||
test("picks the x64 variant from AVX2 support", () => {
|
||||
expect(hostTargetName(linuxModern)).toBe("linux-x64-modern");
|
||||
expect(hostTargetName(linuxBaseline)).toBe("linux-x64-baseline");
|
||||
// darwin x64 ships baseline only; AVX2 must not invent a modern target.
|
||||
expect(hostTargetName({ platform: "darwin", arch: "x64", avx2: true })).toBe("darwin-x64-baseline");
|
||||
});
|
||||
|
||||
test("maps non-x64 and windows hosts", () => {
|
||||
expect(hostTargetName(macArm)).toBe("darwin-arm64");
|
||||
expect(hostTargetName({ platform: "linux", arch: "arm64", avx2: false })).toBe("linux-arm64");
|
||||
expect(hostTargetName({ platform: "win32", arch: "x64", avx2: true })).toBe("win32-x64-baseline");
|
||||
});
|
||||
|
||||
test("rejects hosts without an addon target", () => {
|
||||
expect(() => hostTargetName({ platform: "freebsd", arch: "x64", avx2: false })).toThrow(
|
||||
/No pi_natives addon target/,
|
||||
);
|
||||
expect(() => hostTargetName({ platform: "win32", arch: "arm64", avx2: false })).toThrow(
|
||||
/No pi_natives addon target/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveTargetLabels", () => {
|
||||
test("maps explicit names, pseudo-targets, and aggregates to labels", () => {
|
||||
expect(resolveTargetLabels(["linux-x64-baseline", "linux-x64-modern"], macArm)).toEqual([
|
||||
"//:natives-linux-x64-baseline",
|
||||
"//:natives-linux-x64-modern",
|
||||
]);
|
||||
expect(resolveTargetLabels(["host"], linuxModern)).toEqual(["//:natives-linux-x64-modern"]);
|
||||
expect(resolveTargetLabels(["linux-all", "darwin-all"], macArm)).toEqual([
|
||||
"//:natives-linux-all",
|
||||
"//:natives-darwin-all",
|
||||
]);
|
||||
});
|
||||
|
||||
test("deduplicates and rejects unknown targets", () => {
|
||||
expect(resolveTargetLabels(["host", "darwin-arm64"], macArm)).toEqual(["//:natives-darwin-arm64"]);
|
||||
expect(() => resolveTargetLabels(["linux-x64"], macArm)).toThrow(/Unknown native target "linux-x64"/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("conventionOutputPaths", () => {
|
||||
test("builds bazel-bin paths with canonical filenames (musl reuses linux names)", () => {
|
||||
expect(conventionOutputPaths(["linux-musl-x64-baseline", "win32-x64-baseline"], macArm)).toEqual([
|
||||
"bazel-bin/natives-linux-musl-x64-baseline/pi_natives.linux-x64-baseline.node",
|
||||
"bazel-bin/natives-win32-x64-baseline/pi_natives.win32-x64-baseline.node",
|
||||
]);
|
||||
});
|
||||
|
||||
test("expands aggregates and the host pseudo-target", () => {
|
||||
expect(conventionOutputPaths(["darwin-all"], macArm)).toEqual([
|
||||
"bazel-bin/natives-darwin-arm64/pi_natives.darwin-arm64.node",
|
||||
"bazel-bin/natives-darwin-x64-baseline/pi_natives.darwin-x64-baseline.node",
|
||||
]);
|
||||
expect(conventionOutputPaths(["host"], linuxBaseline)).toEqual([
|
||||
"bazel-bin/natives-linux-x64-baseline/pi_natives.linux-x64-baseline.node",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseBazelFilesOutput", () => {
|
||||
test("keeps only .node paths, trimmed and deduplicated", () => {
|
||||
const output = [
|
||||
"bazel-bin/natives-linux-x64-baseline/pi_natives.linux-x64-baseline.node",
|
||||
" bazel-bin/natives-linux-x64-modern/pi_natives.linux-x64-modern.node ",
|
||||
"bazel-bin/natives-linux-x64-baseline/pi_natives.linux-x64-baseline.node",
|
||||
"INFO: Analyzed 2 targets (0 packages loaded, 0 targets configured).",
|
||||
"",
|
||||
].join("\n");
|
||||
expect(parseBazelFilesOutput(output)).toEqual([
|
||||
"bazel-bin/natives-linux-x64-baseline/pi_natives.linux-x64-baseline.node",
|
||||
"bazel-bin/natives-linux-x64-modern/pi_natives.linux-x64-modern.node",
|
||||
]);
|
||||
});
|
||||
|
||||
test("returns empty for output without addon files", () => {
|
||||
expect(parseBazelFilesOutput("INFO: Build completed successfully\n")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseCliArgs", () => {
|
||||
test("splits targets, --dest, and passthrough bazel args", () => {
|
||||
expect(
|
||||
parseCliArgs(["linux-x64-baseline", "linux-x64-modern", "--dest", "out", "--", "--config=ci", "--dest"]),
|
||||
).toEqual({
|
||||
targets: ["linux-x64-baseline", "linux-x64-modern"],
|
||||
dest: "out",
|
||||
bazelArgs: ["--config=ci", "--dest"],
|
||||
});
|
||||
expect(parseCliArgs(["host"])).toEqual({ targets: ["host"], dest: null, bazelArgs: [] });
|
||||
});
|
||||
|
||||
test("rejects missing targets, stray flags, and a valueless --dest", () => {
|
||||
expect(() => parseCliArgs([])).toThrow(/Usage:/);
|
||||
expect(() => parseCliArgs(["--", "--config=ci"])).toThrow(/Usage:/);
|
||||
expect(() => parseCliArgs(["host", "--config=ci"])).toThrow(/Unknown flag --config=ci/);
|
||||
expect(() => parseCliArgs(["host", "--dest"])).toThrow(/--dest requires/);
|
||||
});
|
||||
});
|
||||
Executable
+274
@@ -0,0 +1,274 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Canonical Bazel driver for the shipping pi_natives addons.
|
||||
*
|
||||
* Usage: bun scripts/bazel-natives.ts <target>... [--dest <dir>] [-- <extra bazel args>]
|
||||
*
|
||||
* Targets are the //:natives-* names from BUILD.bazel (e.g. linux-x64-baseline,
|
||||
* darwin-arm64) plus three pseudo-targets:
|
||||
* - host the single addon matching this machine (x64 hosts pick
|
||||
* modern vs baseline via AVX2 detection)
|
||||
* - linux-all every addon buildable from a linux-x64 host (incl. win32)
|
||||
* - darwin-all both darwin addons (mac hosts only)
|
||||
*
|
||||
* One `bazel build` covers all requested targets; outputs are located via
|
||||
* `bazel cquery --output=files` (falling back to the bazel-bin path convention)
|
||||
* and copied dereferenced into --dest (default packages/natives/native).
|
||||
*
|
||||
* Extra args after `--` are passed to bazel verbatim (cache configs, endpoints,
|
||||
* headers — see .bazelrc for the cache-rw/cache-ro policy configs).
|
||||
*
|
||||
* Note: musl addons intentionally reuse the plain linux-<arch> filenames, so a
|
||||
* `linux-all` copy overwrites the gnu addon with the musl one (and vice versa);
|
||||
* CI jobs that ship files always request an explicit disjoint target set.
|
||||
*/
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as path from "node:path";
|
||||
import { detectHostAvx2Support } from "./host-detect";
|
||||
|
||||
const repoRoot = path.join(import.meta.dir, "..");
|
||||
|
||||
/** //:natives-<name> → canonical addon filename (mirrors _ADDONS in BUILD.bazel). */
|
||||
export const ADDON_OUTPUTS: Record<string, string> = {
|
||||
"linux-x64-baseline": "pi_natives.linux-x64-baseline.node",
|
||||
"linux-x64-modern": "pi_natives.linux-x64-modern.node",
|
||||
"linux-arm64": "pi_natives.linux-arm64.node",
|
||||
"linux-musl-x64-baseline": "pi_natives.linux-x64-baseline.node",
|
||||
"linux-musl-arm64": "pi_natives.linux-arm64.node",
|
||||
"darwin-x64-baseline": "pi_natives.darwin-x64-baseline.node",
|
||||
"darwin-arm64": "pi_natives.darwin-arm64.node",
|
||||
"win32-x64-baseline": "pi_natives.win32-x64-baseline.node",
|
||||
};
|
||||
|
||||
/** Aggregate filegroups → their member addon targets (mirrors BUILD.bazel). */
|
||||
export const AGGREGATE_TARGETS: Record<string, string[]> = {
|
||||
"linux-all": [
|
||||
"linux-arm64",
|
||||
"linux-musl-arm64",
|
||||
"linux-musl-x64-baseline",
|
||||
"linux-x64-baseline",
|
||||
"linux-x64-modern",
|
||||
"win32-x64-baseline",
|
||||
],
|
||||
"darwin-all": ["darwin-arm64", "darwin-x64-baseline"],
|
||||
};
|
||||
|
||||
export interface HostInfo {
|
||||
platform: string;
|
||||
arch: string;
|
||||
avx2: boolean;
|
||||
}
|
||||
|
||||
/** The single addon target matching the host CPU (modern iff x64 + AVX2). */
|
||||
export function hostTargetName(host: HostInfo): string {
|
||||
if (host.platform === "darwin") {
|
||||
if (host.arch === "arm64") return "darwin-arm64";
|
||||
if (host.arch === "x64") return "darwin-x64-baseline";
|
||||
}
|
||||
if (host.platform === "linux") {
|
||||
if (host.arch === "arm64") return "linux-arm64";
|
||||
if (host.arch === "x64") return host.avx2 ? "linux-x64-modern" : "linux-x64-baseline";
|
||||
}
|
||||
if (host.platform === "win32" && host.arch === "x64") return "win32-x64-baseline";
|
||||
throw new Error(`No pi_natives addon target for host ${host.platform}-${host.arch}`);
|
||||
}
|
||||
|
||||
/** Expand pseudo-targets and map names to //:natives-* labels (deduplicated). */
|
||||
export function resolveTargetLabels(names: string[], host: HostInfo): string[] {
|
||||
const labels: string[] = [];
|
||||
for (const name of names) {
|
||||
const resolved = name === "host" ? hostTargetName(host) : name;
|
||||
if (!(resolved in AGGREGATE_TARGETS) && !(resolved in ADDON_OUTPUTS)) {
|
||||
const known = [...Object.keys(ADDON_OUTPUTS), ...Object.keys(AGGREGATE_TARGETS), "host"].join(", ");
|
||||
throw new Error(`Unknown native target "${name}". Known targets: ${known}`);
|
||||
}
|
||||
const label = `//:natives-${resolved}`;
|
||||
if (!labels.includes(label)) labels.push(label);
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
|
||||
/**
|
||||
* Workspace-relative output paths by bazel-bin convention:
|
||||
* bazel-bin/natives-<t>/<canonical>.node. Fallback when cquery is unavailable.
|
||||
*/
|
||||
export function conventionOutputPaths(names: string[], host: HostInfo): string[] {
|
||||
const paths: string[] = [];
|
||||
for (const name of names) {
|
||||
const resolved = name === "host" ? hostTargetName(host) : name;
|
||||
const members = AGGREGATE_TARGETS[resolved] ?? [resolved];
|
||||
for (const member of members) {
|
||||
const out = ADDON_OUTPUTS[member];
|
||||
if (!out) throw new Error(`Unknown native target "${name}"`);
|
||||
const p = `bazel-bin/natives-${member}/${out}`;
|
||||
if (!paths.includes(p)) paths.push(p);
|
||||
}
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
/** Parse `bazel cquery --output=files` stdout into deduplicated .node paths. */
|
||||
export function parseBazelFilesOutput(output: string): string[] {
|
||||
const files: string[] = [];
|
||||
for (const line of output.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed.endsWith(".node")) continue;
|
||||
if (!files.includes(trimmed)) files.push(trimmed);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
export interface CliOptions {
|
||||
targets: string[];
|
||||
dest: string | null;
|
||||
bazelArgs: string[];
|
||||
}
|
||||
|
||||
export function parseCliArgs(argv: string[]): CliOptions {
|
||||
const targets: string[] = [];
|
||||
let dest: string | null = null;
|
||||
const bazelArgs: string[] = [];
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i];
|
||||
if (arg === "--") {
|
||||
bazelArgs.push(...argv.slice(i + 1));
|
||||
break;
|
||||
}
|
||||
if (arg === "--dest") {
|
||||
const value = argv[++i];
|
||||
if (!value) throw new Error("--dest requires a directory argument");
|
||||
dest = value;
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("-")) {
|
||||
throw new Error(`Unknown flag ${arg} (extra bazel args go after \`--\`)`);
|
||||
}
|
||||
targets.push(arg);
|
||||
}
|
||||
if (targets.length === 0) {
|
||||
throw new Error("Usage: bun scripts/bazel-natives.ts <target>... [--dest <dir>] [-- <extra bazel args>]");
|
||||
}
|
||||
return { targets, dest, bazelArgs };
|
||||
}
|
||||
|
||||
function resolveBazelBinary(): string {
|
||||
const bin = Bun.which("bazelisk") ?? Bun.which("bazel");
|
||||
if (!bin) {
|
||||
throw new Error(
|
||||
"Neither `bazelisk` nor `bazel` found on PATH. Install bazelisk: https://github.com/bazelbuild/bazelisk",
|
||||
);
|
||||
}
|
||||
return bin;
|
||||
}
|
||||
|
||||
const STDERR_TAIL_LINES = 40;
|
||||
|
||||
/** Run a bazel command streaming stderr live while keeping a tail for the failure report. */
|
||||
async function runBazel(
|
||||
bin: string,
|
||||
args: string[],
|
||||
stdout: "inherit" | "pipe",
|
||||
): Promise<{ exitCode: number; stdout: string; stderrTail: string }> {
|
||||
const proc = Bun.spawn([bin, ...args], { cwd: repoRoot, stdout, stderr: "pipe" });
|
||||
const decoder = new TextDecoder();
|
||||
let tail = "";
|
||||
const pumpStderr = (async () => {
|
||||
for await (const chunk of proc.stderr) {
|
||||
const text = decoder.decode(chunk, { stream: true });
|
||||
process.stderr.write(text);
|
||||
tail = (tail + text)
|
||||
.split("\n")
|
||||
.slice(-STDERR_TAIL_LINES - 1)
|
||||
.join("\n");
|
||||
}
|
||||
})();
|
||||
const stdoutText = stdout === "pipe" ? await new Response(proc.stdout as ReadableStream).text() : "";
|
||||
const exitCode = await proc.exited;
|
||||
await pumpStderr;
|
||||
return { exitCode, stdout: stdoutText, stderrTail: tail };
|
||||
}
|
||||
|
||||
async function installAddon(sourcePath: string, destPath: string): Promise<void> {
|
||||
const realSource = await fs.realpath(sourcePath); // bazel-bin outputs are symlink-reachable; copy the real bytes
|
||||
const tempPath = `${destPath}.tmp.${process.pid}`;
|
||||
await fs.copyFile(realSource, tempPath);
|
||||
await fs.chmod(tempPath, 0o644);
|
||||
try {
|
||||
await fs.rename(tempPath, destPath); // atomic even if dest is a loaded addon
|
||||
} catch (err) {
|
||||
await fs.unlink(tempPath).catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const options = parseCliArgs(process.argv.slice(2));
|
||||
const host: HostInfo = { platform: process.platform, arch: process.arch, avx2: detectHostAvx2Support() };
|
||||
const labels = resolveTargetLabels(options.targets, host);
|
||||
const destDir = options.dest ? path.resolve(options.dest) : path.join(repoRoot, "packages/natives/native");
|
||||
const bazel = resolveBazelBinary();
|
||||
// CI hands cache wiring (remote or disk) through a bazelrc fragment so
|
||||
// endpoint composition stays in .github/actions/bazel-cache.
|
||||
const rcPath = Bun.env.OMP_BAZEL_RC?.trim();
|
||||
const startupArgs = rcPath ? [`--bazelrc=${rcPath}`] : [];
|
||||
|
||||
const buildArgs = [...startupArgs, "build", ...options.bazelArgs, "--", ...labels];
|
||||
console.log(`$ ${path.basename(bazel)} ${buildArgs.join(" ")}`);
|
||||
const build = await runBazel(bazel, buildArgs, "inherit");
|
||||
if (build.exitCode !== 0) {
|
||||
console.error(`\nbazel build failed (exit ${build.exitCode}). stderr tail:\n${build.stderrTail}`);
|
||||
process.exit(build.exitCode || 1);
|
||||
}
|
||||
|
||||
// Same flags as the build so cquery resolves the identical configuration.
|
||||
// cquery takes exactly one query expression, so multiple targets join
|
||||
// into a single union rather than positional args.
|
||||
const cquery = await runBazel(
|
||||
bazel,
|
||||
[...startupArgs, "cquery", ...options.bazelArgs, "--output=files", labels.join(" + ")],
|
||||
"pipe",
|
||||
);
|
||||
let outputs: string[];
|
||||
if (cquery.exitCode === 0) {
|
||||
outputs = parseBazelFilesOutput(cquery.stdout);
|
||||
} else {
|
||||
console.warn(`bazel cquery failed (exit ${cquery.exitCode}); falling back to bazel-bin path convention`);
|
||||
outputs = conventionOutputPaths(options.targets, host);
|
||||
}
|
||||
if (outputs.length === 0) {
|
||||
console.error("bazel build succeeded but no .node outputs were located");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const seen = new Map<string, string>();
|
||||
for (const output of outputs) {
|
||||
const base = path.basename(output);
|
||||
const prior = seen.get(base);
|
||||
if (prior) {
|
||||
// gnu and musl x64/arm64 addons share canonical basenames by design;
|
||||
// installing both into one dest would silently clobber.
|
||||
console.error(
|
||||
`refusing to install ${output}: ${base} already provided by ${prior}. ` +
|
||||
"Build gnu and musl targets in separate invocations with separate --dest dirs.",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
seen.set(base, output);
|
||||
}
|
||||
await fs.mkdir(destDir, { recursive: true });
|
||||
for (const output of outputs) {
|
||||
const absolute = path.isAbsolute(output) ? output : path.join(repoRoot, output);
|
||||
const destPath = path.join(destDir, path.basename(output));
|
||||
await installAddon(absolute, destPath);
|
||||
console.log(`installed ${path.basename(output)} → ${destPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
try {
|
||||
await main();
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : String(err));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import * as path from "node:path";
|
||||
import { $ } from "bun";
|
||||
|
||||
const repoRoot = path.join(import.meta.dir, "..");
|
||||
|
||||
async function runCiNativeDryRun(env: Record<string, string | undefined> = {}): Promise<string> {
|
||||
const result = await $`bun scripts/ci-build-native.ts --dry-run`
|
||||
.cwd(repoRoot)
|
||||
.quiet()
|
||||
.env({
|
||||
...process.env,
|
||||
PCRE2_SYS_STATIC: "0",
|
||||
RUSTFLAGS: "",
|
||||
TARGET_VARIANT: "",
|
||||
TARGET_VARIANTS: "",
|
||||
...env,
|
||||
})
|
||||
.nothrow();
|
||||
expect(result.exitCode).toBe(0);
|
||||
return result.text();
|
||||
}
|
||||
|
||||
describe("ci native build environment", () => {
|
||||
it("prints static PCRE2 env for the default native build dry run", async () => {
|
||||
await expect(runCiNativeDryRun()).resolves.toBe(
|
||||
"DRY RUN bun --cwd=packages/natives run build [default] PCRE2_SYS_STATIC=1\n",
|
||||
);
|
||||
});
|
||||
|
||||
it("prints static PCRE2 env without dropping x64 variant settings", async () => {
|
||||
await expect(runCiNativeDryRun({ TARGET_VARIANTS: "baseline" })).resolves.toBe(
|
||||
'DRY RUN bun --cwd=packages/natives run build [baseline] PCRE2_SYS_STATIC=1 TARGET_VARIANT=baseline RUSTFLAGS="-C target-cpu=x86-64-v2"\n',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,77 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import * as path from "node:path";
|
||||
import { $ } from "bun";
|
||||
|
||||
interface NativeBuildVariant {
|
||||
name: "baseline" | "modern";
|
||||
rustflags: string;
|
||||
}
|
||||
|
||||
const repoRoot = path.join(import.meta.dir, "..");
|
||||
const isDryRun = process.argv.includes("--dry-run");
|
||||
const variantConfigs: Record<NativeBuildVariant["name"], NativeBuildVariant> = {
|
||||
baseline: {
|
||||
name: "baseline",
|
||||
rustflags: "-C target-cpu=x86-64-v2",
|
||||
},
|
||||
modern: {
|
||||
name: "modern",
|
||||
rustflags: "-C target-cpu=x86-64-v3",
|
||||
},
|
||||
};
|
||||
|
||||
/** Adds release-portability env required by native addon builds. */
|
||||
export function withPortableNativeBuildEnv(
|
||||
env: Record<string, string | undefined>,
|
||||
): Record<string, string | undefined> {
|
||||
return { ...env, PCRE2_SYS_STATIC: "1" };
|
||||
}
|
||||
|
||||
function parseTargetVariants(): NativeBuildVariant[] {
|
||||
const rawVariants = (Bun.env.TARGET_VARIANTS ?? "").trim();
|
||||
if (!rawVariants) return [];
|
||||
|
||||
return rawVariants.split(/\s+/).map(rawVariant => {
|
||||
const variant = variantConfigs[rawVariant as keyof typeof variantConfigs];
|
||||
if (!variant) {
|
||||
throw new Error(`Unsupported TARGET_VARIANTS entry: ${rawVariant}. Expected baseline or modern.`);
|
||||
}
|
||||
return variant;
|
||||
});
|
||||
}
|
||||
|
||||
async function runNativeBuild(env: Record<string, string | undefined>, label: string): Promise<void> {
|
||||
const buildEnv = withPortableNativeBuildEnv(env);
|
||||
if (isDryRun) {
|
||||
const staticPcre = ` PCRE2_SYS_STATIC=${buildEnv.PCRE2_SYS_STATIC}`;
|
||||
const variant = buildEnv.TARGET_VARIANT ? ` TARGET_VARIANT=${buildEnv.TARGET_VARIANT}` : "";
|
||||
const rustflags = buildEnv.RUSTFLAGS ? ` RUSTFLAGS=${JSON.stringify(buildEnv.RUSTFLAGS)}` : "";
|
||||
console.log(`DRY RUN bun --cwd=packages/natives run build [${label}]${staticPcre}${variant}${rustflags}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Building natives [${label}]...`);
|
||||
await $`bun --cwd=packages/natives run build`.cwd(repoRoot).env(buildEnv);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const variants = parseTargetVariants();
|
||||
if (variants.length === 0) {
|
||||
await runNativeBuild(Bun.env, "default");
|
||||
return;
|
||||
}
|
||||
|
||||
for (const variant of variants) {
|
||||
await runNativeBuild(
|
||||
{
|
||||
...Bun.env,
|
||||
RUSTFLAGS: variant.rustflags,
|
||||
TARGET_VARIANT: variant.name,
|
||||
},
|
||||
variant.name,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) await main();
|
||||
@@ -1,83 +0,0 @@
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
|
||||
const script = path.join(import.meta.dir, "ci-native-artifact-cache.ts");
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
async function tempDir(): Promise<string> {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "omp-native-cache-test-"));
|
||||
tempRoots.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
async function run(args: string[], cacheDir: string, outputPath?: string): Promise<string> {
|
||||
const proc = Bun.spawn([process.execPath, script, ...args], {
|
||||
cwd: path.join(import.meta.dir, ".."),
|
||||
env: {
|
||||
...process.env,
|
||||
OMP_NATIVE_CACHE_DIR: cacheDir,
|
||||
GITHUB_OUTPUT: outputPath,
|
||||
},
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
const [stdout, stderr, exitCode] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
proc.exited,
|
||||
]);
|
||||
if (exitCode !== 0) throw new Error(`cache command failed (${exitCode}): ${stderr}`);
|
||||
return stdout;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempRoots.splice(0).map(dir => fs.rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
describe("CI native artifact cache", () => {
|
||||
it("restores a complete artifact set without unrelated build output", async () => {
|
||||
const root = await tempDir();
|
||||
const source = path.join(root, "source");
|
||||
const destination = path.join(root, "destination");
|
||||
const output = path.join(root, "github-output");
|
||||
await fs.mkdir(source);
|
||||
await Promise.all([
|
||||
Bun.write(path.join(source, "pi_natives.linux-x64.node"), "baseline"),
|
||||
Bun.write(path.join(source, "build.log"), "not an artifact"),
|
||||
]);
|
||||
|
||||
await run(["save", "abcdef12", "pi-natives-linux-x64-baseline-habcdef12", source], root);
|
||||
await run(["restore", "abcdef12", destination, "pi-natives-linux-x64-baseline-habcdef12"], root, output);
|
||||
|
||||
expect(await Bun.file(output).text()).toBe("hit=true\n");
|
||||
expect(await Bun.file(path.join(destination, "pi_natives.linux-x64.node")).text()).toBe("baseline");
|
||||
expect(await Bun.file(path.join(destination, "build.log")).exists()).toBe(false);
|
||||
});
|
||||
|
||||
it("reports a miss and copies nothing unless every requested artifact is complete", async () => {
|
||||
const root = await tempDir();
|
||||
const source = path.join(root, "source");
|
||||
const destination = path.join(root, "destination");
|
||||
const output = path.join(root, "github-output");
|
||||
await fs.mkdir(source);
|
||||
await Bun.write(path.join(source, "pi_natives.linux-x64.node"), "baseline");
|
||||
await run(["save", "abcdef12", "pi-natives-linux-x64-baseline-habcdef12", source], root);
|
||||
|
||||
await run(
|
||||
[
|
||||
"restore",
|
||||
"abcdef12",
|
||||
destination,
|
||||
"pi-natives-linux-x64-baseline-habcdef12",
|
||||
"pi-natives-linux-x64-modern-habcdef12",
|
||||
],
|
||||
root,
|
||||
output,
|
||||
);
|
||||
|
||||
expect(await Bun.file(output).text()).toBe("hit=false\n");
|
||||
expect(await Bun.file(destination).exists()).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,129 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as path from "node:path";
|
||||
|
||||
const CACHE_ENV = "OMP_NATIVE_CACHE_DIR";
|
||||
const COMPLETE_FILE = ".complete";
|
||||
|
||||
function validateSegment(value: string, label: string): void {
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value)) {
|
||||
throw new Error(`Invalid ${label} ${JSON.stringify(value)}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeOutput(name: string, value: string): Promise<void> {
|
||||
const outputPath = Bun.env.GITHUB_OUTPUT;
|
||||
if (outputPath) {
|
||||
await fs.appendFile(outputPath, `${name}=${value}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
async function completedFiles(artifactDir: string): Promise<string[] | null> {
|
||||
try {
|
||||
const text = await Bun.file(path.join(artifactDir, COMPLETE_FILE)).text();
|
||||
const files = text.split("\n").filter(Boolean);
|
||||
if (files.length === 0 || files.some(file => path.basename(file) !== file || !file.endsWith(".node"))) {
|
||||
return null;
|
||||
}
|
||||
for (const file of files) {
|
||||
if (!(await Bun.file(path.join(artifactDir, file)).exists())) return null;
|
||||
}
|
||||
return files;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function save(cacheRoot: string, hash: string, artifactName: string, sourceDir: string): Promise<void> {
|
||||
validateSegment(hash, "source hash");
|
||||
validateSegment(artifactName, "artifact name");
|
||||
const entries = await fs.readdir(sourceDir, { withFileTypes: true });
|
||||
const files = entries
|
||||
.filter(entry => entry.isFile() && entry.name.endsWith(".node"))
|
||||
.map(entry => entry.name)
|
||||
.sort();
|
||||
if (files.length === 0) throw new Error(`No native addons found in ${sourceDir}`);
|
||||
|
||||
const hashDir = path.join(cacheRoot, hash);
|
||||
const artifactDir = path.join(hashDir, artifactName);
|
||||
if (await completedFiles(artifactDir)) {
|
||||
console.log(`Native artifact cache already populated: ${artifactName}`);
|
||||
return;
|
||||
}
|
||||
|
||||
await fs.mkdir(hashDir, { recursive: true });
|
||||
const stagingDir = path.join(hashDir, `${artifactName}.tmp-${process.pid}-${crypto.randomUUID()}`);
|
||||
await fs.mkdir(stagingDir);
|
||||
try {
|
||||
for (const file of files) {
|
||||
await fs.copyFile(path.join(sourceDir, file), path.join(stagingDir, file));
|
||||
}
|
||||
await Bun.write(path.join(stagingDir, COMPLETE_FILE), `${files.join("\n")}\n`);
|
||||
try {
|
||||
await fs.rename(stagingDir, artifactDir);
|
||||
} catch (error) {
|
||||
if (!(await completedFiles(artifactDir))) throw error;
|
||||
await fs.rm(stagingDir, { recursive: true, force: true });
|
||||
}
|
||||
} catch (error) {
|
||||
await fs.rm(stagingDir, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
console.log(`Saved native artifact cache: ${artifactName} (${files.join(", ")})`);
|
||||
}
|
||||
|
||||
async function restore(
|
||||
cacheRoot: string,
|
||||
hash: string,
|
||||
destination: string,
|
||||
artifactNames: string[],
|
||||
): Promise<boolean> {
|
||||
validateSegment(hash, "source hash");
|
||||
if (artifactNames.length === 0) throw new Error("At least one artifact name is required");
|
||||
|
||||
const sources: Array<{ dir: string; files: string[] }> = [];
|
||||
for (const artifactName of artifactNames) {
|
||||
validateSegment(artifactName, "artifact name");
|
||||
const dir = path.join(cacheRoot, hash, artifactName);
|
||||
const files = await completedFiles(dir);
|
||||
if (!files) return false;
|
||||
sources.push({ dir, files });
|
||||
}
|
||||
|
||||
await fs.mkdir(destination, { recursive: true });
|
||||
for (const source of sources) {
|
||||
for (const file of source.files) {
|
||||
await fs.copyFile(path.join(source.dir, file), path.join(destination, file));
|
||||
}
|
||||
}
|
||||
console.log(`Restored native artifacts from local cache: ${artifactNames.join(", ")}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const [mode, hash, first, ...rest] = process.argv.slice(2);
|
||||
if ((mode !== "save" && mode !== "restore") || !hash || !first) {
|
||||
throw new Error(
|
||||
"Usage: ci-native-artifact-cache.ts save <hash> <artifact-name> [source-dir] | restore <hash> <destination> <artifact-name>...",
|
||||
);
|
||||
}
|
||||
|
||||
const cacheRoot = Bun.env[CACHE_ENV]?.trim();
|
||||
if (!cacheRoot) {
|
||||
if (mode === "restore") await writeOutput("hit", "false");
|
||||
console.log(`Native artifact cache disabled: ${CACHE_ENV} is unset`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (mode === "save") {
|
||||
await save(cacheRoot, hash, first, rest[0] ?? "packages/natives/native");
|
||||
return;
|
||||
}
|
||||
|
||||
const hit = await restore(cacheRoot, hash, first, rest);
|
||||
await writeOutput("hit", String(hit));
|
||||
if (!hit) console.log(`Native artifact cache miss: ${rest.join(", ")}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) await main();
|
||||
@@ -1,40 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { objectKeyFor, resolveEndpoint } from "./ci-target-cache";
|
||||
|
||||
describe("resolveEndpoint", () => {
|
||||
test("selects scheme from SCCACHE_S3_USE_SSL, defaulting to http for in-cluster RustFS", () => {
|
||||
expect(resolveEndpoint({ SCCACHE_ENDPOINT: "rustfs.sccache.svc.cluster.local:9000" })).toBe(
|
||||
"http://rustfs.sccache.svc.cluster.local:9000",
|
||||
);
|
||||
expect(resolveEndpoint({ SCCACHE_ENDPOINT: "rustfs:9000", SCCACHE_S3_USE_SSL: "true" })).toBe(
|
||||
"https://rustfs:9000",
|
||||
);
|
||||
expect(resolveEndpoint({ SCCACHE_ENDPOINT: "rustfs:9000", SCCACHE_S3_USE_SSL: "false" })).toBe(
|
||||
"http://rustfs:9000",
|
||||
);
|
||||
});
|
||||
|
||||
test("passes through endpoints that already carry a scheme and rejects missing config", () => {
|
||||
expect(resolveEndpoint({ SCCACHE_ENDPOINT: "https://s3.example.com" })).toBe("https://s3.example.com");
|
||||
expect(resolveEndpoint({})).toBeNull();
|
||||
expect(resolveEndpoint({ SCCACHE_ENDPOINT: " " })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("objectKeyFor", () => {
|
||||
test("namespaces snapshots under target-cache/ and separates toolchains", () => {
|
||||
const stable = objectKeyFor("native-linux-default-x64-baseline", "rustc 1.91.0-nightly (abc 2026-04-29)");
|
||||
expect(stable).toMatch(/^target-cache\/native-linux-default-x64-baseline-[0-9a-f]{12}\.tar\.zst$/);
|
||||
// Same inputs must be deterministic; a toolchain bump must be a clean miss.
|
||||
expect(objectKeyFor("native-linux-default-x64-baseline", "rustc 1.91.0-nightly (abc 2026-04-29)")).toBe(stable);
|
||||
expect(objectKeyFor("native-linux-default-x64-baseline", "rustc 1.92.0-nightly (def 2026-06-01)")).not.toBe(
|
||||
stable,
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects keys that could escape the target-cache/ prefix", () => {
|
||||
expect(() => objectKeyFor("../sccache-poison", "rustc 1.91.0")).toThrow(/Invalid cache key/);
|
||||
expect(() => objectKeyFor("a/b", "rustc 1.91.0")).toThrow(/Invalid cache key/);
|
||||
expect(() => objectKeyFor("", "rustc 1.91.0")).toThrow(/Invalid cache key/);
|
||||
});
|
||||
});
|
||||
@@ -1,214 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Persist the cargo `target/` directory to the in-cluster RustFS S3 bucket
|
||||
* between omp-kata CI runs.
|
||||
*
|
||||
* sccache only caches rustc invocations; build-script outputs (57 tree-sitter
|
||||
* grammar C compiles, audiopus_sys' bundled opus via CMake, ring's asm) and
|
||||
* cargo's fingerprint/link work bypass it entirely. Restoring `target/` reuses
|
||||
* all of that, so a warm native job only recompiles workspace crates.
|
||||
*
|
||||
* Storage model: one object per cache key (`target-cache/<key>-<toolchain>.tar.zst`),
|
||||
* overwritten on every save — storage stays bounded at one snapshot per
|
||||
* platform/libc/arch/variant/toolchain. Staleness is safe: cargo fingerprints
|
||||
* invalidate anything that no longer matches, exactly like Swatinem/rust-cache
|
||||
* on the GitHub-hosted runners.
|
||||
*
|
||||
* Credentials/config come from the pod-wide sccache env (`SCCACHE_BUCKET`,
|
||||
* `SCCACHE_ENDPOINT`, `SCCACHE_S3_USE_SSL`, `AWS_*`); Bun's S3Client reads the
|
||||
* AWS credentials from the environment. Off-infra (no `SCCACHE_BUCKET`) and
|
||||
* every failure path degrade to a logged no-op — this script must never fail
|
||||
* a CI job.
|
||||
*
|
||||
* Usage: `bun scripts/ci-target-cache.ts <restore|save> <cache-key>`
|
||||
*/
|
||||
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { $, S3Client } from "bun";
|
||||
|
||||
const repoRoot = path.join(import.meta.dir, "..");
|
||||
|
||||
/** Compressed snapshots above this size are not uploaded; the next full miss rebuilds a compact one. */
|
||||
const MAX_SNAPSHOT_BYTES = 4 * 1024 ** 3;
|
||||
const EXISTS_TIMEOUT_MS = 30_000;
|
||||
const DOWNLOAD_TIMEOUT_MS = 180_000;
|
||||
const UPLOAD_TIMEOUT_MS = 300_000;
|
||||
const configuredCompressionThreads = Number(Bun.env.OMP_CI_CPU_COUNT);
|
||||
const COMPRESSION_THREADS =
|
||||
Number.isInteger(configuredCompressionThreads) && configuredCompressionThreads > 0
|
||||
? configuredCompressionThreads
|
||||
: 2;
|
||||
|
||||
/**
|
||||
* Resolve the S3 endpoint URL from the sccache pod env. `SCCACHE_ENDPOINT` is
|
||||
* host:port without a scheme; `SCCACHE_S3_USE_SSL=true` selects https,
|
||||
* anything else http (in-cluster RustFS serves plain HTTP). A value that
|
||||
* already carries a scheme is passed through untouched.
|
||||
*/
|
||||
export function resolveEndpoint(env: Record<string, string | undefined>): string | null {
|
||||
const endpoint = env.SCCACHE_ENDPOINT?.trim();
|
||||
if (!endpoint) return null;
|
||||
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(endpoint)) return endpoint;
|
||||
const scheme = env.SCCACHE_S3_USE_SSL === "true" ? "https" : "http";
|
||||
return `${scheme}://${endpoint}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Object key for one snapshot. The toolchain fingerprint (`rustc -V`) is
|
||||
* hashed in so a nightly bump becomes a clean miss instead of a useless
|
||||
* multi-GB restore that cargo immediately invalidates.
|
||||
*/
|
||||
export function objectKeyFor(cacheKey: string, rustcVersion: string): string {
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(cacheKey)) {
|
||||
throw new Error(`Invalid cache key ${JSON.stringify(cacheKey)}; expected [A-Za-z0-9._-]+`);
|
||||
}
|
||||
const toolchain = new Bun.CryptoHasher("sha256").update(rustcVersion).digest("hex").slice(0, 12);
|
||||
return `target-cache/${cacheKey}-${toolchain}.tar.zst`;
|
||||
}
|
||||
|
||||
function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
|
||||
const { promise: timeout, reject } = Promise.withResolvers<never>();
|
||||
const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
||||
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
||||
}
|
||||
|
||||
/** `target_directory` from cargo metadata (honors CARGO_TARGET_DIR/config), falling back to `<repo>/target`. */
|
||||
async function resolveTargetDir(): Promise<string> {
|
||||
const meta = await $`cargo metadata --no-deps --format-version 1`.cwd(repoRoot).quiet().nothrow();
|
||||
if (meta.exitCode === 0) {
|
||||
try {
|
||||
const parsed = meta.json() as { target_directory?: string };
|
||||
if (parsed.target_directory) return parsed.target_directory;
|
||||
} catch {
|
||||
// fall through to default
|
||||
}
|
||||
}
|
||||
return path.join(repoRoot, "target");
|
||||
}
|
||||
|
||||
async function restore(s3: S3Client, objectKey: string, targetDir: string): Promise<void> {
|
||||
const object = s3.file(objectKey);
|
||||
if (!(await withTimeout(object.exists(), EXISTS_TIMEOUT_MS, "cache lookup"))) {
|
||||
console.log(`target cache miss: ${objectKey}`);
|
||||
return;
|
||||
}
|
||||
const tmpTar = path.join(Bun.env.RUNNER_TEMP ?? os.tmpdir(), `target-cache-${process.pid}.tar.zst`);
|
||||
const started = Bun.nanoseconds();
|
||||
try {
|
||||
// NB: `Bun.write(dest, new Response(s3file.stream()))` never resolves in
|
||||
// Bun 1.3.x; iterating the stream into a FileSink works.
|
||||
const download = async () => {
|
||||
const sink = Bun.file(tmpTar).writer();
|
||||
for await (const chunk of object.stream()) sink.write(chunk);
|
||||
await sink.end();
|
||||
};
|
||||
await withTimeout(download(), DOWNLOAD_TIMEOUT_MS, "cache download");
|
||||
const sizeMb = (Bun.file(tmpTar).size / 1024 ** 2).toFixed(0);
|
||||
// Explicit decompress pipe: GNU tar passes -d to a --use-compress-program
|
||||
// filter on extract but bsdtar does not, so filter flags are a trap.
|
||||
// The pipe reports only tar's exit code, which reads a truncated zstd
|
||||
// stream as a short-but-valid archive — so verify the zstd layer first.
|
||||
const verify = await $`zstd -tq ${tmpTar}`.quiet().nothrow();
|
||||
const extract =
|
||||
verify.exitCode === 0
|
||||
? await $`zstd -dcq ${tmpTar} | tar -xf - -C ${path.dirname(targetDir)}`.quiet().nothrow()
|
||||
: verify;
|
||||
if (extract.exitCode !== 0) {
|
||||
// A torn/corrupt snapshot must not leave a half-extracted target/
|
||||
// behind: cargo would trust whatever fingerprints survived.
|
||||
await fs.rm(targetDir, { recursive: true, force: true });
|
||||
console.warn(
|
||||
`target cache extract failed (exit ${extract.exitCode}); removed ${targetDir} and continuing cold`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const secs = ((Bun.nanoseconds() - started) / 1e9).toFixed(1);
|
||||
console.log(`target cache restored: ${objectKey} (${sizeMb} MiB in ${secs}s)`);
|
||||
} finally {
|
||||
await fs.rm(tmpTar, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function save(s3: S3Client, objectKey: string, targetDir: string): Promise<void> {
|
||||
try {
|
||||
await fs.stat(targetDir);
|
||||
} catch {
|
||||
console.log(`target cache save skipped: ${targetDir} does not exist`);
|
||||
return;
|
||||
}
|
||||
const tmpTar = path.join(Bun.env.RUNNER_TEMP ?? os.tmpdir(), `target-cache-${process.pid}.tar.zst`);
|
||||
const started = Bun.nanoseconds();
|
||||
try {
|
||||
// CARGO_INCREMENTAL=0 in CI, so incremental/ only exists from stray
|
||||
// local state; exclude it regardless — it is the one cargo dir that is
|
||||
// pure dead weight for a cold consumer. Explicit compress pipe for the
|
||||
// same tar-flavor reason as in restore(). Compression is capped to the
|
||||
// runner's admitted CPU allocation; `-T0` multiplied host contention when
|
||||
// several native matrix jobs saved snapshots together.
|
||||
const create =
|
||||
await $`tar -cf - --exclude=${"*/incremental"} -C ${path.dirname(targetDir)} ${path.basename(targetDir)} | zstd -q -T${COMPRESSION_THREADS} -3 -f -o ${tmpTar}`
|
||||
.quiet()
|
||||
.nothrow();
|
||||
if (create.exitCode !== 0) {
|
||||
console.warn(`target cache save skipped: tar failed (exit ${create.exitCode})`);
|
||||
return;
|
||||
}
|
||||
const size = Bun.file(tmpTar).size;
|
||||
if (size > MAX_SNAPSHOT_BYTES) {
|
||||
// Orphaned artifacts accumulate across restore→build→save cycles;
|
||||
// refusing oversized uploads bounds the object. The stale snapshot
|
||||
// keeps serving restores until a full-miss rebuild saves a compact one.
|
||||
console.warn(
|
||||
`target cache save skipped: snapshot ${(size / 1024 ** 3).toFixed(1)} GiB exceeds ${MAX_SNAPSHOT_BYTES / 1024 ** 3} GiB cap`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
await withTimeout(s3.write(objectKey, Bun.file(tmpTar)), UPLOAD_TIMEOUT_MS, "cache upload");
|
||||
const secs = ((Bun.nanoseconds() - started) / 1e9).toFixed(1);
|
||||
console.log(`target cache saved: ${objectKey} (${(size / 1024 ** 2).toFixed(0)} MiB in ${secs}s)`);
|
||||
} finally {
|
||||
await fs.rm(tmpTar, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const [mode, cacheKey] = [process.argv[2], process.argv[3]];
|
||||
if ((mode !== "restore" && mode !== "save") || !cacheKey) {
|
||||
console.error("Usage: bun scripts/ci-target-cache.ts <restore|save> <cache-key>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const bucket = Bun.env.SCCACHE_BUCKET;
|
||||
const endpoint = resolveEndpoint(Bun.env);
|
||||
if (!bucket || !endpoint) {
|
||||
console.log("target cache skipped: no SCCACHE_BUCKET/SCCACHE_ENDPOINT in env (not on omp-kata infra)");
|
||||
return;
|
||||
}
|
||||
if (!Bun.which("zstd")) {
|
||||
console.warn("target cache skipped: zstd not on PATH");
|
||||
return;
|
||||
}
|
||||
const rustc = await $`rustc -V`.quiet().nothrow();
|
||||
if (rustc.exitCode !== 0) {
|
||||
console.warn("target cache skipped: rustc not on PATH");
|
||||
return;
|
||||
}
|
||||
|
||||
const objectKey = objectKeyFor(cacheKey, rustc.text().trim());
|
||||
const s3 = new S3Client({ bucket, endpoint, region: Bun.env.SCCACHE_REGION ?? Bun.env.AWS_REGION ?? "us-east-1" });
|
||||
const targetDir = await resolveTargetDir();
|
||||
if (mode === "restore") await restore(s3, objectKey, targetDir);
|
||||
else await save(s3, objectKey, targetDir);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
try {
|
||||
await main();
|
||||
} catch (err) {
|
||||
// Cache trouble must never fail a build; cold compile is the fallback.
|
||||
console.warn(`target cache ${process.argv[2] ?? ""} failed non-fatally:`, err);
|
||||
}
|
||||
}
|
||||
@@ -119,8 +119,7 @@ const localOnlyWorkspacePackages = ["packages/mnemopi", "python/robomp/web"];
|
||||
// silently ignores unmatched filters when at least one other filter matches.)
|
||||
const repoScriptTests = [
|
||||
"scripts/ci-concurrency.test.ts",
|
||||
"scripts/ci-build-native.test.ts",
|
||||
"scripts/ci-native-artifact-cache.test.ts",
|
||||
"scripts/bazel-natives.test.ts",
|
||||
"scripts/ci-release-notes.test.ts",
|
||||
"scripts/ci-release-publish.test.ts",
|
||||
"scripts/fix-dts-extensions.test.ts",
|
||||
@@ -356,7 +355,7 @@ async function commandsForMode(mode: Mode): Promise<TestCommand[]> {
|
||||
"--parallel=4",
|
||||
...onlyFailuresArgs,
|
||||
"scripts/ci-concurrency.test.ts",
|
||||
"scripts/ci-build-native.test.ts",
|
||||
"scripts/bazel-natives.test.ts",
|
||||
"scripts/ci-release-publish.test.ts",
|
||||
"scripts/fix-dts-extensions.test.ts",
|
||||
],
|
||||
@@ -410,16 +409,15 @@ async function commandsForMode(mode: Mode): Promise<TestCommand[]> {
|
||||
}
|
||||
}
|
||||
|
||||
// The omp-kata runner pods inject sccache S3 credentials (`AWS_*`) and config
|
||||
// (`SCCACHE_*`) pod-wide via `envFrom`, GitHub Actions injects `GITHUB_TOKEN`,
|
||||
// The omp-kata runner pods may inject cloud credentials (`AWS_*`) pod-wide via
|
||||
// `envFrom`, GitHub Actions injects `GITHUB_TOKEN`,
|
||||
// and a host may carry provider API keys. Any of these make env-sensitive code
|
||||
// non-deterministic in tests — e.g. leaked AWS creds make `amazon-bedrock` look
|
||||
// authenticated and win the provider startup fallback over `anthropic`. Run the
|
||||
// suites in a hermetic environment with all credential / cloud-config variables
|
||||
// stripped so resolution depends only on the test's own fixtures.
|
||||
const SCRUBBED_ENV_PREFIXES = ["AWS_", "SCCACHE_", "GOOGLE_CLOUD_"];
|
||||
const SCRUBBED_ENV_PREFIXES = ["AWS_", "GOOGLE_CLOUD_"];
|
||||
const SCRUBBED_ENV_NAMES = new Set([
|
||||
"RUSTC_WRAPPER",
|
||||
"GITHUB_TOKEN",
|
||||
"GH_TOKEN",
|
||||
"COPILOT_GITHUB_TOKEN",
|
||||
|
||||
+33
-4
@@ -14,18 +14,47 @@ const RUST_AFFECTING_FILE_NAMES = [
|
||||
"rustfmt.toml",
|
||||
".rustfmt.toml",
|
||||
] as const satisfies readonly string[];
|
||||
// brush-core/brush-builtins became workspace members for Bazel hermeticity
|
||||
// (path-patch rendering is machine-local), but the cargo dev tasks keep their
|
||||
// historical scope: the vendored fork is not held to workspace lint/test gates.
|
||||
const VENDORED_FORK_EXCLUDES = [
|
||||
"--exclude",
|
||||
"brush-core",
|
||||
"--exclude",
|
||||
"brush-builtins",
|
||||
] as const satisfies readonly string[];
|
||||
const TASK_COMMANDS = {
|
||||
"check:rs": [
|
||||
["cargo", "fmt", "--all", "--", "--check"],
|
||||
["cargo", "clippy", "--workspace", "--", "-D", "warnings"],
|
||||
["cargo", "clippy", "--workspace", ...VENDORED_FORK_EXCLUDES, "--", "-D", "warnings"],
|
||||
],
|
||||
"fix:rs": [
|
||||
["cargo", "fmt", "--all"],
|
||||
["cargo", "clippy", "--workspace", "--fix", "--allow-dirty", "--no-deps", "--allow-staged", "--allow-no-vcs"],
|
||||
[
|
||||
"cargo",
|
||||
"clippy",
|
||||
"--workspace",
|
||||
...VENDORED_FORK_EXCLUDES,
|
||||
"--fix",
|
||||
"--allow-dirty",
|
||||
"--no-deps",
|
||||
"--allow-staged",
|
||||
"--allow-no-vcs",
|
||||
],
|
||||
],
|
||||
"fmt:rs": [["cargo", "fmt", "--all"]],
|
||||
"lint:rs": [["cargo", "clippy", "--workspace", "--", "-D", "warnings"]],
|
||||
"test:rs": [["cargo", "nextest", "run", "--workspace", "--status-level=fail", "--final-status-level=fail"]],
|
||||
"lint:rs": [["cargo", "clippy", "--workspace", ...VENDORED_FORK_EXCLUDES, "--", "-D", "warnings"]],
|
||||
"test:rs": [
|
||||
[
|
||||
"cargo",
|
||||
"nextest",
|
||||
"run",
|
||||
"--workspace",
|
||||
...VENDORED_FORK_EXCLUDES,
|
||||
"--status-level=fail",
|
||||
"--final-status-level=fail",
|
||||
],
|
||||
],
|
||||
} as const satisfies Record<string, readonly (readonly string[])[]>;
|
||||
|
||||
type RustTaskName = keyof typeof TASK_COMMANDS;
|
||||
|
||||
Reference in New Issue
Block a user