smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: streamlined native addon builds and caching in ci workflows

Browse Source 
- Enhance CI workflows to build and reuse native addon artifacts instead of recompiling.
- Update bazel cache actions with selective backend detection and separate remote and disk modes.
- Add `--source` CLI option to install prebuilt native targets without requiring Bazel.
- Increase Kata runner memory configuration from 12Gi to 24Gi.
This commit is contained in:
can1357
2026-07-28 02:06:13 +02:00
parent bbe0236a0f
commit ed4c78bc0f
9 changed files with 306 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bazel-natives.test.ts
+44 -3
View File
@@ -1,4 +1,8 @@
import { describe, expect, test } from "bun:test";
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { $ } from "bun";
import {
conventionOutputPaths,
type HostInfo,
@@ -95,21 +99,58 @@ describe("parseBazelFilesOutput", () => {
});
describe("parseCliArgs", () => {
test("splits targets, --dest, and passthrough bazel args", () => {
test("splits targets, paths, and passthrough bazel args", () => {
expect(
parseCliArgs(["linux-x64-baseline", "linux-x64-modern", "--dest", "out", "--", "--config=ci", "--dest"]),
).toEqual({
targets: ["linux-x64-baseline", "linux-x64-modern"],
dest: "out",
source: null,
bazelArgs: ["--config=ci", "--dest"],
});
expect(parseCliArgs(["host"])).toEqual({ targets: ["host"], dest: null, bazelArgs: [] });
expect(parseCliArgs(["host", "--source", "artifact"])).toEqual({
targets: ["host"],
dest: null,
source: "artifact",
bazelArgs: [],
});
});
test("rejects missing targets, stray flags, and a valueless --dest", () => {
test("rejects invalid build and artifact source combinations", () => {
expect(() => parseCliArgs([])).toThrow(/Usage:/);
expect(() => parseCliArgs(["--", "--config=ci"])).toThrow(/Usage:/);
expect(() => parseCliArgs(["host", "--config=ci"])).toThrow(/Unknown flag --config=ci/);
expect(() => parseCliArgs(["host", "--dest"])).toThrow(/--dest requires/);
expect(() => parseCliArgs(["host", "--source"])).toThrow(/--source requires/);
expect(() => parseCliArgs(["host", "--source", "artifact", "--", "--config=ci"])).toThrow(
/--source cannot be combined/,
);
});
});
describe("artifact source install", () => {
test("installs exact target outputs without invoking Bazel", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "omp-native-artifacts-"));
const source = path.join(root, "source");
const dest = path.join(root, "dest");
const baseline = "pi_natives.linux-x64-baseline.node";
const modern = "pi_natives.linux-x64-modern.node";
try {
await fs.mkdir(path.join(source, "natives-linux-x64-baseline"), { recursive: true });
await fs.mkdir(path.join(source, "natives-linux-x64-modern"), { recursive: true });
await Bun.write(path.join(source, "natives-linux-x64-baseline", baseline), "baseline");
await Bun.write(path.join(source, "natives-linux-x64-modern", modern), "modern");
const result =
await $`${process.execPath} ${path.join(import.meta.dir, "bazel-natives.ts")} linux-x64-baseline linux-x64-modern --source ${source} --dest ${dest}`
.quiet()
.nothrow();
expect(result.exitCode).toBe(0);
expect(await Bun.file(path.join(dest, baseline)).text()).toBe("baseline");
expect(await Bun.file(path.join(dest, modern)).text()).toBe("modern");
} finally {
await fs.rm(root, { recursive: true, force: true });
}
});
});
scripts/bazel-natives.ts
+58 -37
View File
@@ -2,7 +2,7 @@
/**
* Canonical Bazel driver for the shipping pi_natives addons.
*
* Usage: bun scripts/bazel-natives.ts <target>... [--dest <dir>] [-- <extra bazel args>]
* Usage: bun scripts/bazel-natives.ts <target>... [--dest <dir>] [--source <dir>] [-- <extra bazel args>]
*
* Targets are the //:natives-* names from BUILD.bazel (e.g. linux-x64-baseline,
* darwin-arm64) plus three pseudo-targets:
@@ -118,15 +118,19 @@ export function parseBazelFilesOutput(output: string): string[] {
return files;
}
/** Parsed options for the native addon build and artifact install modes. */
export interface CliOptions {
targets: string[];
dest: string | null;
source: string | null;
bazelArgs: string[];
}
/** Parse target names and the mutually exclusive build or artifact source options. */
export function parseCliArgs(argv: string[]): CliOptions {
const targets: string[] = [];
let dest: string | null = null;
let source: string | null = null;
const bazelArgs: string[] = [];
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
@@ -134,10 +138,14 @@ export function parseCliArgs(argv: string[]): CliOptions {
bazelArgs.push(...argv.slice(i + 1));
break;
}
if (arg === "--dest") {
if (arg === "--dest" || arg === "--source") {
const value = argv[++i];
if (!value) throw new Error("--dest requires a directory argument");
dest = value;
if (!value) throw new Error(`${arg} requires a directory argument`);
if (arg === "--dest") {
dest = value;
} else {
source = value;
}
continue;
}
if (arg.startsWith("-")) {
@@ -146,9 +154,14 @@ export function parseCliArgs(argv: string[]): CliOptions {
targets.push(arg);
}
if (targets.length === 0) {
throw new Error("Usage: bun scripts/bazel-natives.ts <target>... [--dest <dir>] [-- <extra bazel args>]");
throw new Error(
"Usage: bun scripts/bazel-natives.ts <target>... [--dest <dir>] [--source <dir>] [-- <extra bazel args>]",
);
}
return { targets, dest, bazelArgs };
if (source && bazelArgs.length > 0) {
throw new Error("--source cannot be combined with extra bazel arguments");
}
return { targets, dest, source, bazelArgs };
}
function resolveBazelBinary(): string {
@@ -204,40 +217,48 @@ async function installAddon(sourcePath: string, destPath: string): Promise<void>
async function main(): Promise<void> {
const options = parseCliArgs(process.argv.slice(2));
const host: HostInfo = { platform: process.platform, arch: process.arch, avx2: detectHostAvx2Support() };
const labels = resolveTargetLabels(options.targets, host);
const destDir = options.dest ? path.resolve(options.dest) : path.join(repoRoot, "packages/natives/native");
const bazel = resolveBazelBinary();
// CI hands cache wiring (remote or disk) through a bazelrc fragment so
// endpoint composition stays in .github/actions/bazel-cache.
const rcPath = Bun.env.OMP_BAZEL_RC?.trim();
const startupArgs = rcPath ? [`--bazelrc=${rcPath}`] : [];
const buildArgs = [...startupArgs, "build", ...options.bazelArgs, "--", ...labels];
console.log(`$ ${path.basename(bazel)} ${buildArgs.join(" ")}`);
const build = await runBazel(bazel, buildArgs, "inherit");
if (build.exitCode !== 0) {
console.error(`\nbazel build failed (exit ${build.exitCode}). stderr tail:\n${build.stderrTail}`);
process.exit(build.exitCode || 1);
}
// Same flags as the build so cquery resolves the identical configuration.
// cquery takes exactly one query expression, so multiple targets join
// into a single union rather than positional args.
const cquery = await runBazel(
bazel,
[...startupArgs, "cquery", ...options.bazelArgs, "--output=files", labels.join(" + ")],
"pipe",
);
let outputs: string[];
if (cquery.exitCode === 0) {
outputs = parseBazelFilesOutput(cquery.stdout);
if (options.source) {
const sourceDir = path.resolve(options.source);
outputs = conventionOutputPaths(options.targets, host).map(output =>
path.join(sourceDir, path.relative("bazel-bin", output)),
);
} else {
console.warn(`bazel cquery failed (exit ${cquery.exitCode}); falling back to bazel-bin path convention`);
outputs = conventionOutputPaths(options.targets, host);
}
if (outputs.length === 0) {
console.error("bazel build succeeded but no .node outputs were located");
process.exit(1);
const labels = resolveTargetLabels(options.targets, host);
const bazel = resolveBazelBinary();
// CI hands cache wiring (remote or disk) through a bazelrc fragment so
// endpoint composition stays in .github/actions/bazel-cache.
const rcPath = Bun.env.OMP_BAZEL_RC?.trim();
const startupArgs = rcPath ? [`--bazelrc=${rcPath}`] : [];
const buildArgs = [...startupArgs, "build", ...options.bazelArgs, "--", ...labels];
console.log(`$ ${path.basename(bazel)} ${buildArgs.join(" ")}`);
const build = await runBazel(bazel, buildArgs, "inherit");
if (build.exitCode !== 0) {
console.error(`\nbazel build failed (exit ${build.exitCode}). stderr tail:\n${build.stderrTail}`);
process.exit(build.exitCode || 1);
}
// Same flags as the build so cquery resolves the identical configuration.
// cquery takes exactly one query expression, so multiple targets join
// into a single union rather than positional args.
const cquery = await runBazel(
bazel,
[...startupArgs, "cquery", ...options.bazelArgs, "--output=files", labels.join(" + ")],
"pipe",
);
if (cquery.exitCode === 0) {
outputs = parseBazelFilesOutput(cquery.stdout);
} else {
console.warn(`bazel cquery failed (exit ${cquery.exitCode}); falling back to bazel-bin path convention`);
outputs = conventionOutputPaths(options.targets, host);
}
if (outputs.length === 0) {
console.error("bazel build succeeded but no .node outputs were located");
process.exit(1);
}
}
const seen = new Map<string, string>();