diff --git a/infra/docs/02-kata-runtime.md b/infra/docs/02-kata-runtime.md index 069d600f4..33e3137f3 100644 --- a/infra/docs/02-kata-runtime.md +++ b/infra/docs/02-kata-runtime.md @@ -264,7 +264,7 @@ image = "/opt/kata/share/kata-containers/kata-containers.img" machine_type = "q35" rootfs_type = "ext4" cpu_features = "pmu=off" -kernel_params = "cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1" +kernel_params = "cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 sysctl.fs.nr_open=8388608" default_vcpus = 2 default_maxvcpus = 0 @@ -275,7 +275,7 @@ memory_slots = 10 shared_fs = "virtio-fs" virtio_fs_daemon = "/opt/kata/libexec/virtiofsd" virtio_fs_cache = "auto" -virtio_fs_extra_args = ["--thread-pool-size=4", "--announce-submounts"] +virtio_fs_extra_args = ["--thread-pool-size=4", "--announce-submounts", "--rlimit-nofile=8388608"] disable_block_device_use = true block_device_driver = "virtio-scsi" @@ -300,9 +300,21 @@ sandbox_cgroup_only = false `kata-agent` baked in as PID 1's manager). These three are the entire guest — none of them is the host kernel, which is the whole point. `machine_type = "q35"` is the modern PCIe QEMU machine (needed for PCIe hotplug); -`cpu_features = "pmu=off"` disables the virtual perf-monitoring unit (avoids -spurious PMU passthrough issues). `kernel_params` forces cgroup v2-only in the -guest, matching a modern systemd userspace. +`cpu_features = "pmu=off"` disables the virtual perf-monitoring unit and avoids +spurious PMU passthrough issues. The first two `kernel_params` entries force +cgroup v2-only in the guest, matching a modern systemd userspace. + +### Open-file ceilings — Bazel sandbox headroom + +Two limits protect each runner. `sysctl.fs.nr_open=8388608` raises the guest +open-file ceiling from 1,048,576. New runner containers inherit this value for +both soft and hard open-file resource limits (`RLIMIT_NOFILE`). + +`--rlimit-nofile=8388608` raises the host `virtiofsd` process ceiling. +Virtiofsd otherwise caps itself at 1,000,000 descriptors. Cold Bazel builds +index large Zig and xwin trees through this daemon. The daemon exhausted its old +ceiling and returned `EMFILE` to Bazel. Linux grows descriptor tables on demand, +so unused headroom has no fixed allocation. ### vCPU / memory sizing — hotplug from pod requests/limits diff --git a/infra/tune-kata-runtime.sh b/infra/tune-kata-runtime.sh index 8c782a523..2766d7c24 100755 --- a/infra/tune-kata-runtime.sh +++ b/infra/tune-kata-runtime.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# Patch the live Kata QEMU config on the CI host to match the runner pod's -# guaranteed boot shape and a larger virtiofsd worker pool, then smoke-test that -# a new kata-qemu pod still boots. Driven over SSH from this repo so the desired -# values stay version-controlled. +# Patch the live Kata QEMU config to match the runner pod's guaranteed boot +# shape, raise guest and host open-file limits, and enlarge the virtiofsd worker pool. +# The script runs over SSH, which keeps the desired values version-controlled. +# It then smoke-tests a new kata-qemu pod. # # Usage: # CI_HOST=my-ci-host ./infra/tune-kata-runtime.sh @@ -16,6 +16,7 @@ # BOOT_VCPUS Kata default_vcpus [2] # BOOT_MEMORY_MIB Kata default_memory (MiB) [4096] # VIRTIOFSD_THREAD_POOL virtiofsd --thread-pool-size [4] +# OPEN_FILE_LIMIT guest and virtiofsd open-file limit [8388608] set -euo pipefail : "${CI_HOST:?set CI_HOST to the ssh target of your CI host, e.g. CI_HOST=my-ci-host}" @@ -26,10 +27,11 @@ ARC_NAMESPACE="${ARC_NAMESPACE:-arc-runners}" BOOT_VCPUS="${BOOT_VCPUS:-2}" BOOT_MEMORY_MIB="${BOOT_MEMORY_MIB:-4096}" VIRTIOFSD_THREAD_POOL="${VIRTIOFSD_THREAD_POOL:-4}" +OPEN_FILE_LIMIT="${OPEN_FILE_LIMIT:-8388608}" ssh "$CI_HOST" bash -s -- \ "$KATA_CONFIG_REMOTE" "$KUBECONFIG_REMOTE" "$ARC_RELEASE" "$ARC_NAMESPACE" \ - "$BOOT_VCPUS" "$BOOT_MEMORY_MIB" "$VIRTIOFSD_THREAD_POOL" <<'REMOTE' + "$BOOT_VCPUS" "$BOOT_MEMORY_MIB" "$VIRTIOFSD_THREAD_POOL" "$OPEN_FILE_LIMIT" <<'REMOTE' set -euo pipefail KATA_CONFIG="$1" export KUBECONFIG="$2" @@ -38,12 +40,13 @@ ARC_NAMESPACE="$4" BOOT_VCPUS="$5" BOOT_MEMORY_MIB="$6" THREAD_POOL="$7" +OPEN_FILE_LIMIT="$8" backup="${KATA_CONFIG}.bak.$(date +%Y%m%d-%H%M%S)" cp "$KATA_CONFIG" "$backup" echo "==> backup: $backup" -python3 - "$KATA_CONFIG" "$BOOT_VCPUS" "$BOOT_MEMORY_MIB" "$THREAD_POOL" <<'PY' +python3 - "$KATA_CONFIG" "$BOOT_VCPUS" "$BOOT_MEMORY_MIB" "$THREAD_POOL" "$OPEN_FILE_LIMIT" <<'PY' from pathlib import Path import re import sys @@ -51,21 +54,43 @@ path = Path(sys.argv[1]) boot_vcpus = sys.argv[2] boot_mem = sys.argv[3] thread_pool = sys.argv[4] +fd_limit_value = int(sys.argv[5]) +if fd_limit_value < 1: + raise SystemExit("OPEN_FILE_LIMIT must be positive") +fd_limit = str(fd_limit_value) text = path.read_text() replacements = [ (r'(^\s*default_vcpus\s*=\s*)\d+', rf'\g<1>{boot_vcpus}'), (r'(^\s*default_memory\s*=\s*)\d+', rf'\g<1>{boot_mem}'), - (r'(^\s*virtio_fs_extra_args\s*=\s*)\[[^\]]*\]', rf'\g<1>["--thread-pool-size={thread_pool}", "--announce-submounts"]'), + (r'(^\s*virtio_fs_extra_args\s*=\s*)\[[^\]]*\]', rf'\g<1>["--thread-pool-size={thread_pool}", "--announce-submounts", "--rlimit-nofile={fd_limit}"]'), ] for pattern, replacement in replacements: text, n = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE) if n != 1: raise SystemExit(f"failed to patch {pattern}") +kernel_pattern = r'^(\s*kernel_params\s*=\s*")([^"]*)(".*)$' +def update_kernel_params(match): + params = [ + param + for param in match.group(2).split() + if not param.startswith("sysctl.fs.nr_open=") + ] + params.append(f"sysctl.fs.nr_open={fd_limit}") + return f'{match.group(1)}{" ".join(params)}{match.group(3)}' +text, n = re.subn( + kernel_pattern, + update_kernel_params, + text, + count=1, + flags=re.MULTILINE, +) +if n != 1: + raise SystemExit("failed to patch kernel_params") path.write_text(text) PY echo "==> active Kata knobs" -grep -nE 'default_vcpus|default_memory|virtio_fs_extra_args' "$KATA_CONFIG" +grep -nE 'kernel_params|default_vcpus|default_memory|virtio_fs_extra_args' "$KATA_CONFIG" image="$(kubectl get autoscalingrunnerset "$ARC_RELEASE" -n "$ARC_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}')" pod="kata-runtime-smoke-$(date +%H%M%S)" @@ -77,5 +102,26 @@ kubectl run "$pod" -n "$ARC_NAMESPACE" --restart=Never --image="$image" \ --command -- bash -lc 'sleep 120' >/dev/null kubectl wait --for=condition=Ready "pod/$pod" -n "$ARC_NAMESPACE" --timeout=120s >/dev/null kubectl exec -n "$ARC_NAMESPACE" "$pod" -- bash -lc 'bun --version; rustc --version | head -1' +fd_probe="$(kubectl exec -n "$ARC_NAMESPACE" "$pod" -- bash -lc \ + 'printf "%s %s %s" "$(cat /proc/sys/fs/nr_open)" "$(ulimit -Sn)" "$(ulimit -Hn)"')" +read -r guest_nr_open soft_limit hard_limit <<<"$fd_probe" +if [[ "$guest_nr_open" != "$OPEN_FILE_LIMIT" || + "$soft_limit" != "$OPEN_FILE_LIMIT" || + "$hard_limit" != "$OPEN_FILE_LIMIT" ]]; then + echo "fd limit mismatch: nr_open=$guest_nr_open soft=$soft_limit hard=$hard_limit expected=$OPEN_FILE_LIMIT" >&2 + exit 1 +fi +echo "guest fd limits: nr_open=$guest_nr_open soft=$soft_limit hard=$hard_limit" +sandbox_id="$(k3s crictl pods --name "$pod" -q)" +virtiofsd_pid="$(pgrep -fo "/opt/kata/libexec/virtiofsd.*sandboxes/${sandbox_id}/")" +read -r virtiofsd_soft_limit virtiofsd_hard_limit < <( + awk '/^Max open files/ { print $4, $5 }' "/proc/$virtiofsd_pid/limits" +) +if [[ "$virtiofsd_soft_limit" != "$OPEN_FILE_LIMIT" || + "$virtiofsd_hard_limit" != "$OPEN_FILE_LIMIT" ]]; then + echo "virtiofsd fd limit mismatch: soft=$virtiofsd_soft_limit hard=$virtiofsd_hard_limit expected=$OPEN_FILE_LIMIT" >&2 + exit 1 +fi +echo "virtiofsd fd limits: soft=$virtiofsd_soft_limit hard=$virtiofsd_hard_limit" echo "OK: kata-qemu still boots after tuning" REMOTE