ci(infra): raised kata guest file descriptor limit
- Raised fs.nr_open from 1,048,576 to 8,388,608 through the Kata guest kernel command line. - Added a smoke check for guest, soft, and hard open-file limits. - Documented the cold Bazel sandbox failure mode and lazy descriptor allocation.
This commit is contained in:
can1357
@@ -264,7 +264,7 @@ image = "/opt/kata/share/kata-containers/kata-containers.img"
|
||||
machine_type = "q35"
|
||||
rootfs_type = "ext4"
|
||||
cpu_features = "pmu=off"
|
||||
kernel_params = "cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1"
|
||||
kernel_params = "cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 sysctl.fs.nr_open=8388608"
|
||||
|
||||
default_vcpus = 2
|
||||
default_maxvcpus = 0
|
||||
@@ -275,7 +275,7 @@ memory_slots = 10
|
||||
shared_fs = "virtio-fs"
|
||||
virtio_fs_daemon = "/opt/kata/libexec/virtiofsd"
|
||||
virtio_fs_cache = "auto"
|
||||
virtio_fs_extra_args = ["--thread-pool-size=4", "--announce-submounts"]
|
||||
virtio_fs_extra_args = ["--thread-pool-size=4", "--announce-submounts", "--rlimit-nofile=8388608"]
|
||||
|
||||
disable_block_device_use = true
|
||||
block_device_driver = "virtio-scsi"
|
||||
@@ -300,9 +300,21 @@ sandbox_cgroup_only = false
|
||||
`kata-agent` baked in as PID 1's manager). These three are the entire guest —
|
||||
none of them is the host kernel, which is the whole point. `machine_type =
|
||||
"q35"` is the modern PCIe QEMU machine (needed for PCIe hotplug);
|
||||
`cpu_features = "pmu=off"` disables the virtual perf-monitoring unit (avoids
|
||||
spurious PMU passthrough issues). `kernel_params` forces cgroup v2-only in the
|
||||
guest, matching a modern systemd userspace.
|
||||
`cpu_features = "pmu=off"` disables the virtual perf-monitoring unit and avoids
|
||||
spurious PMU passthrough issues. The first two `kernel_params` entries force
|
||||
cgroup v2-only in the guest, matching a modern systemd userspace.
|
||||
|
||||
### Open-file ceilings — Bazel sandbox headroom
|
||||
|
||||
Two limits protect each runner. `sysctl.fs.nr_open=8388608` raises the guest
|
||||
open-file ceiling from 1,048,576. New runner containers inherit this value for
|
||||
both soft and hard open-file resource limits (`RLIMIT_NOFILE`).
|
||||
|
||||
`--rlimit-nofile=8388608` raises the host `virtiofsd` process ceiling.
|
||||
Virtiofsd otherwise caps itself at 1,000,000 descriptors. Cold Bazel builds
|
||||
index large Zig and xwin trees through this daemon. The daemon exhausted its old
|
||||
ceiling and returned `EMFILE` to Bazel. Linux grows descriptor tables on demand,
|
||||
so unused headroom has no fixed allocation.
|
||||
|
||||
### vCPU / memory sizing — hotplug from pod requests/limits
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# Patch the live Kata QEMU config on the CI host to match the runner pod's
|
||||
# guaranteed boot shape and a larger virtiofsd worker pool, then smoke-test that
|
||||
# a new kata-qemu pod still boots. Driven over SSH from this repo so the desired
|
||||
# values stay version-controlled.
|
||||
# Patch the live Kata QEMU config to match the runner pod's guaranteed boot
|
||||
# shape, raise guest and host open-file limits, and enlarge the virtiofsd worker pool.
|
||||
# The script runs over SSH, which keeps the desired values version-controlled.
|
||||
# It then smoke-tests a new kata-qemu pod.
|
||||
#
|
||||
# Usage:
|
||||
# CI_HOST=my-ci-host ./infra/tune-kata-runtime.sh
|
||||
@@ -16,6 +16,7 @@
|
||||
# BOOT_VCPUS Kata default_vcpus [2]
|
||||
# BOOT_MEMORY_MIB Kata default_memory (MiB) [4096]
|
||||
# VIRTIOFSD_THREAD_POOL virtiofsd --thread-pool-size [4]
|
||||
# OPEN_FILE_LIMIT guest and virtiofsd open-file limit [8388608]
|
||||
set -euo pipefail
|
||||
|
||||
: "${CI_HOST:?set CI_HOST to the ssh target of your CI host, e.g. CI_HOST=my-ci-host}"
|
||||
@@ -26,10 +27,11 @@ ARC_NAMESPACE="${ARC_NAMESPACE:-arc-runners}"
|
||||
BOOT_VCPUS="${BOOT_VCPUS:-2}"
|
||||
BOOT_MEMORY_MIB="${BOOT_MEMORY_MIB:-4096}"
|
||||
VIRTIOFSD_THREAD_POOL="${VIRTIOFSD_THREAD_POOL:-4}"
|
||||
OPEN_FILE_LIMIT="${OPEN_FILE_LIMIT:-8388608}"
|
||||
|
||||
ssh "$CI_HOST" bash -s -- \
|
||||
"$KATA_CONFIG_REMOTE" "$KUBECONFIG_REMOTE" "$ARC_RELEASE" "$ARC_NAMESPACE" \
|
||||
"$BOOT_VCPUS" "$BOOT_MEMORY_MIB" "$VIRTIOFSD_THREAD_POOL" <<'REMOTE'
|
||||
"$BOOT_VCPUS" "$BOOT_MEMORY_MIB" "$VIRTIOFSD_THREAD_POOL" "$OPEN_FILE_LIMIT" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
KATA_CONFIG="$1"
|
||||
export KUBECONFIG="$2"
|
||||
@@ -38,12 +40,13 @@ ARC_NAMESPACE="$4"
|
||||
BOOT_VCPUS="$5"
|
||||
BOOT_MEMORY_MIB="$6"
|
||||
THREAD_POOL="$7"
|
||||
OPEN_FILE_LIMIT="$8"
|
||||
|
||||
backup="${KATA_CONFIG}.bak.$(date +%Y%m%d-%H%M%S)"
|
||||
cp "$KATA_CONFIG" "$backup"
|
||||
echo "==> backup: $backup"
|
||||
|
||||
python3 - "$KATA_CONFIG" "$BOOT_VCPUS" "$BOOT_MEMORY_MIB" "$THREAD_POOL" <<'PY'
|
||||
python3 - "$KATA_CONFIG" "$BOOT_VCPUS" "$BOOT_MEMORY_MIB" "$THREAD_POOL" "$OPEN_FILE_LIMIT" <<'PY'
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
@@ -51,21 +54,43 @@ path = Path(sys.argv[1])
|
||||
boot_vcpus = sys.argv[2]
|
||||
boot_mem = sys.argv[3]
|
||||
thread_pool = sys.argv[4]
|
||||
fd_limit_value = int(sys.argv[5])
|
||||
if fd_limit_value < 1:
|
||||
raise SystemExit("OPEN_FILE_LIMIT must be positive")
|
||||
fd_limit = str(fd_limit_value)
|
||||
text = path.read_text()
|
||||
replacements = [
|
||||
(r'(^\s*default_vcpus\s*=\s*)\d+', rf'\g<1>{boot_vcpus}'),
|
||||
(r'(^\s*default_memory\s*=\s*)\d+', rf'\g<1>{boot_mem}'),
|
||||
(r'(^\s*virtio_fs_extra_args\s*=\s*)\[[^\]]*\]', rf'\g<1>["--thread-pool-size={thread_pool}", "--announce-submounts"]'),
|
||||
(r'(^\s*virtio_fs_extra_args\s*=\s*)\[[^\]]*\]', rf'\g<1>["--thread-pool-size={thread_pool}", "--announce-submounts", "--rlimit-nofile={fd_limit}"]'),
|
||||
]
|
||||
for pattern, replacement in replacements:
|
||||
text, n = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE)
|
||||
if n != 1:
|
||||
raise SystemExit(f"failed to patch {pattern}")
|
||||
kernel_pattern = r'^(\s*kernel_params\s*=\s*")([^"]*)(".*)$'
|
||||
def update_kernel_params(match):
|
||||
params = [
|
||||
param
|
||||
for param in match.group(2).split()
|
||||
if not param.startswith("sysctl.fs.nr_open=")
|
||||
]
|
||||
params.append(f"sysctl.fs.nr_open={fd_limit}")
|
||||
return f'{match.group(1)}{" ".join(params)}{match.group(3)}'
|
||||
text, n = re.subn(
|
||||
kernel_pattern,
|
||||
update_kernel_params,
|
||||
text,
|
||||
count=1,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
if n != 1:
|
||||
raise SystemExit("failed to patch kernel_params")
|
||||
path.write_text(text)
|
||||
PY
|
||||
|
||||
echo "==> active Kata knobs"
|
||||
grep -nE 'default_vcpus|default_memory|virtio_fs_extra_args' "$KATA_CONFIG"
|
||||
grep -nE 'kernel_params|default_vcpus|default_memory|virtio_fs_extra_args' "$KATA_CONFIG"
|
||||
|
||||
image="$(kubectl get autoscalingrunnerset "$ARC_RELEASE" -n "$ARC_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
||||
pod="kata-runtime-smoke-$(date +%H%M%S)"
|
||||
@@ -77,5 +102,26 @@ kubectl run "$pod" -n "$ARC_NAMESPACE" --restart=Never --image="$image" \
|
||||
--command -- bash -lc 'sleep 120' >/dev/null
|
||||
kubectl wait --for=condition=Ready "pod/$pod" -n "$ARC_NAMESPACE" --timeout=120s >/dev/null
|
||||
kubectl exec -n "$ARC_NAMESPACE" "$pod" -- bash -lc 'bun --version; rustc --version | head -1'
|
||||
fd_probe="$(kubectl exec -n "$ARC_NAMESPACE" "$pod" -- bash -lc \
|
||||
'printf "%s %s %s" "$(cat /proc/sys/fs/nr_open)" "$(ulimit -Sn)" "$(ulimit -Hn)"')"
|
||||
read -r guest_nr_open soft_limit hard_limit <<<"$fd_probe"
|
||||
if [[ "$guest_nr_open" != "$OPEN_FILE_LIMIT" ||
|
||||
"$soft_limit" != "$OPEN_FILE_LIMIT" ||
|
||||
"$hard_limit" != "$OPEN_FILE_LIMIT" ]]; then
|
||||
echo "fd limit mismatch: nr_open=$guest_nr_open soft=$soft_limit hard=$hard_limit expected=$OPEN_FILE_LIMIT" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "guest fd limits: nr_open=$guest_nr_open soft=$soft_limit hard=$hard_limit"
|
||||
sandbox_id="$(k3s crictl pods --name "$pod" -q)"
|
||||
virtiofsd_pid="$(pgrep -fo "/opt/kata/libexec/virtiofsd.*sandboxes/${sandbox_id}/")"
|
||||
read -r virtiofsd_soft_limit virtiofsd_hard_limit < <(
|
||||
awk '/^Max open files/ { print $4, $5 }' "/proc/$virtiofsd_pid/limits"
|
||||
)
|
||||
if [[ "$virtiofsd_soft_limit" != "$OPEN_FILE_LIMIT" ||
|
||||
"$virtiofsd_hard_limit" != "$OPEN_FILE_LIMIT" ]]; then
|
||||
echo "virtiofsd fd limit mismatch: soft=$virtiofsd_soft_limit hard=$virtiofsd_hard_limit expected=$OPEN_FILE_LIMIT" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "virtiofsd fd limits: soft=$virtiofsd_soft_limit hard=$virtiofsd_hard_limit"
|
||||
echo "OK: kata-qemu still boots after tuning"
|
||||
REMOTE
|
||||
|
||||
Reference in New Issue
Block a user