From ade5298e1ec74cc1b18dddb34a3e9f53a1a66693 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 18 Aug 2026 20:06:44 +0000 Subject: [PATCH] add cgroupe v1 analysis --- .github/workflows/bench.yaml | 48 +++++++ README.md | 14 ++ probe.py | 69 ++++++++-- report.py | 11 +- scenario-helpers.sh | 6 + scenarios/systemd-ancestor.sh | 2 +- scenarios/systemd-memory-above-host.sh | 2 +- scenarios/systemd-own.sh | 11 +- scenarios/systemd-quota-above-host.sh | 2 +- v1-guest.sh | 176 +++++++++++++++++++++++++ 10 files changed, 318 insertions(+), 23 deletions(-) create mode 100755 v1-guest.sh diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 691292e..966dcfe 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -21,6 +21,11 @@ on: image: description: 'Base image the container scenarios run in. Any glibc distro works - uv is mounted in' default: 'ghcr.io/astral-sh/uv:python3.13-bookworm' + cgroup_v1: + description: 'Also run the bench in a guest booted on cgroup v1 (adds a few minutes)' + type: choice + options: ['off', hybrid, legacy] + default: 'off' run-name: 'bench: ${{ inputs.ref }} / ${{ inputs.scenarios }} / cpus ${{ inputs.cpu_budgets }}' @@ -65,3 +70,46 @@ jobs: with: name: results-${{ matrix.cpus }}cpu path: results/ + + bench-v1: + # The runner is cgroup v2 only, and a controller cannot be mounted as v1 while the unified hierarchy owns + # it, so the v1 side of the library is only reachable inside a guest booted in legacy mode. + if: inputs.cgroup_v1 != 'off' + name: bench (cgroup v1 guest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + + - name: Allow KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Install qemu + run: sudo apt-get install -y --no-install-recommends qemu-system-x86 cloud-image-utils + + - name: Boot the guest and run the bench inside it + env: + CRAWLEE_REPO: ${{ inputs.repo }} + CRAWLEE_REF: ${{ inputs.ref }} + V1_MODE: ${{ inputs.cgroup_v1 }} + run: ./v1-guest.sh results/ + + - name: Report + if: always() + run: python3 report.py results/ >> "$GITHUB_STEP_SUMMARY" + + - name: Check + if: always() + run: python3 report.py results/ --check >/dev/null + + - name: Upload raw results + if: always() + uses: actions/upload-artifact@v4 + with: + name: results-cgroup-v1-${{ inputs.cgroup_v1 }} + path: results/ diff --git a/README.md b/README.md index fd13727..2000942 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,19 @@ Scenarios whose prerequisites are missing (docker, passwordless sudo, a systemd cores) are recorded as `skipped` with the reason; only a crashed probe reddens a run. The `k8s-*` scenarios bring up a kind cluster named `cgroups-bench` and leave it running — `kind delete cluster --name cgroups-bench`. +**cgroup v1** cannot be produced on a modern host: a controller belongs to one hierarchy at a time, and on a +unified system it cannot be mounted as v1 even by root — inside a user namespace, not at all. So the v1 half of +the library is reachable only from a guest kernel, which is what `./v1-guest.sh` does: it boots Ubuntu 22.04 +(systemd 249, the last releases that still honour the flags), runs the same scenarios inside and copies the +results back into the same report. `V1_MODE` picks what the guest boots into — `hybrid` puts the v1 controllers +next to a controller-less cgroup2, so the library has to notice the unified hierarchy is empty and fall back +per controller, while `legacy` is plain v1. In the workflow both are the `cgroup_v1` input. + +It needs `qemu-system-x86 cloud-image-utils` and a usable `/dev/kvm` (`sudo usermod -aG kvm $USER` locally, a +udev rule on a runner); `V1_ACCEL=tcg` emulates instead, which is slow but needs no privileges. The report's +`v2` column says what the guest actually came up as, and a v1 host with no limit reports the sentinel that +shows up in the table as `8.00 EB`. + ## Files | file | what it does | @@ -39,6 +52,7 @@ bring up a kind cluster named `cgroups-bench` and leave it running — `kind del | [probe.py](probe.py) | Runs inside the prepared environment and prints one JSON object with what Crawlee sees there. | | [wrap.py](wrap.py) | Folds the probe's output and the scenario's configuration into one result file. | | [report.py](report.py) | Turns a results directory into the tables. `--check` is the only gate. | +| [v1-guest.sh](v1-guest.sh) | Boots a guest on cgroup v1 and runs the bench inside it, bringing the results back. | | [.github/workflows/bench.yaml](.github/workflows/bench.yaml) | Manual trigger only, one job per CPU budget. | ## Scenarios diff --git a/probe.py b/probe.py index 1e0c1ec..7ffec8d 100644 --- a/probe.py +++ b/probe.py @@ -9,7 +9,7 @@ SCHEMA = 1 -_EVIDENCE_FILES = ( +_V2_EVIDENCE_FILES = ( 'memory.max', 'memory.current', 'cpu.max', @@ -18,6 +18,17 @@ 'cgroup.controllers', ) +_V1_EVIDENCE_FILES = ( + 'memory.limit_in_bytes', + 'memory.usage_in_bytes', + 'cpu.cfs_quota_us', + 'cpu.cfs_period_us', + 'cpuacct.usage', + 'cpuset.cpus', +) + +_V1_CONTROLLERS = ('memory', 'cpu', 'cpuacct', 'cpuset') + _MAX_LEVELS = 20 """How far up the cgroup chain the evidence dump walks. Deeper than any real hierarchy, so it only bounds a loop.""" @@ -60,26 +71,58 @@ def _read_control_file(path: Path) -> str | None: return None +def _walk_levels(mount: Path, own_path: str, names: tuple[str, ...]) -> list[dict[str, Any]]: + """Read the named control files at the process's own cgroup and at every level above it, leaf first.""" + levels = [] + directory = mount / own_path.lstrip('/') if '..' not in own_path else mount + for _ in range(_MAX_LEVELS): + files = {name: content for name in names if (content := _read_control_file(directory / name))} + levels.append({'path': str(directory), 'files': files}) + if directory in (mount, directory.parent): + break + directory = directory.parent + return levels + + def _evidence() -> dict[str, Any]: - """Dump raw cgroup file contents along the chain, verbatim - receipts, not a verdict, never interpreted.""" + """Dump raw cgroup file contents along the chain, verbatim - receipts, not a verdict, never interpreted. + + Both cgroup versions are dumped the same way, from the standard mount points: the unified hierarchy sits at + /sys/fs/cgroup, while a v1 controller has a directory of its own under it. Resolving mounts properly is the + library's job, and this deliberately does not repeat it - the receipts only have to be readable next to the + values the library reported. + """ evidence: dict[str, Any] = {'proc_cgroup': None, 'levels': []} try: evidence['proc_cgroup'] = Path('/proc/self/cgroup').read_text().strip() except OSError: return evidence - own_path = next((line[3:] for line in evidence['proc_cgroup'].splitlines() if line.startswith('0::')), None) - if own_path is None: - return evidence - mount = Path('/sys/fs/cgroup') - directory = mount / own_path.lstrip('/') if '..' not in own_path else mount - for _ in range(_MAX_LEVELS): - files = {name: content for name in _EVIDENCE_FILES if (content := _read_control_file(directory / name))} - evidence['levels'].append({'path': str(directory), 'files': files}) - if directory in (mount, directory.parent): - break - directory = directory.parent + seen: set[str] = set() + + for line in evidence['proc_cgroup'].splitlines(): + parts = line.split(':', 2) + if len(parts) != 3: + continue + _hierarchy_id, controllers, own_path = parts + + if not controllers: + levels = _walk_levels(mount, own_path, _V2_EVIDENCE_FILES) + else: + # A v1 line can name several controllers sharing one mount, as `cpu,cpuacct` usually does. + names = [name for name in controllers.split(',') if name in _V1_CONTROLLERS] + levels = [ + level + for name in names + for level in _walk_levels(mount / name, own_path, _V1_EVIDENCE_FILES) + ] + + for level in levels: + if level['path'] not in seen: + seen.add(level['path']) + evidence['levels'].append(level) + return evidence diff --git a/report.py b/report.py index e264f75..d195851 100644 --- a/report.py +++ b/report.py @@ -6,16 +6,17 @@ from pathlib import Path from typing import Any -_BYTES_PER_GB = 1024**3 -_BYTES_PER_MB = 1024**2 +_BYTE_UNITS = (('EB', 1024**6), ('PB', 1024**5), ('TB', 1024**4), ('GB', 1024**3), ('MB', 1024**2)) def _bytes_h(value: Any) -> str: + """Format a byte count. The largest units only ever show up as the v1 sentinel for "no limit at all".""" if not isinstance(value, (int, float)): return '-' - if value >= _BYTES_PER_GB: - return f'{value / _BYTES_PER_GB:.2f} GB' - return f'{value / _BYTES_PER_MB:.2f} MB' + for unit, size in _BYTE_UNITS: + if value >= size: + return f'{value / size:.2f} {unit}' + return f'{value / _BYTE_UNITS[-1][1]:.2f} MB' def _num(value: Any) -> str: diff --git a/scenario-helpers.sh b/scenario-helpers.sh index a02912c..bc87487 100644 --- a/scenario-helpers.sh +++ b/scenario-helpers.sh @@ -17,6 +17,11 @@ have() { command -v "$1" >/dev/null 2>&1; } cpuset_list() { [ "$BENCH_CPUS" -le 1 ] && echo '0' || echo "0-$((BENCH_CPUS - 1))"; } have_engine() { "$ENGINE" info >/dev/null 2>&1; } +# True when the unified hierarchy is the one carrying the controllers. Under cgroup v1 - and under the hybrid +# layout, where a controller-less cgroup2 sits next to the v1 mounts - systemd puts resource limits only on +# system units, and properties that exist solely for the unified hierarchy are accepted and then ignored. +# Read it rather than test its size: every file in cgroupfs reports zero bytes, so -s is always false here. +have_unified() { [ -n "$(cat /sys/fs/cgroup/cgroup.controllers 2>/dev/null)" ]; } have_sudo() { sudo -n true 2>/dev/null; } have_systemd_user() { systemd-run --user --scope -q true 2>/dev/null; } cgroup_driver() { "$ENGINE" info -f '{{.CgroupDriver}}' 2>/dev/null; } @@ -31,6 +36,7 @@ unmet_requirement() { engine) have_engine || { echo "container engine '$ENGINE' is not available"; return; } ;; sudo) have_sudo || { echo "passwordless sudo is not available"; return; } ;; systemd-user) have_systemd_user || { echo "systemd user manager is not available"; return; } ;; + unified) have_unified || { echo "the controllers are not on the unified hierarchy, and systemd gives user scopes no cgroup of their own under cgroup v1"; return; } ;; systemd-driver) [ "$(cgroup_driver)" = systemd ] || { echo "$ENGINE uses the '$(cgroup_driver)' cgroup driver, this scenario needs 'systemd'"; return; } ;; kind) diff --git a/scenarios/systemd-ancestor.sh b/scenarios/systemd-ancestor.sh index d401fac..0f84827 100644 --- a/scenarios/systemd-ancestor.sh +++ b/scenarios/systemd-ancestor.sh @@ -1,6 +1,6 @@ # shellcheck shell=bash SCENARIO_DESC="limit on an ancestor, leaf without memory controller files" -REQUIRES="systemd-user" +REQUIRES="systemd-user unified" INNER_STYLE=local SET_MEMORY_BYTES=$((512 * 1024 * 1024)) diff --git a/scenarios/systemd-memory-above-host.sh b/scenarios/systemd-memory-above-host.sh index ad6c4d6..2f3a095 100644 --- a/scenarios/systemd-memory-above-host.sh +++ b/scenarios/systemd-memory-above-host.sh @@ -1,6 +1,6 @@ # shellcheck shell=bash SCENARIO_DESC="a memory limit larger than the machine has RAM" -REQUIRES="systemd-user" +REQUIRES="systemd-user unified" INNER_STYLE=local # The memory twin of systemd-quota-above-host, and the axis where the library already guards itself: a limit at diff --git a/scenarios/systemd-own.sh b/scenarios/systemd-own.sh index a16cfbb..86e8951 100644 --- a/scenarios/systemd-own.sh +++ b/scenarios/systemd-own.sh @@ -12,14 +12,21 @@ QUOTA_CORES=$((BENCH_CPUS + 1)) SET_MEMORY_BYTES=$((512 * 1024 * 1024)) SET_CPU_CORES=$QUOTA_CORES -SET_CPUSET_CORES=$BENCH_CPUS + +# AllowedCPUs= exists only for the unified hierarchy: under cgroup v1 systemd accepts it and silently applies +# nothing, so the ask is left out there rather than recorded as a limit that was never set. +CPUSET_PROPS=() +if have_unified; then + SET_CPUSET_CORES=$BENCH_CPUS + CPUSET_PROPS=(-p "AllowedCPUs=$(cpuset_list)") +fi UNIT="bench-systemd-own-$$" # sudo strips the environment, so the probe needs PATH and a cache root can write to. scenario_exec() { sudo systemd-run --scope -q --unit "$UNIT" \ - -p "MemoryMax=$SET_MEMORY_BYTES" -p "CPUQuota=$((QUOTA_CORES * 100))%" -p "AllowedCPUs=$(cpuset_list)" \ + -p "MemoryMax=$SET_MEMORY_BYTES" -p "CPUQuota=$((QUOTA_CORES * 100))%" "${CPUSET_PROPS[@]}" \ env "PATH=$PATH" "HOME=/root" "UV_CACHE_DIR=$BENCH_UV_CACHE/root" sh -c "$1" } diff --git a/scenarios/systemd-quota-above-host.sh b/scenarios/systemd-quota-above-host.sh index 81eea0e..e28f8bb 100644 --- a/scenarios/systemd-quota-above-host.sh +++ b/scenarios/systemd-quota-above-host.sh @@ -1,6 +1,6 @@ # shellcheck shell=bash SCENARIO_DESC="a cpu quota larger than the machine has cores" -REQUIRES="systemd-user" +REQUIRES="systemd-user unified" INNER_STYLE=local # Nothing rejects a quota above the machine's core count: systemd writes it as asked, and a kubernetes limit diff --git a/v1-guest.sh b/v1-guest.sh new file mode 100755 index 0000000..f5a0923 --- /dev/null +++ b/v1-guest.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# Run the whole bench inside a guest booted on cgroup v1, and bring its results back. +# +# cgroup v1 cannot be produced on a modern host: a controller belongs to exactly one hierarchy, and on a +# unified system they all belong to v2 - mounting them as v1 fails with EBUSY even as root, and inside a user +# namespace v1 cannot be mounted at all. A guest kernel booted in legacy mode is the only way to see those code +# paths, so this is a lane rather than a scenario: the same run.sh runs inside, and every scenario that can run +# there exercises the v1 side of the library. +# +# Usage: ./v1-guest.sh [results_dir] +# +# Environment: +# V1_IMAGE_URL / V1_KERNEL_URL / V1_INITRD_URL guest to boot (default: Ubuntu 22.04 cloud image) +# V1_CMDLINE kernel command line (default: legacy cgroup mode) +# V1_SCENARIOS what to run inside (default: everything the guest can do) +# V1_MEMORY / V1_CPUS guest size (default 4096 MB, 2 cores) +# plus CRAWLEE_REPO, CRAWLEE_REF, BENCH_CPUS, which are passed through to the bench + +set -eu + +SENSOR_DIR=$(cd "$(dirname "$0")" && pwd) +RESULTS=${1:-results} + +WORK=${V1_WORK:-/tmp/cgroups-bench-v1} +SSH_PORT=${V1_SSH_PORT:-2222} +GUEST_USER=bench + +BASE=https://cloud-images.ubuntu.com/releases/22.04/release +V1_IMAGE_URL=${V1_IMAGE_URL:-$BASE/ubuntu-22.04-server-cloudimg-amd64.img} +V1_KERNEL_URL=${V1_KERNEL_URL:-$BASE/unpacked/ubuntu-22.04-server-cloudimg-amd64-vmlinuz-generic} +V1_INITRD_URL=${V1_INITRD_URL:-$BASE/unpacked/ubuntu-22.04-server-cloudimg-amd64-initrd-generic} + +# systemd honours these up to v255; they are ignored from v256 and cgroup v1 is gone in v258, which is why the +# guest is Ubuntu 22.04 (systemd 249) rather than something newer. Booting the kernel directly, rather than +# through the image's own bootloader, is what lets the line be set from here without editing the image first. +# +# hybrid (the default) mounts the v1 controllers and a controller-less cgroup2 next to them, so the library has +# to notice that the unified hierarchy carries nothing and fall back per controller. legacy is plain v1. +case ${V1_MODE:-hybrid} in + hybrid) CGROUP_ARGS="systemd.unified_cgroup_hierarchy=0" ;; + legacy) CGROUP_ARGS="systemd.unified_cgroup_hierarchy=0 systemd.legacy_systemd_cgroup_controller=1" ;; + *) echo "v1-guest: V1_MODE must be hybrid or legacy" >&2; exit 1 ;; +esac +V1_CMDLINE=${V1_CMDLINE:-"root=/dev/vda1 console=ttyS0 $CGROUP_ARGS"} + +# kind needs a container runtime and a lot of memory; the k8s shapes are already covered on the v2 host. +V1_SCENARIOS=${V1_SCENARIOS:-"bare systemd-own systemd-ancestor systemd-quota-above-host systemd-memory-above-host docker-private docker-memory-only docker-cpuset-only docker-host-ns docker-nested-subgroup"} + +CRAWLEE_REPO=${CRAWLEE_REPO:-https://github.com/Mantisus/crawlee-python} +CRAWLEE_REF=${CRAWLEE_REF:-container-limits} +BENCH_CPUS=${BENCH_CPUS:-1} + +need() { command -v "$1" >/dev/null 2>&1 || { echo "v1-guest: $1 is required (apt install $2)" >&2; exit 1; }; } +need qemu-system-x86_64 qemu-system-x86 +need cloud-localds cloud-image-utils +need ssh openssh-client +# Emulation works without any privileges but is slow enough to be useful only for checking the plumbing, so it +# has to be asked for by name rather than silently turning a five minute lane into a thirty minute one. +V1_ACCEL=${V1_ACCEL:-kvm} +if [ "$V1_ACCEL" = kvm ] && ! { [ -r /dev/kvm ] && [ -w /dev/kvm ]; }; then + echo "v1-guest: /dev/kvm is not usable. On a GitHub runner:" >&2 + echo " echo 'KERNEL==\"kvm\", GROUP=\"kvm\", MODE=\"0666\", OPTIONS+=\"static_node=kvm\"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules" >&2 + echo " sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm" >&2 + echo " locally: sudo usermod -aG kvm \$USER (then log in again), or run with V1_ACCEL=tcg to emulate" >&2 + exit 1 +fi +[ "$V1_ACCEL" = kvm ] || echo "v1-guest: no KVM, emulating - expect this to be several times slower" >&2 + +mkdir -p "$WORK" "$RESULTS" + +fetch() { # url -> cached file, downloaded once per machine + local url=$1 dest="$WORK/$(basename "$1")" + [ -s "$dest" ] || { echo "v1-guest: downloading $(basename "$url")" >&2; curl -fsSL -o "$dest" "$url"; } + echo "$dest" +} + +IMAGE=$(fetch "$V1_IMAGE_URL") +KERNEL=$(fetch "$V1_KERNEL_URL") +INITRD=$(fetch "$V1_INITRD_URL") + +# A throwaway overlay, so a run never dirties the downloaded image and every run starts clean. +DISK=$WORK/run.qcow2 +rm -f "$DISK" +qemu-img create -q -f qcow2 -F qcow2 -b "$IMAGE" "$DISK" 8G + +KEY=$WORK/id_ed25519 +[ -s "$KEY" ] || ssh-keygen -q -t ed25519 -N '' -f "$KEY" + +cat > "$WORK/user-data" </dev/null + rm -f "$WORK/qemu.pid" +} +trap cleanup EXIT + +# scp spells the port -P and reads -p as "preserve timestamps", so the two need separate option strings. +SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" +# shellcheck disable=SC2086 +in_guest() { ssh $SSH_OPTS -p "$SSH_PORT" "$GUEST_USER@127.0.0.1" "$@"; } +# shellcheck disable=SC2086 +copy_in() { scp -q $SSH_OPTS -P "$SSH_PORT" -r "$@" "$GUEST_USER@127.0.0.1:bench/"; } +# shellcheck disable=SC2086 +copy_out() { scp -q $SSH_OPTS -P "$SSH_PORT" "$GUEST_USER@127.0.0.1:$1" "$2"; } + +echo -n "v1-guest: waiting for ssh" +for _ in $(seq 1 120); do + in_guest true 2>/dev/null && break + echo -n . + sleep 2 +done +echo +in_guest true 2>/dev/null || { + echo "v1-guest: the guest never came up; the last of its console output:" >&2 + tail -30 "$WORK/console.log" >&2 + exit 1 +} + +# v1 controllers each get a mount of their own, so counting both kinds says which mode the guest came up in: +# only cgroup mounts is legacy, both kinds is hybrid, only cgroup2 means the kernel line did not take. +echo -n 'v1-guest: guest is up, ' +in_guest 'printf "cgroup mounts: %s v1, %s v2\n" \ + "$(grep -c " - cgroup " /proc/self/mountinfo || true)" "$(grep -c " - cgroup2 " /proc/self/mountinfo || true)"' + +# ssh answers as soon as sshd is up, while cloud-init is still installing packages behind it. +echo "v1-guest: waiting for cloud-init to finish" +in_guest 'cloud-init status --wait >/dev/null 2>&1 || true' + +in_guest 'mkdir -p bench' +copy_in "$SENSOR_DIR/run.sh" "$SENSOR_DIR/probe.py" "$SENSOR_DIR/wrap.py" "$SENSOR_DIR/report.py" \ + "$SENSOR_DIR/scenario-helpers.sh" "$SENSOR_DIR/scenarios" "$(command -v uv)" +in_guest 'sudo install -m 0755 bench/uv /usr/local/bin/uv' +in_guest 'getent group docker >/dev/null && sudo usermod -aG docker '"$GUEST_USER"' || true' + +echo "v1-guest: running the bench inside" +BENCH_CMD="cd bench && CRAWLEE_REPO='$CRAWLEE_REPO' CRAWLEE_REF='$CRAWLEE_REF' BENCH_CPUS='$BENCH_CPUS' \ + ./run.sh $V1_SCENARIOS results/" +# The freshly granted docker group only applies to new logins, so borrow it for this command when it exists. +if in_guest 'getent group docker >/dev/null'; then + in_guest "sg docker -c \"$BENCH_CMD\"" +else + in_guest "$BENCH_CMD" +fi + +# shellcheck disable=SC2086 +copy_out 'bench/results/*.json' "$RESULTS/" +in_guest 'sudo poweroff' 2>/dev/null || true + +echo "v1-guest: results in $RESULTS"