From 537aa03893bd08c21e4a6a265c76fb4c4d9fb21b Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Wed, 19 Aug 2026 11:27:09 +0200 Subject: [PATCH] ci: bound apt-get so a dead mirror cannot wedge a job Runners intermittently lose egress to azure.archive.ubuntu.com. apt falls back through the mirror list, then stalls with no output at all, so the job sits dead until it burns the full 60 minute budget. Three consecutive doctest runs on PR #3921 died this way, each on a different runner, while sibling jobs on the same run finished apt in nine seconds. apt's own Acquire::*::Timeout does not bound this - the observed stalls ran half an hour past the 120s default - so scripts/ci/apt-install.sh caps both the index refresh and the install with timeout(1). A wedged runner now fails loudly in minutes instead of dying quietly at the job limit. Installation still runs after a failed update, so a merely stale index set is not fatal. Every call site routes through it rather than repeating the loop inline. --- .../actions/cpp-bazel/pre-merge/action.yml | 3 +- .github/actions/node-npm/pre-merge/action.yml | 2 +- .github/actions/php/pre-merge/action.yml | 3 +- .github/actions/rust/pre-merge/action.yml | 4 +- .../utils/setup-rust-with-cache/action.yml | 4 +- .github/workflows/_build_rust_artifacts.yml | 2 +- .github/workflows/_test_examples.yml | 3 +- .github/workflows/coverage-baseline.yml | 5 +- scripts/ci/apt-install.sh | 78 +++++++++++++++++++ 9 files changed, 88 insertions(+), 16 deletions(-) create mode 100755 scripts/ci/apt-install.sh diff --git a/.github/actions/cpp-bazel/pre-merge/action.yml b/.github/actions/cpp-bazel/pre-merge/action.yml index 20e7e465b1..102a9302c0 100644 --- a/.github/actions/cpp-bazel/pre-merge/action.yml +++ b/.github/actions/cpp-bazel/pre-merge/action.yml @@ -41,8 +41,7 @@ runs: clang-format-18 --version exit 0 fi - sudo apt-get update --yes - sudo apt-get install --yes clang-format-18 + ./scripts/ci/apt-install.sh clang-format-18 clang-format-18 --version - name: Lint (clang-format) diff --git a/.github/actions/node-npm/pre-merge/action.yml b/.github/actions/node-npm/pre-merge/action.yml index 34b83e0b2f..97771a05b9 100644 --- a/.github/actions/node-npm/pre-merge/action.yml +++ b/.github/actions/node-npm/pre-merge/action.yml @@ -44,7 +44,7 @@ runs: - name: Install netcat if: inputs.task == 'e2e' - run: sudo apt-get update && sudo apt-get install -y netcat-openbsd + run: ./scripts/ci/apt-install.sh netcat-openbsd shell: bash - name: Install dependencies diff --git a/.github/actions/php/pre-merge/action.yml b/.github/actions/php/pre-merge/action.yml index 02b1c70e4b..789e9db400 100644 --- a/.github/actions/php/pre-merge/action.yml +++ b/.github/actions/php/pre-merge/action.yml @@ -29,8 +29,7 @@ runs: - name: Install PHP build dependencies shell: bash run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + ./scripts/ci/apt-install.sh --no-install-recommends \ clang \ composer \ libclang-dev \ diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index ec5d327fd1..0c8f47a24f 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -195,7 +195,7 @@ runs: - name: Install dependencies for Rust tests if: startsWith(inputs.task, 'test-') && runner.os == 'Linux' run: | - sudo apt-get install --yes musl-tools gnome-keyring keyutils dbus-x11 libsecret-tools + ./scripts/ci/apt-install.sh musl-tools gnome-keyring keyutils dbus-x11 libsecret-tools rm -f $HOME/.local/share/keyrings/* shell: bash @@ -370,7 +370,7 @@ runs: - name: Install musl tools for aarch64-musl if: inputs.task == 'build-aarch64-musl' && runner.os == 'Linux' run: | - sudo apt-get update && sudo apt-get install -y musl-tools + ./scripts/ci/apt-install.sh musl-tools rustup target add aarch64-unknown-linux-musl shell: bash diff --git a/.github/actions/utils/setup-rust-with-cache/action.yml b/.github/actions/utils/setup-rust-with-cache/action.yml index 6c3bf16fb4..82867a968e 100644 --- a/.github/actions/utils/setup-rust-with-cache/action.yml +++ b/.github/actions/utils/setup-rust-with-cache/action.yml @@ -64,9 +64,7 @@ runs: - name: Install system dependencies (Linux) if: runner.os == 'Linux' && inputs.install-system-dependencies == 'true' - run: | - sudo apt-get update - sudo apt-get install -y libhwloc-dev pkg-config libudev-dev + run: ./scripts/ci/apt-install.sh libhwloc-dev pkg-config libudev-dev shell: bash - name: Install system dependencies (macOS) diff --git a/.github/workflows/_build_rust_artifacts.yml b/.github/workflows/_build_rust_artifacts.yml index 56232ff1b0..bd56b10716 100644 --- a/.github/workflows/_build_rust_artifacts.yml +++ b/.github/workflows/_build_rust_artifacts.yml @@ -100,7 +100,7 @@ jobs: - name: Install musl-tools if: matrix.libc == 'musl' - run: sudo apt-get update && sudo apt-get install -y musl-tools + run: ./scripts/ci/apt-install.sh musl-tools - name: Setup Rust with cache uses: ./.github/actions/utils/setup-rust-with-cache diff --git a/.github/workflows/_test_examples.yml b/.github/workflows/_test_examples.yml index 1382f9ee55..5f09558048 100644 --- a/.github/workflows/_test_examples.yml +++ b/.github/workflows/_test_examples.yml @@ -69,8 +69,7 @@ jobs: if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-php' shell: bash run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + ./scripts/ci/apt-install.sh --no-install-recommends \ clang \ libclang-dev \ libhwloc-dev \ diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index 209f40724a..8cae6d27db 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -89,8 +89,7 @@ jobs: - name: Install system dependencies run: | - sudo apt-get update --yes - sudo apt-get install --yes gnome-keyring keyutils dbus-x11 libsecret-tools + ./scripts/ci/apt-install.sh gnome-keyring keyutils dbus-x11 libsecret-tools rm -f $HOME/.local/share/keyrings/* - name: Setup Rust with cache @@ -433,7 +432,7 @@ jobs: save-cache: "false" - name: Install netcat - run: sudo apt-get update && sudo apt-get install -y netcat-openbsd + run: ./scripts/ci/apt-install.sh netcat-openbsd - name: Install dependencies run: | diff --git a/scripts/ci/apt-install.sh b/scripts/ci/apt-install.sh new file mode 100755 index 0000000000..7d5c46fc63 --- /dev/null +++ b/scripts/ci/apt-install.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: scripts/ci/apt-install.sh [apt-get install flags] ... + +Install Debian packages behind a bounded `apt-get update`. + +GitHub runners intermittently lose egress to azure.archive.ubuntu.com. apt +falls back through /etc/apt/apt-mirrors.txt and can then wedge fetching the +package indices with no output and no timeout of its own, so the job sits +dead until it burns the whole `timeout-minutes` budget. apt's own +Acquire::*::Timeout does not bound this - the observed stalls ran for half an +hour past the 120s default - so both phases are capped externally instead. + +The install runs even when every update attempt failed, so a stale but +present index set is not fatal. + +Environment: + APT_UPDATE_TIMEOUT Seconds allowed per update attempt (default: 120) + APT_UPDATE_ATTEMPTS Update attempts before giving up (default: 3) + APT_INSTALL_TIMEOUT Seconds allowed for the install (default: 600) +USAGE +} + +case "${1:-}" in + -h|--help) + usage + exit 0 + ;; + "") + usage >&2 + exit 1 + ;; +esac + +timeout_seconds="${APT_UPDATE_TIMEOUT:-120}" +attempts="${APT_UPDATE_ATTEMPTS:-3}" +install_timeout="${APT_INSTALL_TIMEOUT:-600}" + +for attempt in $(seq 1 "${attempts}"); do + if sudo timeout --kill-after=10 "${timeout_seconds}" apt-get \ + -o Acquire::Retries=2 \ + -o Acquire::http::Timeout=15 \ + -o Acquire::https::Timeout=15 \ + update; then + break + fi + echo "::warning::apt-get update attempt ${attempt}/${attempts} timed out or failed" + if [ "${attempt}" -lt "${attempts}" ]; then + sleep 5 + fi +done + +# Generous cap: a healthy install of the heaviest package set here takes well +# under a minute, and killing dpkg mid-configure leaves a broken package DB. +if ! sudo timeout --kill-after=10 "${install_timeout}" apt-get install -y "$@"; then + echo "::error::apt-get install exceeded ${install_timeout}s or failed" + exit 1 +fi