From 1115d9956d35d641f9d2161bf606b1276bfac6e5 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 09:51:21 +0000 Subject: [PATCH 01/34] write library version to /system-tests-library-version in install scripts --- utils/build/docker/cpp_httpd/install_ddtrace.sh | 1 + utils/build/docker/cpp_kong/install_ddtrace.sh | 1 + utils/build/docker/cpp_nginx/install_ddtrace.sh | 1 + utils/build/docker/dotnet/install_ddtrace.sh | 2 ++ utils/build/docker/golang/install_ddtrace.sh | 1 + utils/build/docker/java/install_ddtrace.sh | 2 ++ utils/build/docker/nodejs/install_ddtrace.sh | 2 ++ utils/build/docker/php/common/install_ddtrace.sh | 2 ++ utils/build/docker/python/install_ddtrace.sh | 2 ++ utils/build/docker/ruby/install_ddtrace.sh | 3 +++ 10 files changed, 17 insertions(+) diff --git a/utils/build/docker/cpp_httpd/install_ddtrace.sh b/utils/build/docker/cpp_httpd/install_ddtrace.sh index dece759f857..cf1c8d67026 100755 --- a/utils/build/docker/cpp_httpd/install_ddtrace.sh +++ b/utils/build/docker/cpp_httpd/install_ddtrace.sh @@ -25,5 +25,6 @@ fi echo '{"status": "ok", "library": {"name": "cpp_httpd", "version": "'"$HTTPD_DATADOG_VERSION"'"}}' > /app/healthcheck.json echo "$HTTPD_DATADOG_VERSION" > SYSTEM_TESTS_LIBRARY_VERSION +echo "${HTTPD_DATADOG_VERSION#v}" > /system-tests-library-version cat /app/healthcheck.json diff --git a/utils/build/docker/cpp_kong/install_ddtrace.sh b/utils/build/docker/cpp_kong/install_ddtrace.sh index 11ef895c154..1ce4c60dcb1 100755 --- a/utils/build/docker/cpp_kong/install_ddtrace.sh +++ b/utils/build/docker/cpp_kong/install_ddtrace.sh @@ -122,6 +122,7 @@ if [ "$KONG_IS_RELEASE" = "false" ]; then fi echo "${PLUGIN_VERSION}" > /builds/SYSTEM_TESTS_LIBRARY_VERSION +echo "${PLUGIN_VERSION}" > /system-tests-library-version printf '{"status":"ok","library":{"name":"cpp_kong","version":"%s"}}' \ "$PLUGIN_VERSION" > /builds/healthcheck.json diff --git a/utils/build/docker/cpp_nginx/install_ddtrace.sh b/utils/build/docker/cpp_nginx/install_ddtrace.sh index 078afb5c30a..a1aa443a1ab 100755 --- a/utils/build/docker/cpp_nginx/install_ddtrace.sh +++ b/utils/build/docker/cpp_nginx/install_ddtrace.sh @@ -28,6 +28,7 @@ function epilogue { echo "DataDog/nginx-datadog version: ${module_version}" echo "$module_version" | tr -d v > SYSTEM_TESTS_LIBRARY_VERSION + echo "$module_version" | tr -d v > /system-tests-library-version rm -f /etc/nginx/nginx.conf if version_first_is_greater "$module_version" "v1.1.0"; then diff --git a/utils/build/docker/dotnet/install_ddtrace.sh b/utils/build/docker/dotnet/install_ddtrace.sh index f785cf445cd..d66538aeb8c 100755 --- a/utils/build/docker/dotnet/install_ddtrace.sh +++ b/utils/build/docker/dotnet/install_ddtrace.sh @@ -28,6 +28,7 @@ if [ $(ls /binaries/Datadog.Trace.ClrProfiler.Native.so | wc -l) = 1 ]; then else if [ $(ls datadog-dotnet-apm*.tar.gz | wc -l) = 1 ]; then echo "Install ddtrace from $(ls datadog-dotnet-apm*.tar.gz)" + DDTRACE_VERSION=$(ls datadog-dotnet-apm*.tar.gz | grep -oP '\d+\.\d+\.\d+' | head -1) else echo "Install ddtrace from github releases" if ! DDTRACE_VERSION="$(get_latest_release DataDog/dd-trace-dotnet)"; then @@ -46,4 +47,5 @@ else fi tar xzf $(ls datadog-dotnet-apm*.tar.gz) -C /opt/datadog + echo "$DDTRACE_VERSION" > /system-tests-library-version fi diff --git a/utils/build/docker/golang/install_ddtrace.sh b/utils/build/docker/golang/install_ddtrace.sh index a5520203713..d013246ddf7 100755 --- a/utils/build/docker/golang/install_ddtrace.sh +++ b/utils/build/docker/golang/install_ddtrace.sh @@ -53,6 +53,7 @@ go mod tidy lib_mod_dir=$(go list -f '{{.Dir}}' -m github.com/DataDog/dd-trace-go/v2) version=$(sed -nrE 's#.*"v(.*)".*#\1#p' "${lib_mod_dir}/internal/version/version.go") # Parse the version string content "v.*" echo "${version}" > SYSTEM_TESTS_LIBRARY_VERSION +echo "${version}" > /system-tests-library-version # Output the version of dd-trace-go (per go.mod, as well as the built-in tag). echo "dd-trace-go go.mod version: $(go list -f '{{ .Version }}' -m github.com/DataDog/dd-trace-go/v2)" diff --git a/utils/build/docker/java/install_ddtrace.sh b/utils/build/docker/java/install_ddtrace.sh index 20d67c955ac..30eeb608ee1 100755 --- a/utils/build/docker/java/install_ddtrace.sh +++ b/utils/build/docker/java/install_ddtrace.sh @@ -55,3 +55,5 @@ SYSTEM_TESTS_LIBRARY_VERSION=$(cat /binaries/SYSTEM_TESTS_LIBRARY_VERSION) echo "dd-trace version: $(cat /binaries/SYSTEM_TESTS_LIBRARY_VERSION)" +cp /binaries/SYSTEM_TESTS_LIBRARY_VERSION /system-tests-library-version + diff --git a/utils/build/docker/nodejs/install_ddtrace.sh b/utils/build/docker/nodejs/install_ddtrace.sh index 057f0a0b373..e9fbc6201b1 100755 --- a/utils/build/docker/nodejs/install_ddtrace.sh +++ b/utils/build/docker/nodejs/install_ddtrace.sh @@ -35,4 +35,6 @@ else echo "install from NPM" npm install "$target" || npm install "$target" fi + + node -e "process.stdout.write(require('dd-trace/package.json').version)" > /system-tests-library-version fi diff --git a/utils/build/docker/php/common/install_ddtrace.sh b/utils/build/docker/php/common/install_ddtrace.sh index a87bdcc02cb..bfe7389547a 100755 --- a/utils/build/docker/php/common/install_ddtrace.sh +++ b/utils/build/docker/php/common/install_ddtrace.sh @@ -150,6 +150,8 @@ fi php -d error_reporting='' -d extension=ddtrace.so -d extension=ddappsec.so -r 'echo phpversion("ddtrace");' > \ /binaries/SYSTEM_TESTS_LIBRARY_VERSION +cp /binaries/SYSTEM_TESTS_LIBRARY_VERSION /system-tests-library-version + find /opt -name ddappsec-helper -exec ln -s '{}' /usr/local/bin/ \; mkdir -p /etc/dd-appsec diff --git a/utils/build/docker/python/install_ddtrace.sh b/utils/build/docker/python/install_ddtrace.sh index 97e3994e323..b28bac8f918 100755 --- a/utils/build/docker/python/install_ddtrace.sh +++ b/utils/build/docker/python/install_ddtrace.sh @@ -38,3 +38,5 @@ else echo "ERROR: Found several usable wheel files in binaries/, abort." exit 1 fi + +python -c "import ddtrace; print(ddtrace.__version__, end='')" > /system-tests-library-version diff --git a/utils/build/docker/ruby/install_ddtrace.sh b/utils/build/docker/ruby/install_ddtrace.sh index 67e4b52a67a..5f1eede64da 100755 --- a/utils/build/docker/ruby/install_ddtrace.sh +++ b/utils/build/docker/ruby/install_ddtrace.sh @@ -67,3 +67,6 @@ fi bundle config set --local without test development bundle install + +bundle exec ruby -e "require 'datadog'; print Datadog::VERSION::STRING" > /system-tests-library-version 2>/dev/null \ + || bundle exec ruby -e "require 'ddtrace'; print DDTrace::VERSION" > /system-tests-library-version From a876bfa1c894ffd6f56edfcd7df4883e7e33d521 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 09:51:24 +0000 Subject: [PATCH 02/34] add system-tests-library-version label to weblog image after build --- utils/build/build.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/utils/build/build.sh b/utils/build/build.sh index c0efcf0c033..2d234d3004d 100755 --- a/utils/build/build.sh +++ b/utils/build/build.sh @@ -295,6 +295,15 @@ build() { $EXTRA_DOCKER_ARGS \ . + CID=$(docker create system_tests/weblog) + LIBRARY_VERSION=$(docker cp "${CID}:/system-tests-library-version" - 2>/dev/null | tar -xO || true) + docker rm "${CID}" > /dev/null + if [ -n "${LIBRARY_VERSION}" ]; then + docker build \ + --label "system-tests-library-version=${LIBRARY_VERSION}" \ + -t system_tests/weblog - <<< "FROM system_tests/weblog" + fi + if test -f "binaries/waf_rule_set.json"; then docker buildx build \ From a601baf36e61f05ddb01c65f27273cb70df9f2b6 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 09:51:27 +0000 Subject: [PATCH 03/34] read library version from image label in WeblogContainer.configure() --- utils/_context/containers.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 1b68ad32924..3360a7a1e5a 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -1014,6 +1014,10 @@ def configure(self, *, host_log_folder: str, replay: bool): library = self.image.labels["system-tests-library"] + version_from_label = self.image.labels.get("system-tests-library-version") + if version_from_label: + self._library = ComponentVersion(library, version_from_label) + header_tags = "" if library in ("cpp_nginx", "cpp_httpd", "dotnet", "java", "python"): header_tags = "user-agent:http.request.headers.user-agent" @@ -1118,11 +1122,13 @@ def set_weblog_domain_for_ipv6(self, network: Network): def post_start(self): logger.debug(f"Docker host is {weblog.domain}") - with open(self.healthcheck_log_file, encoding="utf-8") as f: - data = json.load(f) - lib = data["library"] + if self._library is None: + with open(self.healthcheck_log_file, encoding="utf-8") as f: + data = json.load(f) + lib = data["library"] - self._library = ComponentVersion(lib["name"], lib["version"]) + self._library = ComponentVersion(lib["name"], lib["version"]) + logger.warning("Library version read from healthcheck endpoint — add system-tests-library-version label to the image to speed up startup") logger.stdout(f"Library: {self.library}") From 3ddf1ba0cbe9994b221fa26e101b8ab1fb39822c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 09:51:30 +0000 Subject: [PATCH 04/34] move container startup to post-collection, skip if no tests selected --- conftest.py | 5 +++++ utils/_context/_scenarios/core.py | 10 +++++++++ utils/_context/_scenarios/endtoend.py | 32 ++++++++++++++++++--------- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/conftest.py b/conftest.py index 218eaccac4f..4dd02c12fef 100644 --- a/conftest.py +++ b/conftest.py @@ -416,6 +416,11 @@ def pytest_collection_finish(session: pytest.Session) -> None: if session.config.option.replay: setup_properties.load(context.scenario.host_log_folder) + if len(session.items) == 0: + return + + context.scenario.execute_post_collection_warmups() + last_item_file = "" for item in session.items: if _item_is_skipped(item): diff --git a/utils/_context/_scenarios/core.py b/utils/_context/_scenarios/core.py index b605c2c3a3c..a54a795afd5 100644 --- a/utils/_context/_scenarios/core.py +++ b/utils/_context/_scenarios/core.py @@ -128,6 +128,7 @@ def __init__( group.scenarios.append(self) self.warmups: list[Callable] = [] + self.post_collection_warmups: list[Callable] = [] self.collect_only: bool = False def _create_log_subfolder(self, subfolder: str, *, remove_if_exists: bool = False): @@ -196,6 +197,15 @@ def pytest_sessionstart(self, session: pytest.Session): # noqa: ARG002 self.close_targets() raise + def execute_post_collection_warmups(self): + try: + for warmup in self.post_collection_warmups: + logger.info(f"Executing post-collection warmup {warmup}") + warmup() + except: + self.close_targets() + raise + def post_setup(self, session: pytest.Session): """Called after test setup""" diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 3b276ebf344..46c335eefee 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -89,14 +89,14 @@ def configure(self, config: pytest.Config): # noqa: ARG002 if not self.replay: docker_info = get_docker_client().info() self.components["docker.Cgroup"] = docker_info.get("CgroupVersion", None) - self.warmups.append(self._create_network) - self.warmups.append(self._start_containers) + self.post_collection_warmups.append(self._create_network) + self.post_collection_warmups.append(self._start_containers) for container in reversed(self._containers): container.configure(host_log_folder=self.host_log_folder, replay=self.replay) for container in self._containers: - self.warmups.append(container.post_start) + self.post_collection_warmups.append(container.post_start) def get_container_by_dd_integration_name(self, name: str): for container in self._containers: @@ -327,11 +327,17 @@ def configure(self, config: pytest.Config): self.library_interface_timeout = self._library_interface_timeout if not self.replay: - self.warmups.insert(1, self._start_interfaces_watchdog) - self.warmups.append(self._get_weblog_system_info) - self.warmups.append(self._wait_for_app_readiness) - self.warmups.append(self._set_weblog_domain) - self.warmups.append(self._set_components) + self.post_collection_warmups.append(self._start_interfaces_watchdog) + self.post_collection_warmups.append(self._get_weblog_system_info) + self.post_collection_warmups.append(self._wait_for_app_readiness) + self.post_collection_warmups.append(self._set_weblog_domain) + + if self.weblog_container._library is not None: + self._set_library_component() + else: + self.post_collection_warmups.append(self._set_library_component) + + self.post_collection_warmups.append(self._set_agent_component) def _set_containers_dependancies(self) -> None: if self._use_proxy_for_agent: @@ -377,11 +383,17 @@ def _set_weblog_domain(self): if self.enable_ipv6: self.weblog_container.set_weblog_domain_for_ipv6(self._network) - def _set_components(self): - self.components["agent"] = self.agent_version + def _set_library_component(self): self.components["library"] = self.library.version self.components[self.library.name] = self.library.version + def _set_agent_component(self): + self.components["agent"] = self.agent_version + + def _set_components(self): + self._set_library_component() + self._set_agent_component() + def _wait_for_app_readiness(self): if self._use_proxy_for_weblog: logger.debug("Wait for app readiness") From 01ce802f3ad141abb3822a0069a367aea8f5a9f6 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 09:53:10 +0000 Subject: [PATCH 05/34] fix agent_version access before post_start, move junit props after warmups --- conftest.py | 17 +++++++++-------- utils/_context/containers.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/conftest.py b/conftest.py index 4dd02c12fef..081b2a91b0f 100644 --- a/conftest.py +++ b/conftest.py @@ -233,14 +233,6 @@ def pytest_sessionstart(session: pytest.Session) -> None: if not session.config.option.collectonly: context.scenario.pytest_sessionstart(session) - # The canonical way o adding Junit properties to testsuite is not working with xdist - # Workaround to tackle this issue - # https://github.com/pytest-dev/pytest/issues/7767#issuecomment-698560400 - xml = session.config._store.get(xml_key, None) # noqa: SLF001 - if xml: - properties = context.scenario.get_junit_properties() - for key, value in properties.items(): - xml.add_global_property(key, value or "") if session.config.option.sleep: logger.terminal.write("\n ********************************************************** \n") @@ -421,6 +413,15 @@ def pytest_collection_finish(session: pytest.Session) -> None: context.scenario.execute_post_collection_warmups() + # The canonical way of adding Junit properties to testsuite is not working with xdist + # Workaround to tackle this issue + # https://github.com/pytest-dev/pytest/issues/7767#issuecomment-698560400 + xml = session.config._store.get(xml_key, None) # noqa: SLF001 + if xml: + properties = context.scenario.get_junit_properties() + for key, value in properties.items(): + xml.add_global_property(key, value or "") + last_item_file = "" for item in session.items: if _item_is_skipped(item): diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 3360a7a1e5a..198b55eaffe 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -741,7 +741,7 @@ def post_start(self): class AgentContainer(TestedContainer): apm_receiver_port: int = 8127 dogstatsd_port: int = 8125 - agent_version: Version + agent_version: Version | None = None def __init__( self, From 1aa9d267b0e81691c6b45ee50cd65dadd4be5109 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 11:32:03 +0000 Subject: [PATCH 06/34] add timing instrumentation to warmup phases --- conftest.py | 4 ++++ utils/_context/_scenarios/core.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/conftest.py b/conftest.py index 081b2a91b0f..484a95b001d 100644 --- a/conftest.py +++ b/conftest.py @@ -232,6 +232,7 @@ def pytest_sessionstart(session: pytest.Session) -> None: # if only collect tests, do not start the scenario if not session.config.option.collectonly: context.scenario.pytest_sessionstart(session) + logger.stdout(f"[timing] collection starts at t={time.perf_counter():.2f}s") if session.config.option.sleep: @@ -282,10 +283,12 @@ def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config """Unselect items that were deactivated in the manifests or that are not included in the current scenario""" logger.debug("pytest_collection_modifyitems") + _t0 = time.perf_counter() manifest_components: dict[str, Version] = { name: version for name, version in context.scenario.components.items() if isinstance(version, Version) } manifest = Manifest(manifest_components, context.weblog_variant) + logger.stdout(f" [timing] manifest load: {time.perf_counter() - _t0:.2f}s") for item in items: assert isinstance(item, pytest.Function) declarations = manifest.get_declarations(item.nodeid) @@ -391,6 +394,7 @@ def _item_is_skipped(item: pytest.Item): def pytest_collection_finish(session: pytest.Session) -> None: + logger.stdout(f"[timing] collection ends at t={time.perf_counter():.2f}s") if session.config.option.collectonly: return diff --git a/utils/_context/_scenarios/core.py b/utils/_context/_scenarios/core.py index a54a795afd5..bb19fb586f2 100644 --- a/utils/_context/_scenarios/core.py +++ b/utils/_context/_scenarios/core.py @@ -2,6 +2,7 @@ import os from pathlib import Path import shutil +import time from typing import TYPE_CHECKING @@ -189,22 +190,30 @@ def pytest_sessionstart(self, session: pytest.Session): # noqa: ARG002 logger.terminal.write_sep("=", "test context", bold=True) + _t0 = time.perf_counter() try: for warmup in self.warmups: logger.info(f"Executing warmup {warmup}") + _wt = time.perf_counter() warmup() + logger.stdout(f" [timing] {warmup.__name__}: {time.perf_counter() - _wt:.2f}s") except: self.close_targets() raise + logger.stdout(f"[timing] pre-collection warmups total: {time.perf_counter() - _t0:.2f}s") def execute_post_collection_warmups(self): + _t0 = time.perf_counter() try: for warmup in self.post_collection_warmups: logger.info(f"Executing post-collection warmup {warmup}") + _wt = time.perf_counter() warmup() + logger.stdout(f" [timing] {warmup.__name__}: {time.perf_counter() - _wt:.2f}s") except: self.close_targets() raise + logger.stdout(f"[timing] post-collection warmups total: {time.perf_counter() - _t0:.2f}s") def post_setup(self, session: pytest.Session): """Called after test setup""" From 04c38dc662c4b99384dccddcd95e3ff00540fad6 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 11:32:05 +0000 Subject: [PATCH 07/34] fix library version extraction for multi-stage Dockerfiles --- utils/build/build.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/build/build.sh b/utils/build/build.sh index 2d234d3004d..ac24f98d4fb 100755 --- a/utils/build/build.sh +++ b/utils/build/build.sh @@ -296,7 +296,11 @@ build() { . CID=$(docker create system_tests/weblog) - LIBRARY_VERSION=$(docker cp "${CID}:/system-tests-library-version" - 2>/dev/null | tar -xO || true) + LIBRARY_VERSION="" + for _path in /system-tests-library-version /app/SYSTEM_TESTS_LIBRARY_VERSION /binaries/SYSTEM_TESTS_LIBRARY_VERSION /builds/SYSTEM_TESTS_LIBRARY_VERSION /SYSTEM_TESTS_LIBRARY_VERSION; do + LIBRARY_VERSION=$(docker cp "${CID}:${_path}" - 2>/dev/null | tar -xO 2>/dev/null | tr -d '[:space:]') + [ -n "${LIBRARY_VERSION}" ] && break + done docker rm "${CID}" > /dev/null if [ -n "${LIBRARY_VERSION}" ]; then docker build \ From 979fa8bf34e7000ee6e07a432a112f4d9b339967 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 11:32:15 +0000 Subject: [PATCH 08/34] skip container startup when no tests selected Read agent version from image label in AgentContainer.configure(). When both library and agent versions are known pre-collection, defer container startup to post-collection warmups so containers are never started when no tests are selected for the scenario. --- utils/_context/_scenarios/endtoend.py | 30 ++++++++++++++++++++------- utils/_context/containers.py | 6 ++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 46c335eefee..c17301998f2 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -89,14 +89,14 @@ def configure(self, config: pytest.Config): # noqa: ARG002 if not self.replay: docker_info = get_docker_client().info() self.components["docker.Cgroup"] = docker_info.get("CgroupVersion", None) - self.post_collection_warmups.append(self._create_network) - self.post_collection_warmups.append(self._start_containers) + self.warmups.append(self._create_network) + self.warmups.append(self._start_containers) for container in reversed(self._containers): container.configure(host_log_folder=self.host_log_folder, replay=self.replay) for container in self._containers: - self.post_collection_warmups.append(container.post_start) + self.warmups.append(container.post_start) def get_container_by_dd_integration_name(self, name: str): for container in self._containers: @@ -332,12 +332,28 @@ def configure(self, config: pytest.Config): self.post_collection_warmups.append(self._wait_for_app_readiness) self.post_collection_warmups.append(self._set_weblog_domain) - if self.weblog_container._library is not None: + if ( + not self.replay + and self.weblog_container._library is not None + and self.agent_container.agent_version is not None + ): + # Both versions known from image labels: defer container startup to post-collection + # so containers are skipped entirely when no tests are selected self._set_library_component() + self._defer_container_startup() + elif self.weblog_container._library is not None: + self._set_library_component() + self.warmups.append(self._set_agent_component) else: - self.post_collection_warmups.append(self._set_library_component) - - self.post_collection_warmups.append(self._set_agent_component) + self.warmups.append(self._set_library_component) + self.warmups.append(self._set_agent_component) + + def _defer_container_startup(self): + """Move container startup warmups to post_collection_warmups (inserted before interface warmups).""" + container_warmups = [self._create_network, self._start_containers] + [c.post_start for c in self._containers] + for w in container_warmups: + self.warmups.remove(w) + self.post_collection_warmups[0:0] = container_warmups + [self._set_agent_component] def _set_containers_dependancies(self) -> None: if self._use_proxy_for_agent: diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 198b55eaffe..d42d8f03388 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -797,6 +797,12 @@ def __init__( }, ) + def configure(self, *, host_log_folder: str, replay: bool): + super().configure(host_log_folder=host_log_folder, replay=replay) + version_str = self.image.labels.get("org.opencontainers.image.version") + if version_str: + self.agent_version = ComponentVersion("agent", version_str).version + def post_start(self): with open(self.healthcheck_log_file, encoding="utf-8") as f: data = json.load(f) From 03626df3bc6d528b84fe640a589be61e85cacfdd Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 11:44:47 +0000 Subject: [PATCH 09/34] remove timing instrumentation --- conftest.py | 4 ---- utils/_context/_scenarios/core.py | 9 --------- 2 files changed, 13 deletions(-) diff --git a/conftest.py b/conftest.py index 484a95b001d..081b2a91b0f 100644 --- a/conftest.py +++ b/conftest.py @@ -232,7 +232,6 @@ def pytest_sessionstart(session: pytest.Session) -> None: # if only collect tests, do not start the scenario if not session.config.option.collectonly: context.scenario.pytest_sessionstart(session) - logger.stdout(f"[timing] collection starts at t={time.perf_counter():.2f}s") if session.config.option.sleep: @@ -283,12 +282,10 @@ def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config """Unselect items that were deactivated in the manifests or that are not included in the current scenario""" logger.debug("pytest_collection_modifyitems") - _t0 = time.perf_counter() manifest_components: dict[str, Version] = { name: version for name, version in context.scenario.components.items() if isinstance(version, Version) } manifest = Manifest(manifest_components, context.weblog_variant) - logger.stdout(f" [timing] manifest load: {time.perf_counter() - _t0:.2f}s") for item in items: assert isinstance(item, pytest.Function) declarations = manifest.get_declarations(item.nodeid) @@ -394,7 +391,6 @@ def _item_is_skipped(item: pytest.Item): def pytest_collection_finish(session: pytest.Session) -> None: - logger.stdout(f"[timing] collection ends at t={time.perf_counter():.2f}s") if session.config.option.collectonly: return diff --git a/utils/_context/_scenarios/core.py b/utils/_context/_scenarios/core.py index bb19fb586f2..a54a795afd5 100644 --- a/utils/_context/_scenarios/core.py +++ b/utils/_context/_scenarios/core.py @@ -2,7 +2,6 @@ import os from pathlib import Path import shutil -import time from typing import TYPE_CHECKING @@ -190,30 +189,22 @@ def pytest_sessionstart(self, session: pytest.Session): # noqa: ARG002 logger.terminal.write_sep("=", "test context", bold=True) - _t0 = time.perf_counter() try: for warmup in self.warmups: logger.info(f"Executing warmup {warmup}") - _wt = time.perf_counter() warmup() - logger.stdout(f" [timing] {warmup.__name__}: {time.perf_counter() - _wt:.2f}s") except: self.close_targets() raise - logger.stdout(f"[timing] pre-collection warmups total: {time.perf_counter() - _t0:.2f}s") def execute_post_collection_warmups(self): - _t0 = time.perf_counter() try: for warmup in self.post_collection_warmups: logger.info(f"Executing post-collection warmup {warmup}") - _wt = time.perf_counter() warmup() - logger.stdout(f" [timing] {warmup.__name__}: {time.perf_counter() - _wt:.2f}s") except: self.close_targets() raise - logger.stdout(f"[timing] post-collection warmups total: {time.perf_counter() - _t0:.2f}s") def post_setup(self, session: pytest.Session): """Called after test setup""" From 61747c43fdc1b5265551abab761925cdb32a19d8 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 11:50:20 +0000 Subject: [PATCH 10/34] log library version in pre-collection warmup when known from label --- utils/_context/_scenarios/endtoend.py | 10 ++++++++++ utils/_context/containers.py | 13 ++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index c17301998f2..7ac60ad73a4 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -340,9 +340,11 @@ def configure(self, config: pytest.Config): # Both versions known from image labels: defer container startup to post-collection # so containers are skipped entirely when no tests are selected self._set_library_component() + self.warmups.append(self._log_weblog_info) self._defer_container_startup() elif self.weblog_container._library is not None: self._set_library_component() + self.warmups.append(self._log_weblog_info) self.warmups.append(self._set_agent_component) else: self.warmups.append(self._set_library_component) @@ -399,6 +401,14 @@ def _set_weblog_domain(self): if self.enable_ipv6: self.weblog_container.set_weblog_domain_for_ipv6(self._network) + def _log_weblog_info(self): + logger.stdout(f"Library: {self.library}") + if self.weblog_container.appsec_rules_file: + logger.stdout("Using a custom appsec rules file") + if self.weblog_container.uds_mode: + logger.stdout(f"UDS socket: {self.weblog_container.uds_socket}") + logger.stdout(f"Weblog variant: {self.weblog_variant}") + def _set_library_component(self): self.components["library"] = self.library.version self.components[self.library.name] = self.library.version diff --git a/utils/_context/containers.py b/utils/_context/containers.py index d42d8f03388..b7af18b6ae6 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -1135,16 +1135,15 @@ def post_start(self): self._library = ComponentVersion(lib["name"], lib["version"]) logger.warning("Library version read from healthcheck endpoint — add system-tests-library-version label to the image to speed up startup") + logger.stdout(f"Library: {self.library}") - logger.stdout(f"Library: {self.library}") - - if self.appsec_rules_file: - logger.stdout("Using a custom appsec rules file") + if self.appsec_rules_file: + logger.stdout("Using a custom appsec rules file") - if self.uds_mode: - logger.stdout(f"UDS socket: {self.uds_socket}") + if self.uds_mode: + logger.stdout(f"UDS socket: {self.uds_socket}") - logger.stdout(f"Weblog variant: {self.weblog_variant}") + logger.stdout(f"Weblog variant: {self.weblog_variant}") self.stdout_interface.init_patterns(self.library) From 91f92d257f5438d1c57b0265946901731b96a9e0 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 12:02:11 +0000 Subject: [PATCH 11/34] restore stdout format: log context info from labels in pre-collection warmup --- utils/_context/_scenarios/endtoend.py | 10 +++++++++- utils/_context/containers.py | 6 ++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 7ac60ad73a4..dc8c485692a 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -89,6 +89,7 @@ def configure(self, config: pytest.Config): # noqa: ARG002 if not self.replay: docker_info = get_docker_client().info() self.components["docker.Cgroup"] = docker_info.get("CgroupVersion", None) + self.warmups.append(self._log_starting_containers) self.warmups.append(self._create_network) self.warmups.append(self._start_containers) @@ -128,6 +129,9 @@ def _ingest(self, event: FileSystemEvent): observer.start() + def _log_starting_containers(self): + logger.stdout("Starting containers...") + def _create_network(self) -> None: name = "system-tests-ipv6" if self.enable_ipv6 else "system-tests-ipv4" @@ -153,7 +157,6 @@ def _create_network(self) -> None: self._network = get_docker_client().networks.create(name, check_duplicate=True) def _start_containers(self): - logger.stdout("Starting containers...") threads = [] for container in self._containers: @@ -340,6 +343,7 @@ def configure(self, config: pytest.Config): # Both versions known from image labels: defer container startup to post-collection # so containers are skipped entirely when no tests are selected self._set_library_component() + self.warmups.append(self._log_agent_info) self.warmups.append(self._log_weblog_info) self._defer_container_startup() elif self.weblog_container._library is not None: @@ -401,6 +405,10 @@ def _set_weblog_domain(self): if self.enable_ipv6: self.weblog_container.set_weblog_domain_for_ipv6(self._network) + def _log_agent_info(self): + logger.stdout(f"Agent: {self.agent_container.agent_version}") + logger.stdout(f"Backend: {self.agent_container.dd_site}") + def _log_weblog_info(self): logger.stdout(f"Library: {self.library}") if self.weblog_container.appsec_rules_file: diff --git a/utils/_context/containers.py b/utils/_context/containers.py index b7af18b6ae6..5c0e561f4cd 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -804,13 +804,15 @@ def configure(self, *, host_log_folder: str, replay: bool): self.agent_version = ComponentVersion("agent", version_str).version def post_start(self): + already_known = self.agent_version is not None with open(self.healthcheck_log_file, encoding="utf-8") as f: data = json.load(f) self.agent_version = ComponentVersion("agent", data["version"]).version - logger.stdout(f"Agent: {self.agent_version}") - logger.stdout(f"Backend: {self.dd_site}") + if not already_known: + logger.stdout(f"Agent: {self.agent_version}") + logger.stdout(f"Backend: {self.dd_site}") @property def dd_site(self): From 5706a8e101c8527f1108387fcb26878e52da493a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 12:04:32 +0000 Subject: [PATCH 12/34] move weblog system info to test context section --- utils/_context/_scenarios/endtoend.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index dc8c485692a..3111d1b54b4 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -331,7 +331,6 @@ def configure(self, config: pytest.Config): if not self.replay: self.post_collection_warmups.append(self._start_interfaces_watchdog) - self.post_collection_warmups.append(self._get_weblog_system_info) self.post_collection_warmups.append(self._wait_for_app_readiness) self.post_collection_warmups.append(self._set_weblog_domain) @@ -350,16 +349,18 @@ def configure(self, config: pytest.Config): self._set_library_component() self.warmups.append(self._log_weblog_info) self.warmups.append(self._set_agent_component) + self.warmups.append(self._get_weblog_system_info) else: self.warmups.append(self._set_library_component) self.warmups.append(self._set_agent_component) + self.warmups.append(self._get_weblog_system_info) def _defer_container_startup(self): """Move container startup warmups to post_collection_warmups (inserted before interface warmups).""" container_warmups = [self._create_network, self._start_containers] + [c.post_start for c in self._containers] for w in container_warmups: self.warmups.remove(w) - self.post_collection_warmups[0:0] = container_warmups + [self._set_agent_component] + self.post_collection_warmups[0:0] = container_warmups + [self._set_agent_component, self._get_weblog_system_info] def _set_containers_dependancies(self) -> None: if self._use_proxy_for_agent: From c9be9f0a0e66eb3871db24085564f79cb2467f88 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 12:11:31 +0000 Subject: [PATCH 13/34] add docs/adr/ with process setup and ADR-002 for post-collection container startup --- docs/adr/000-template.md | 77 +++++++++++ docs/adr/001-adr-process-adoption.md | 122 ++++++++++++++++++ .../adr/002-skip-empty-scenario-containers.md | 105 +++++++++++++++ docs/adr/README.md | 49 +++++++ docs/adr/empty-scenario-overhead-analysis.md | 63 +++++++++ 5 files changed, 416 insertions(+) create mode 100644 docs/adr/000-template.md create mode 100644 docs/adr/001-adr-process-adoption.md create mode 100644 docs/adr/002-skip-empty-scenario-containers.md create mode 100644 docs/adr/README.md create mode 100644 docs/adr/empty-scenario-overhead-analysis.md diff --git a/docs/adr/000-template.md b/docs/adr/000-template.md new file mode 100644 index 00000000000..1730a14a42b --- /dev/null +++ b/docs/adr/000-template.md @@ -0,0 +1,77 @@ +# ADR-NNN: Title + +## Metadata + +- **Status**: [Proposed | Accepted | Deprecated | Superseded] +- **Date**: YYYY-MM-DD +- **Tags**: [e.g., scenarios, manifests, weblogs, ci, framework] +- **Affected areas**: [e.g., test activation, end-to-end scenarios, parametric tests] +- **Authors**: [Names] + +## Context + +What is the issue that we're seeing that is motivating this decision or change? + +## Decision + +What is the change that we're proposing and/or doing? + +## Consequences + +What becomes easier or more difficult to do because of this change? + +### Positive + +- ... + +### Negative + +- ... + +### Risks + +- ... + +## Decision Log + +This section tracks how this ADR evolves as we learn more and make new decisions. +Each entry should include the date, what was decided, and why. + +| Date | Decision | Rationale | +|------|----------|-----------| +| YYYY-MM-DD | Initial ADR created | [Why this decision was made] | + +### How to Add Entries + +When making an architectural decision related to this ADR: + +1. Add a new row to the table above with today's date +2. Briefly describe the decision made +3. Explain the rationale (why this choice over alternatives) +4. If the decision significantly changes the original ADR, update the relevant sections above + +Examples of decisions to log: + +- Changed implementation approach based on learnings +- Added constraints or requirements discovered during implementation +- Chose between alternative approaches during development +- Modified scope based on technical discoveries +- Deferred or rejected features with reasoning + +## Implementation Status + +### Implemented + +| Component | Location | Status | +|-----------|----------|--------| +| ... | ... | ... | + +### Not Yet Implemented + +| Component | Notes | +|-----------|-------| +| ... | ... | + +## References + +- ... diff --git a/docs/adr/001-adr-process-adoption.md b/docs/adr/001-adr-process-adoption.md new file mode 100644 index 00000000000..c3f2122c245 --- /dev/null +++ b/docs/adr/001-adr-process-adoption.md @@ -0,0 +1,122 @@ +# ADR-001: ADR Process Adoption + +## Metadata + +- **Status**: Accepted +- **Date**: 2026-02-12 +- **Tags**: `process`, `documentation` +- **Affected areas**: all -- this ADR defines how architectural decisions are recorded +- **Authors**: system-tests maintainers + +## Context + +Design decisions in system-tests were recorded in two separate locations with +inconsistent formats: + +- `docs/RFCs/` contained one RFC (manifest files) and a minimal README. +- `docs/internals/RFC/DEPRECATED/` contained a deprecated RFC (feature coverage). +- `docs/internals/revamp-asynchronous-model.md` was a design decision document + that lived alongside operational docs. + +This made it hard to discover past decisions, understand their status, or follow +a consistent process when proposing new ones. + +[Architecture Decision Records](https://adr.github.io/) (ADRs) are a +well-established practice for capturing significant design decisions alongside +their context, consequences, and evolution over time. The +[quickhouse](https://github.com/DataDog/quickhouse) repository uses a mature ADR +model that includes numbered records, a template, a knowledge-map index, and +additional mechanisms for tracking gaps, deviations, supplements, and +architecture evolution. + +## Decision + +Adopt an ADR process for system-tests, modeled on quickhouse but **simplified** +to match the needs of a testing framework. + +### What we adopt + +| Element | Description | +|---------|-------------| +| Numbered ADRs | `docs/adr/NNN-slug.md`, one file per decision | +| Template | `docs/adr/000-template.md` with metadata, context, decision, consequences, decision log, implementation status, and references | +| README index | `docs/adr/README.md` with a knowledge map and master table | +| Status definitions | Proposed, Accepted, Deprecated, Superseded | + +### What we skip (for now) + +| Element | Why it is deferred | +|---------|--------------------| +| `gaps/` | Gaps track design limitations discovered in production. System-tests does not run a production service; gaps in test coverage are tracked differently (manifests, features dashboard). | +| `deviations/` | Deviations document intentional divergences from ADR intent during implementation. System-tests has fewer moving parts and shorter feedback loops, making deviations easier to capture in the decision log of each ADR. | +| `supplements/` | Supplements hold detailed implementation plans and load-test runbooks. System-tests design decisions are smaller in scope and do not require separate supplement documents. | +| `EVOLUTION.md` | The evolution document ties gaps, deviations, and characteristics together. Without those sub-directories, the evolution document has no content to link to. | + +Any of these can be introduced later by creating a new ADR that extends this +process. + +### Migration of existing documents + +Existing RFCs and design decision documents are **copied** into ADR format. +The original files remain in place to preserve existing links and history. + +| Original location | ADR | +|--------------------|-----| +| `docs/RFCs/manifest.md` | [ADR-002](002-manifest-files.md) | +| `docs/internals/revamp-asynchronous-model.md` | [ADR-003](003-setup-test-split.md) | +| `docs/internals/RFC/DEPRECATED/RFC-feature-coverage.md` | [ADR-004](004-feature-coverage.md) | + +## Consequences + +### Positive + +- All design decisions are discoverable from a single index. +- New decisions follow a consistent template, making them easier to write and + review. +- The decision log section in each ADR captures how decisions evolve without + requiring a new document. + +### Negative + +- Existing RFCs are now duplicated (original files kept, ADR copies created). + This may cause drift if someone edits only one copy. +- Contributors must learn the ADR format, adding a small step to the process. + +### Risks + +- If the ADR process is not adopted by contributors, the directory becomes + stale. Mitigated by linking the ADR index from the main documentation README + and the contributing section. + +## Decision Log + +| Date | Decision | Rationale | +|------|----------|-----------| +| 2026-02-12 | Adopted simplified ADR model | Full quickhouse model (gaps, deviations, supplements, EVOLUTION.md) is designed for a database engine under active architecture evolution. System-tests is a testing framework with fewer architectural pivots; the simplified model covers current needs without unnecessary overhead. | +| 2026-02-12 | Keep original RFC files in place | Preserves existing links and git history. ADR copies serve as the canonical format going forward. | + +## Implementation Status + +### Implemented + +| Component | Location | Status | +|-----------|----------|--------| +| ADR template | `docs/adr/000-template.md` | Done | +| ADR-001 (this document) | `docs/adr/001-adr-process-adoption.md` | Done | +| ADR-002 (manifest files) | `docs/adr/002-manifest-files.md` | Done | +| ADR-003 (setup/test split) | `docs/adr/003-setup-test-split.md` | Done | +| ADR-004 (feature coverage) | `docs/adr/004-feature-coverage.md` | Done | +| ADR index | `docs/adr/README.md` | Done | + +### Not Yet Implemented + +| Component | Notes | +|-----------|-------| +| gaps/, deviations/, supplements/ | Deferred -- see decision above | +| EVOLUTION.md | Deferred -- see decision above | + +## References + +- [ADR homepage](https://adr.github.io/) +- [quickhouse ADR directory](https://github.com/DataDog/quickhouse/tree/main/docs/adr) -- the model this process is based on +- [Documenting Architecture Decisions](https://cognitect.com/blog/2011/11/15/documenting-architecture-decisions) -- Michael Nygard's original blog post diff --git a/docs/adr/002-skip-empty-scenario-containers.md b/docs/adr/002-skip-empty-scenario-containers.md new file mode 100644 index 00000000000..0c8737bf444 --- /dev/null +++ b/docs/adr/002-skip-empty-scenario-containers.md @@ -0,0 +1,105 @@ +# ADR-002: Skip Container Startup for Empty Scenarios + +## Metadata + +- **Status**: Accepted +- **Date**: 2026-04-15 +- **Tags**: `scenarios`, `containers`, `ci`, `framework` +- **Affected areas**: end-to-end scenarios, container lifecycle, pytest collection +- **Authors**: nccatoni + +## Context + +In system-tests CI, a scenario is "empty" when every collected test is deselected +(marked `missing_feature`, `irrelevant`, `bug`, or `skip`) for a given library/weblog +combination. The `--skip-empty-scenario` flag (used in `ci.yml`) exits gracefully +instead of failing, but Docker infrastructure was still started and torn down for every +empty scenario. + +Detailed timing analysis showed this wasted roughly **17 hours of compute per full CI +run** (~38% of scenario invocations are empty), with PHP alone accounting for 8.8h. See +[empty-scenario-overhead-analysis.md](./empty-scenario-overhead-analysis.md) for the +full breakdown. + +The root cause was the pytest hook order: `pytest_sessionstart` executes warmups +(which start containers) **before** collection and deselection happen, so there was no +point in the lifecycle where we knew "zero tests will run" before containers started. + +## Decision + +Move container startup from `pytest_sessionstart` (pre-collection) to a new +post-collection warmup phase that runs inside `pytest_collection_finish`. Combined with +reading library and agent versions from Docker image labels (rather than from running +containers), this enables: + +1. **Version info in test context** — `Agent:`, `Library:`, `Weblog variant:` lines + appear in the `=== test context ===` section even when containers never start. +2. **Zero-cost empty scenarios** — `pytest_collection_finish` returns early when + `len(session.items) == 0`, so containers are never created for empty scenarios. + +The implementation uses a three-way branch in `EndToEndScenario.configure()`: + +- **Defer path** (both versions known from image labels, not replay): container startup + and all dependent steps move to `post_collection_warmups`. Selected in the fast path. +- **Fallback path** (library version from label, agent version unknown): containers + still start in `pytest_sessionstart`; agent version read from healthcheck. +- **Legacy path** (no label data): original behaviour, both versions read from running + containers. + +## Consequences + +### Positive + +- Empty scenarios complete in ~2.5s instead of 20-40s. +- ~17h of CI compute saved per full CI run. +- The `=== test context ===` block is populated before "test session starts" in all + paths, making logs consistent and easier to grep. + +### Negative + +- For the defer path, `Weblog system:` appears after `=== test session starts ===` + rather than in `=== test context ===`. This is a minor cosmetic change. +- Scenarios where the weblog image was built without the `system-tests-library-version` + label (older images, custom builds) fall back to the legacy path and do not get the + speedup. + +### Risks + +- Edge cases not fully tested at time of writing: replay mode, `include_agent=False` + scenarios, scenarios with buddy containers, OTel scenarios. Code paths exist for all + of these (they hit the fallback or legacy path), but integration testing is pending. + +## Decision Log + +| Date | Decision | Rationale | +|------|----------|-----------| +| 2026-04-15 | Created ADR, implementation accepted | Feature functionally complete; delivers measurable CI savings. Edge-case testing deferred to follow-up. | + +## Implementation Status + +### Implemented + +| Component | Location | Status | +|-----------|----------|--------| +| Library version label in image | `utils/build/build.sh:296` | Done | +| Write version in install scripts | `utils/build/docker/*/install_ddtrace.sh` | Done | +| `WeblogContainer.configure()` reads label | `utils/_context/containers.py:1022` | Done | +| `AgentContainer.configure()` reads label | `utils/_context/containers.py:800` | Done | +| `post_collection_warmups` + `execute_post_collection_warmups()` | `utils/_context/_scenarios/core.py:131` | Done | +| Three-way branch + `_defer_container_startup()` | `utils/_context/_scenarios/endtoend.py:337` | Done | +| Early return in `pytest_collection_finish` | `conftest.py:411` | Done | + +### Not Yet Implemented + +| Component | Notes | +|-----------|-------| +| Integration tests for edge cases | replay mode, buddy containers, OTel, `include_agent=False` | + +## References + +- [empty-scenario-overhead-analysis.md](./empty-scenario-overhead-analysis.md) — timing data and root-cause analysis +- `conftest.py` — `pytest_collection_finish` hook +- `utils/_context/_scenarios/core.py` — `Scenario.post_collection_warmups` +- `utils/_context/_scenarios/endtoend.py` — `EndToEndScenario.configure`, `_defer_container_startup` +- `utils/_context/containers.py` — `AgentContainer.configure`, `WeblogContainer.configure` +- `docs/execute/skip-empty-scenario.md` — `--skip-empty-scenario` documentation diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 00000000000..cbdd9f1ea7e --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,49 @@ +# Architecture Decision Records (ADR) Index + +This directory captures significant design decisions for system-tests. +Each ADR records the context, decision, consequences, and evolution of an +architectural choice. See [ADR-001](./001-adr-process-adoption.md) for why and +how this process was adopted. + +## Knowledge Map + +### Process + +- **[ADR-001](./001-adr-process-adoption.md)**: Why system-tests uses ADRs and + what was adopted (and deferred) from the quickhouse model. + +### Scenarios / Container lifecycle + +- **[ADR-002](./002-skip-empty-scenario-containers.md)**: Skip container startup + entirely when no tests are selected, using image labels for version info. + +--- + +## Master Index + +| ADR | Title | Status | Tags | +|-----|-------|--------|------| +| [000](./000-template.md) | Template | - | `meta` | +| [001](./001-adr-process-adoption.md) | ADR Process Adoption | **Accepted** | `process`, `documentation` | +| [002](./002-skip-empty-scenario-containers.md) | Skip Container Startup for Empty Scenarios | **Accepted** | `scenarios`, `containers`, `ci`, `framework` | + +## Creating a New ADR + +1. Copy [000-template.md](./000-template.md) to `NNN-slug.md` where `NNN` is + the next available number. +2. Fill in the metadata, context, decision, and consequences sections. +3. Set status to **Proposed**. +4. Open a PR for review. Once approved, change status to **Accepted**. +5. Add an entry to the Master Index table above. + +## Status Definitions + +- **Proposed**: Under discussion, awaiting review. +- **Accepted**: Approved plan of record. Implementation should follow this. +- **Deprecated**: No longer applies. Kept for historical context. +- **Superseded**: Replaced by a newer ADR (link to the replacement). + +## See Also + +- [Documentation index](../README.md) +- Internal design docs: [docs/internals/](../internals/README.md) diff --git a/docs/adr/empty-scenario-overhead-analysis.md b/docs/adr/empty-scenario-overhead-analysis.md new file mode 100644 index 00000000000..f0efee2a627 --- /dev/null +++ b/docs/adr/empty-scenario-overhead-analysis.md @@ -0,0 +1,63 @@ +# Empty Scenario Overhead Analysis + +## Context + +In system-tests CI, scenarios are "empty" when all their collected tests are marked as +`missing_feature`, `irrelevant`, `bug` (xfail), or `skip` for a given library. The +`skip_empty_scenarios` flag (enabled in `ci.yml`) detects this and exits gracefully +instead of failing. However, Docker infrastructure is still started and torn down for +each empty scenario, wasting compute time. + +## Root cause: containers start before empty detection + +The pytest hook order in the scenario lifecycle means containers start **before** the +empty check runs: + +``` +1. pytest_configure → scenario.configure() adds _start_containers to warmups +2. pytest_sessionstart → executes warmups → CONTAINERS START (20-40s overhead) +3. pytest_collection → tests collected +4. collection_modifyitems → empty scenario detected, all tests deselected +5. pytest_sessionfinish → containers torn down, exit code changed to OK +``` + +Every empty scenario pays the full Docker infrastructure cost (container start + +health check + teardown) even though zero tests run. + +## Estimated time savings if empty scenarios had 0 cost + +Analysis based on `utils/scripts/ci_orchestrators/time-stats.json`, using per-library +infrastructure overhead thresholds to identify likely-empty scenario runs. + +| Library | Threshold | Total runs | Empty runs | % Empty | Time wasted | Per-job impact | +| ---------- | --------- | ---------- | ---------- | ------- | ----------- | -------------- | +| **PHP** | <40s | 1561 | ~879 | 56% | **8.8h** | ~21 min/job | +| **Ruby** | <30s | 1346 | ~612 | 45% | **4.5h** | ~1.5 min/job | +| **Golang** | <28s | 396 | ~205 | 52% | **1.4h** | ~14 min/job | +| **Python** | <35s | 454 | ~141 | 31% | **1.3h** | ~1.6 min/job | +| **Nodejs** | <30s | 330 | ~138 | 42% | **1.0h** | ~2.5 min/job | +| **Java** | <40s | 975 | ~16 | 1.6% | **0.1h** | negligible | +| **Dotnet** | <40s | 131 | ~4 | 3.1% | negligible | negligible | +| **TOTAL** | | **5194** | **~1995** | **38%** | **~17h** | | + +### Key takeaways + +- **~17 hours of compute time per full CI run** could be saved (~17% of ~100h total + end-to-end compute). +- **~14-21 minutes wall-clock** on the critical path (PHP and Golang jobs). +- **PHP is the biggest offender**: 24 weblogs x 35 empty scenarios x 36s each = 8.8 + hours wasted on container start/stop for nothing. + +## Potential fix + +Move the empty-scenario check **before** `pytest_sessionstart`, so containers never +start for empty scenarios. A pre-collection manifest check could determine if a +scenario has any eligible tests before spinning up infrastructure. + +## References + +- `conftest.py` — `pytest_collection_modifyitems` and `_item_must_pass` +- `utils/_context/_scenarios/core.py` — `pytest_sessionstart` warmup execution +- `utils/_context/_scenarios/endtoend.py` — `_start_containers` in warmups +- `utils/scripts/ci_orchestrators/time-stats.json` — scenario execution time data +- `docs/execute/skip-empty-scenario.md` — `--skip-empty-scenario` documentation From cdbf529adc44a8e0a4ce396507146ef243275307 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 15 Apr 2026 12:16:53 +0000 Subject: [PATCH 14/34] fix replay guard on _get_weblog_system_info, remove unused _set_components --- utils/_context/_scenarios/endtoend.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 3111d1b54b4..2017639f842 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -349,11 +349,13 @@ def configure(self, config: pytest.Config): self._set_library_component() self.warmups.append(self._log_weblog_info) self.warmups.append(self._set_agent_component) - self.warmups.append(self._get_weblog_system_info) + if not self.replay: + self.warmups.append(self._get_weblog_system_info) else: self.warmups.append(self._set_library_component) self.warmups.append(self._set_agent_component) - self.warmups.append(self._get_weblog_system_info) + if not self.replay: + self.warmups.append(self._get_weblog_system_info) def _defer_container_startup(self): """Move container startup warmups to post_collection_warmups (inserted before interface warmups).""" @@ -425,10 +427,6 @@ def _set_library_component(self): def _set_agent_component(self): self.components["agent"] = self.agent_version - def _set_components(self): - self._set_library_component() - self._set_agent_component() - def _wait_for_app_readiness(self): if self._use_proxy_for_weblog: logger.debug("Wait for app readiness") From 5411af3cfb6a02cf83d8b9c30a78223f3ffc6c46 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 16 Apr 2026 13:59:59 +0200 Subject: [PATCH 15/34] fix format and tests --- conftest.py | 17 ++++++++--------- utils/_context/_scenarios/endtoend.py | 12 ++++++++---- utils/_context/_scenarios/go_proxies.py | 3 ++- utils/_context/containers.py | 4 +++- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/conftest.py b/conftest.py index 081b2a91b0f..66a58a684de 100644 --- a/conftest.py +++ b/conftest.py @@ -233,6 +233,14 @@ def pytest_sessionstart(session: pytest.Session) -> None: if not session.config.option.collectonly: context.scenario.pytest_sessionstart(session) + # The canonical way of adding Junit properties to testsuite is not working with xdist + # Workaround to tackle this issue + # https://github.com/pytest-dev/pytest/issues/7767#issuecomment-698560400 + xml = session.config._store.get(xml_key, None) # noqa: SLF001 + if xml: + properties = context.scenario.get_junit_properties() + for key, value in properties.items(): + xml.add_global_property(key, value or "") if session.config.option.sleep: logger.terminal.write("\n ********************************************************** \n") @@ -413,15 +421,6 @@ def pytest_collection_finish(session: pytest.Session) -> None: context.scenario.execute_post_collection_warmups() - # The canonical way of adding Junit properties to testsuite is not working with xdist - # Workaround to tackle this issue - # https://github.com/pytest-dev/pytest/issues/7767#issuecomment-698560400 - xml = session.config._store.get(xml_key, None) # noqa: SLF001 - if xml: - properties = context.scenario.get_junit_properties() - for key, value in properties.items(): - xml.add_global_property(key, value or "") - last_item_file = "" for item in session.items: if _item_is_skipped(item): diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 2017639f842..ae3e3df0144 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -336,7 +336,7 @@ def configure(self, config: pytest.Config): if ( not self.replay - and self.weblog_container._library is not None + and self.weblog_container._library is not None # noqa: SLF001 and self.agent_container.agent_version is not None ): # Both versions known from image labels: defer container startup to post-collection @@ -345,7 +345,7 @@ def configure(self, config: pytest.Config): self.warmups.append(self._log_agent_info) self.warmups.append(self._log_weblog_info) self._defer_container_startup() - elif self.weblog_container._library is not None: + elif self.weblog_container._library is not None: # noqa: SLF001 self._set_library_component() self.warmups.append(self._log_weblog_info) self.warmups.append(self._set_agent_component) @@ -359,10 +359,14 @@ def configure(self, config: pytest.Config): def _defer_container_startup(self): """Move container startup warmups to post_collection_warmups (inserted before interface warmups).""" - container_warmups = [self._create_network, self._start_containers] + [c.post_start for c in self._containers] + container_warmups = [self._create_network, self._start_containers, *[c.post_start for c in self._containers]] for w in container_warmups: self.warmups.remove(w) - self.post_collection_warmups[0:0] = container_warmups + [self._set_agent_component, self._get_weblog_system_info] + self.post_collection_warmups[0:0] = [ + *container_warmups, + self._set_agent_component, + self._get_weblog_system_info, + ] def _set_containers_dependancies(self) -> None: if self._use_proxy_for_agent: diff --git a/utils/_context/_scenarios/go_proxies.py b/utils/_context/_scenarios/go_proxies.py index 4df5f66d5f9..352f14bc981 100644 --- a/utils/_context/_scenarios/go_proxies.py +++ b/utils/_context/_scenarios/go_proxies.py @@ -108,7 +108,8 @@ def _wait_for_app_readiness(self) -> None: logger.debug("Agent ready") def _set_components(self) -> None: - self.components["agent"] = self._agent_container.agent_version + if self._agent_container.agent_version is not None: + self.components["agent"] = self._agent_container.agent_version lib = self.library self.components["library"] = lib.version self.components[lib.name] = lib.version diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 5c0e561f4cd..88772827d91 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -1136,7 +1136,9 @@ def post_start(self): lib = data["library"] self._library = ComponentVersion(lib["name"], lib["version"]) - logger.warning("Library version read from healthcheck endpoint — add system-tests-library-version label to the image to speed up startup") + logger.warning( + "Library version from healthcheck — add system-tests-library-version label to speed up startup" + ) logger.stdout(f"Library: {self.library}") if self.appsec_rules_file: From fa1fd53ace26c94138b3ddc00d710308c21a3fee Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 16 Apr 2026 14:01:20 +0200 Subject: [PATCH 16/34] cleanup --- docs/adr/000-template.md | 77 ----------- docs/adr/001-adr-process-adoption.md | 122 ------------------ .../adr/002-skip-empty-scenario-containers.md | 105 --------------- docs/adr/README.md | 49 ------- docs/adr/empty-scenario-overhead-analysis.md | 63 --------- 5 files changed, 416 deletions(-) delete mode 100644 docs/adr/000-template.md delete mode 100644 docs/adr/001-adr-process-adoption.md delete mode 100644 docs/adr/002-skip-empty-scenario-containers.md delete mode 100644 docs/adr/README.md delete mode 100644 docs/adr/empty-scenario-overhead-analysis.md diff --git a/docs/adr/000-template.md b/docs/adr/000-template.md deleted file mode 100644 index 1730a14a42b..00000000000 --- a/docs/adr/000-template.md +++ /dev/null @@ -1,77 +0,0 @@ -# ADR-NNN: Title - -## Metadata - -- **Status**: [Proposed | Accepted | Deprecated | Superseded] -- **Date**: YYYY-MM-DD -- **Tags**: [e.g., scenarios, manifests, weblogs, ci, framework] -- **Affected areas**: [e.g., test activation, end-to-end scenarios, parametric tests] -- **Authors**: [Names] - -## Context - -What is the issue that we're seeing that is motivating this decision or change? - -## Decision - -What is the change that we're proposing and/or doing? - -## Consequences - -What becomes easier or more difficult to do because of this change? - -### Positive - -- ... - -### Negative - -- ... - -### Risks - -- ... - -## Decision Log - -This section tracks how this ADR evolves as we learn more and make new decisions. -Each entry should include the date, what was decided, and why. - -| Date | Decision | Rationale | -|------|----------|-----------| -| YYYY-MM-DD | Initial ADR created | [Why this decision was made] | - -### How to Add Entries - -When making an architectural decision related to this ADR: - -1. Add a new row to the table above with today's date -2. Briefly describe the decision made -3. Explain the rationale (why this choice over alternatives) -4. If the decision significantly changes the original ADR, update the relevant sections above - -Examples of decisions to log: - -- Changed implementation approach based on learnings -- Added constraints or requirements discovered during implementation -- Chose between alternative approaches during development -- Modified scope based on technical discoveries -- Deferred or rejected features with reasoning - -## Implementation Status - -### Implemented - -| Component | Location | Status | -|-----------|----------|--------| -| ... | ... | ... | - -### Not Yet Implemented - -| Component | Notes | -|-----------|-------| -| ... | ... | - -## References - -- ... diff --git a/docs/adr/001-adr-process-adoption.md b/docs/adr/001-adr-process-adoption.md deleted file mode 100644 index c3f2122c245..00000000000 --- a/docs/adr/001-adr-process-adoption.md +++ /dev/null @@ -1,122 +0,0 @@ -# ADR-001: ADR Process Adoption - -## Metadata - -- **Status**: Accepted -- **Date**: 2026-02-12 -- **Tags**: `process`, `documentation` -- **Affected areas**: all -- this ADR defines how architectural decisions are recorded -- **Authors**: system-tests maintainers - -## Context - -Design decisions in system-tests were recorded in two separate locations with -inconsistent formats: - -- `docs/RFCs/` contained one RFC (manifest files) and a minimal README. -- `docs/internals/RFC/DEPRECATED/` contained a deprecated RFC (feature coverage). -- `docs/internals/revamp-asynchronous-model.md` was a design decision document - that lived alongside operational docs. - -This made it hard to discover past decisions, understand their status, or follow -a consistent process when proposing new ones. - -[Architecture Decision Records](https://adr.github.io/) (ADRs) are a -well-established practice for capturing significant design decisions alongside -their context, consequences, and evolution over time. The -[quickhouse](https://github.com/DataDog/quickhouse) repository uses a mature ADR -model that includes numbered records, a template, a knowledge-map index, and -additional mechanisms for tracking gaps, deviations, supplements, and -architecture evolution. - -## Decision - -Adopt an ADR process for system-tests, modeled on quickhouse but **simplified** -to match the needs of a testing framework. - -### What we adopt - -| Element | Description | -|---------|-------------| -| Numbered ADRs | `docs/adr/NNN-slug.md`, one file per decision | -| Template | `docs/adr/000-template.md` with metadata, context, decision, consequences, decision log, implementation status, and references | -| README index | `docs/adr/README.md` with a knowledge map and master table | -| Status definitions | Proposed, Accepted, Deprecated, Superseded | - -### What we skip (for now) - -| Element | Why it is deferred | -|---------|--------------------| -| `gaps/` | Gaps track design limitations discovered in production. System-tests does not run a production service; gaps in test coverage are tracked differently (manifests, features dashboard). | -| `deviations/` | Deviations document intentional divergences from ADR intent during implementation. System-tests has fewer moving parts and shorter feedback loops, making deviations easier to capture in the decision log of each ADR. | -| `supplements/` | Supplements hold detailed implementation plans and load-test runbooks. System-tests design decisions are smaller in scope and do not require separate supplement documents. | -| `EVOLUTION.md` | The evolution document ties gaps, deviations, and characteristics together. Without those sub-directories, the evolution document has no content to link to. | - -Any of these can be introduced later by creating a new ADR that extends this -process. - -### Migration of existing documents - -Existing RFCs and design decision documents are **copied** into ADR format. -The original files remain in place to preserve existing links and history. - -| Original location | ADR | -|--------------------|-----| -| `docs/RFCs/manifest.md` | [ADR-002](002-manifest-files.md) | -| `docs/internals/revamp-asynchronous-model.md` | [ADR-003](003-setup-test-split.md) | -| `docs/internals/RFC/DEPRECATED/RFC-feature-coverage.md` | [ADR-004](004-feature-coverage.md) | - -## Consequences - -### Positive - -- All design decisions are discoverable from a single index. -- New decisions follow a consistent template, making them easier to write and - review. -- The decision log section in each ADR captures how decisions evolve without - requiring a new document. - -### Negative - -- Existing RFCs are now duplicated (original files kept, ADR copies created). - This may cause drift if someone edits only one copy. -- Contributors must learn the ADR format, adding a small step to the process. - -### Risks - -- If the ADR process is not adopted by contributors, the directory becomes - stale. Mitigated by linking the ADR index from the main documentation README - and the contributing section. - -## Decision Log - -| Date | Decision | Rationale | -|------|----------|-----------| -| 2026-02-12 | Adopted simplified ADR model | Full quickhouse model (gaps, deviations, supplements, EVOLUTION.md) is designed for a database engine under active architecture evolution. System-tests is a testing framework with fewer architectural pivots; the simplified model covers current needs without unnecessary overhead. | -| 2026-02-12 | Keep original RFC files in place | Preserves existing links and git history. ADR copies serve as the canonical format going forward. | - -## Implementation Status - -### Implemented - -| Component | Location | Status | -|-----------|----------|--------| -| ADR template | `docs/adr/000-template.md` | Done | -| ADR-001 (this document) | `docs/adr/001-adr-process-adoption.md` | Done | -| ADR-002 (manifest files) | `docs/adr/002-manifest-files.md` | Done | -| ADR-003 (setup/test split) | `docs/adr/003-setup-test-split.md` | Done | -| ADR-004 (feature coverage) | `docs/adr/004-feature-coverage.md` | Done | -| ADR index | `docs/adr/README.md` | Done | - -### Not Yet Implemented - -| Component | Notes | -|-----------|-------| -| gaps/, deviations/, supplements/ | Deferred -- see decision above | -| EVOLUTION.md | Deferred -- see decision above | - -## References - -- [ADR homepage](https://adr.github.io/) -- [quickhouse ADR directory](https://github.com/DataDog/quickhouse/tree/main/docs/adr) -- the model this process is based on -- [Documenting Architecture Decisions](https://cognitect.com/blog/2011/11/15/documenting-architecture-decisions) -- Michael Nygard's original blog post diff --git a/docs/adr/002-skip-empty-scenario-containers.md b/docs/adr/002-skip-empty-scenario-containers.md deleted file mode 100644 index 0c8737bf444..00000000000 --- a/docs/adr/002-skip-empty-scenario-containers.md +++ /dev/null @@ -1,105 +0,0 @@ -# ADR-002: Skip Container Startup for Empty Scenarios - -## Metadata - -- **Status**: Accepted -- **Date**: 2026-04-15 -- **Tags**: `scenarios`, `containers`, `ci`, `framework` -- **Affected areas**: end-to-end scenarios, container lifecycle, pytest collection -- **Authors**: nccatoni - -## Context - -In system-tests CI, a scenario is "empty" when every collected test is deselected -(marked `missing_feature`, `irrelevant`, `bug`, or `skip`) for a given library/weblog -combination. The `--skip-empty-scenario` flag (used in `ci.yml`) exits gracefully -instead of failing, but Docker infrastructure was still started and torn down for every -empty scenario. - -Detailed timing analysis showed this wasted roughly **17 hours of compute per full CI -run** (~38% of scenario invocations are empty), with PHP alone accounting for 8.8h. See -[empty-scenario-overhead-analysis.md](./empty-scenario-overhead-analysis.md) for the -full breakdown. - -The root cause was the pytest hook order: `pytest_sessionstart` executes warmups -(which start containers) **before** collection and deselection happen, so there was no -point in the lifecycle where we knew "zero tests will run" before containers started. - -## Decision - -Move container startup from `pytest_sessionstart` (pre-collection) to a new -post-collection warmup phase that runs inside `pytest_collection_finish`. Combined with -reading library and agent versions from Docker image labels (rather than from running -containers), this enables: - -1. **Version info in test context** — `Agent:`, `Library:`, `Weblog variant:` lines - appear in the `=== test context ===` section even when containers never start. -2. **Zero-cost empty scenarios** — `pytest_collection_finish` returns early when - `len(session.items) == 0`, so containers are never created for empty scenarios. - -The implementation uses a three-way branch in `EndToEndScenario.configure()`: - -- **Defer path** (both versions known from image labels, not replay): container startup - and all dependent steps move to `post_collection_warmups`. Selected in the fast path. -- **Fallback path** (library version from label, agent version unknown): containers - still start in `pytest_sessionstart`; agent version read from healthcheck. -- **Legacy path** (no label data): original behaviour, both versions read from running - containers. - -## Consequences - -### Positive - -- Empty scenarios complete in ~2.5s instead of 20-40s. -- ~17h of CI compute saved per full CI run. -- The `=== test context ===` block is populated before "test session starts" in all - paths, making logs consistent and easier to grep. - -### Negative - -- For the defer path, `Weblog system:` appears after `=== test session starts ===` - rather than in `=== test context ===`. This is a minor cosmetic change. -- Scenarios where the weblog image was built without the `system-tests-library-version` - label (older images, custom builds) fall back to the legacy path and do not get the - speedup. - -### Risks - -- Edge cases not fully tested at time of writing: replay mode, `include_agent=False` - scenarios, scenarios with buddy containers, OTel scenarios. Code paths exist for all - of these (they hit the fallback or legacy path), but integration testing is pending. - -## Decision Log - -| Date | Decision | Rationale | -|------|----------|-----------| -| 2026-04-15 | Created ADR, implementation accepted | Feature functionally complete; delivers measurable CI savings. Edge-case testing deferred to follow-up. | - -## Implementation Status - -### Implemented - -| Component | Location | Status | -|-----------|----------|--------| -| Library version label in image | `utils/build/build.sh:296` | Done | -| Write version in install scripts | `utils/build/docker/*/install_ddtrace.sh` | Done | -| `WeblogContainer.configure()` reads label | `utils/_context/containers.py:1022` | Done | -| `AgentContainer.configure()` reads label | `utils/_context/containers.py:800` | Done | -| `post_collection_warmups` + `execute_post_collection_warmups()` | `utils/_context/_scenarios/core.py:131` | Done | -| Three-way branch + `_defer_container_startup()` | `utils/_context/_scenarios/endtoend.py:337` | Done | -| Early return in `pytest_collection_finish` | `conftest.py:411` | Done | - -### Not Yet Implemented - -| Component | Notes | -|-----------|-------| -| Integration tests for edge cases | replay mode, buddy containers, OTel, `include_agent=False` | - -## References - -- [empty-scenario-overhead-analysis.md](./empty-scenario-overhead-analysis.md) — timing data and root-cause analysis -- `conftest.py` — `pytest_collection_finish` hook -- `utils/_context/_scenarios/core.py` — `Scenario.post_collection_warmups` -- `utils/_context/_scenarios/endtoend.py` — `EndToEndScenario.configure`, `_defer_container_startup` -- `utils/_context/containers.py` — `AgentContainer.configure`, `WeblogContainer.configure` -- `docs/execute/skip-empty-scenario.md` — `--skip-empty-scenario` documentation diff --git a/docs/adr/README.md b/docs/adr/README.md deleted file mode 100644 index cbdd9f1ea7e..00000000000 --- a/docs/adr/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Architecture Decision Records (ADR) Index - -This directory captures significant design decisions for system-tests. -Each ADR records the context, decision, consequences, and evolution of an -architectural choice. See [ADR-001](./001-adr-process-adoption.md) for why and -how this process was adopted. - -## Knowledge Map - -### Process - -- **[ADR-001](./001-adr-process-adoption.md)**: Why system-tests uses ADRs and - what was adopted (and deferred) from the quickhouse model. - -### Scenarios / Container lifecycle - -- **[ADR-002](./002-skip-empty-scenario-containers.md)**: Skip container startup - entirely when no tests are selected, using image labels for version info. - ---- - -## Master Index - -| ADR | Title | Status | Tags | -|-----|-------|--------|------| -| [000](./000-template.md) | Template | - | `meta` | -| [001](./001-adr-process-adoption.md) | ADR Process Adoption | **Accepted** | `process`, `documentation` | -| [002](./002-skip-empty-scenario-containers.md) | Skip Container Startup for Empty Scenarios | **Accepted** | `scenarios`, `containers`, `ci`, `framework` | - -## Creating a New ADR - -1. Copy [000-template.md](./000-template.md) to `NNN-slug.md` where `NNN` is - the next available number. -2. Fill in the metadata, context, decision, and consequences sections. -3. Set status to **Proposed**. -4. Open a PR for review. Once approved, change status to **Accepted**. -5. Add an entry to the Master Index table above. - -## Status Definitions - -- **Proposed**: Under discussion, awaiting review. -- **Accepted**: Approved plan of record. Implementation should follow this. -- **Deprecated**: No longer applies. Kept for historical context. -- **Superseded**: Replaced by a newer ADR (link to the replacement). - -## See Also - -- [Documentation index](../README.md) -- Internal design docs: [docs/internals/](../internals/README.md) diff --git a/docs/adr/empty-scenario-overhead-analysis.md b/docs/adr/empty-scenario-overhead-analysis.md deleted file mode 100644 index f0efee2a627..00000000000 --- a/docs/adr/empty-scenario-overhead-analysis.md +++ /dev/null @@ -1,63 +0,0 @@ -# Empty Scenario Overhead Analysis - -## Context - -In system-tests CI, scenarios are "empty" when all their collected tests are marked as -`missing_feature`, `irrelevant`, `bug` (xfail), or `skip` for a given library. The -`skip_empty_scenarios` flag (enabled in `ci.yml`) detects this and exits gracefully -instead of failing. However, Docker infrastructure is still started and torn down for -each empty scenario, wasting compute time. - -## Root cause: containers start before empty detection - -The pytest hook order in the scenario lifecycle means containers start **before** the -empty check runs: - -``` -1. pytest_configure → scenario.configure() adds _start_containers to warmups -2. pytest_sessionstart → executes warmups → CONTAINERS START (20-40s overhead) -3. pytest_collection → tests collected -4. collection_modifyitems → empty scenario detected, all tests deselected -5. pytest_sessionfinish → containers torn down, exit code changed to OK -``` - -Every empty scenario pays the full Docker infrastructure cost (container start + -health check + teardown) even though zero tests run. - -## Estimated time savings if empty scenarios had 0 cost - -Analysis based on `utils/scripts/ci_orchestrators/time-stats.json`, using per-library -infrastructure overhead thresholds to identify likely-empty scenario runs. - -| Library | Threshold | Total runs | Empty runs | % Empty | Time wasted | Per-job impact | -| ---------- | --------- | ---------- | ---------- | ------- | ----------- | -------------- | -| **PHP** | <40s | 1561 | ~879 | 56% | **8.8h** | ~21 min/job | -| **Ruby** | <30s | 1346 | ~612 | 45% | **4.5h** | ~1.5 min/job | -| **Golang** | <28s | 396 | ~205 | 52% | **1.4h** | ~14 min/job | -| **Python** | <35s | 454 | ~141 | 31% | **1.3h** | ~1.6 min/job | -| **Nodejs** | <30s | 330 | ~138 | 42% | **1.0h** | ~2.5 min/job | -| **Java** | <40s | 975 | ~16 | 1.6% | **0.1h** | negligible | -| **Dotnet** | <40s | 131 | ~4 | 3.1% | negligible | negligible | -| **TOTAL** | | **5194** | **~1995** | **38%** | **~17h** | | - -### Key takeaways - -- **~17 hours of compute time per full CI run** could be saved (~17% of ~100h total - end-to-end compute). -- **~14-21 minutes wall-clock** on the critical path (PHP and Golang jobs). -- **PHP is the biggest offender**: 24 weblogs x 35 empty scenarios x 36s each = 8.8 - hours wasted on container start/stop for nothing. - -## Potential fix - -Move the empty-scenario check **before** `pytest_sessionstart`, so containers never -start for empty scenarios. A pre-collection manifest check could determine if a -scenario has any eligible tests before spinning up infrastructure. - -## References - -- `conftest.py` — `pytest_collection_modifyitems` and `_item_must_pass` -- `utils/_context/_scenarios/core.py` — `pytest_sessionstart` warmup execution -- `utils/_context/_scenarios/endtoend.py` — `_start_containers` in warmups -- `utils/scripts/ci_orchestrators/time-stats.json` — scenario execution time data -- `docs/execute/skip-empty-scenario.md` — `--skip-empty-scenario` documentation From d994d6c9220bb6b41ee4022c3091014935fd10ca Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 16 Apr 2026 15:37:54 +0200 Subject: [PATCH 17/34] fix --- utils/_context/_scenarios/debugger.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/_context/_scenarios/debugger.py b/utils/_context/_scenarios/debugger.py index 2d91293d594..5e44e23b345 100644 --- a/utils/_context/_scenarios/debugger.py +++ b/utils/_context/_scenarios/debugger.py @@ -72,7 +72,10 @@ def configure(self, config: pytest.Config): self.agent_container.environment["DD_AGENT_HOST"] = weblog_env["DD_AGENT_HOST"] if not self.replay: - self.warmups.append(self._wait_for_agent_debugging) + if self._start_containers in self.post_collection_warmups: + self.post_collection_warmups.append(self._wait_for_agent_debugging) + else: + self.warmups.append(self._wait_for_agent_debugging) def _wait_for_agent_debugging(self) -> None: logger.stdout("Wait for /debugger/v1/diagnostics endpoint on agent") From c91ffc80bcc950e48bc02415a7ef6e784e93e2b0 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 16 Apr 2026 13:58:17 +0000 Subject: [PATCH 18/34] write setup_properties.json when no tests selected, fix replay on empty scenarios --- conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conftest.py b/conftest.py index 66a58a684de..c3555f8231f 100644 --- a/conftest.py +++ b/conftest.py @@ -417,6 +417,8 @@ def pytest_collection_finish(session: pytest.Session) -> None: setup_properties.load(context.scenario.host_log_folder) if len(session.items) == 0: + if not session.config.option.replay: + setup_properties.dump(context.scenario.host_log_folder) return context.scenario.execute_post_collection_warmups() From 4bfc9e8b70eb969ae0c39686727baf5c5a0ba7b9 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 16 Apr 2026 14:35:02 +0000 Subject: [PATCH 19/34] skip healthcheck read in post_start when agent_version already known from label --- utils/_context/containers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 88772827d91..9dc0618249c 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -804,15 +804,15 @@ def configure(self, *, host_log_folder: str, replay: bool): self.agent_version = ComponentVersion("agent", version_str).version def post_start(self): - already_known = self.agent_version is not None + if self.agent_version is not None: + return + with open(self.healthcheck_log_file, encoding="utf-8") as f: data = json.load(f) self.agent_version = ComponentVersion("agent", data["version"]).version - - if not already_known: - logger.stdout(f"Agent: {self.agent_version}") - logger.stdout(f"Backend: {self.dd_site}") + logger.stdout(f"Agent: {self.agent_version}") + logger.stdout(f"Backend: {self.dd_site}") @property def dd_site(self): From 9a3af9b6ce11cb91d3ede4263196bfd440ece766 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 16 Apr 2026 18:40:44 +0200 Subject: [PATCH 20/34] fix: start interfaces watchdog before containers in deferred path --- utils/_context/_scenarios/endtoend.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index ae3e3df0144..8ad26d872e0 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -362,8 +362,14 @@ def _defer_container_startup(self): container_warmups = [self._create_network, self._start_containers, *[c.post_start for c in self._containers]] for w in container_warmups: self.warmups.remove(w) + # Watchdog must start after network creation but before containers to capture early output + watchdog = self._start_interfaces_watchdog + self.post_collection_warmups.remove(watchdog) self.post_collection_warmups[0:0] = [ - *container_warmups, + self._create_network, + watchdog, + self._start_containers, + *[c.post_start for c in self._containers], self._set_agent_component, self._get_weblog_system_info, ] From 1f4feb1ad64bac7c75a0dc4c0c22a6e959314d5f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 17 Apr 2026 14:45:22 +0200 Subject: [PATCH 21/34] fix: restore watchdog-before-containers for non-deferred scenarios For the non-deferred path (old images without labels), the watchdog was moved to post_collection_warmups alongside _wait_for_app_readiness. This means the watchdog starts only after collection, by which time the proxy has already written files that end up in the observer's initial snapshot and are never ingested. Restore the original behaviour: insert _start_interfaces_watchdog at position 1 in warmups (before _create_network) for the elif/else paths, matching what the original code did with warmups.insert(1, ...). Also move _log_starting_containers into _defer_container_startup so the "Starting containers..." message is printed when containers actually start rather than during the pre-collection warmup phase. --- utils/_context/_scenarios/endtoend.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 8ad26d872e0..0f7275573f5 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -330,7 +330,6 @@ def configure(self, config: pytest.Config): self.library_interface_timeout = self._library_interface_timeout if not self.replay: - self.post_collection_warmups.append(self._start_interfaces_watchdog) self.post_collection_warmups.append(self._wait_for_app_readiness) self.post_collection_warmups.append(self._set_weblog_domain) @@ -350,24 +349,30 @@ def configure(self, config: pytest.Config): self.warmups.append(self._log_weblog_info) self.warmups.append(self._set_agent_component) if not self.replay: + self.warmups.insert(1, self._start_interfaces_watchdog) self.warmups.append(self._get_weblog_system_info) else: self.warmups.append(self._set_library_component) self.warmups.append(self._set_agent_component) if not self.replay: + self.warmups.insert(1, self._start_interfaces_watchdog) self.warmups.append(self._get_weblog_system_info) def _defer_container_startup(self): """Move container startup warmups to post_collection_warmups (inserted before interface warmups).""" - container_warmups = [self._create_network, self._start_containers, *[c.post_start for c in self._containers]] + container_warmups = [ + self._log_starting_containers, + self._create_network, + self._start_containers, + *[c.post_start for c in self._containers], + ] for w in container_warmups: self.warmups.remove(w) # Watchdog must start after network creation but before containers to capture early output - watchdog = self._start_interfaces_watchdog - self.post_collection_warmups.remove(watchdog) self.post_collection_warmups[0:0] = [ + self._log_starting_containers, self._create_network, - watchdog, + self._start_interfaces_watchdog, self._start_containers, *[c.post_start for c in self._containers], self._set_agent_component, From 216c33d53a48682c47dfa60ff0b199fa5efdf106 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 20 Apr 2026 14:48:23 +0200 Subject: [PATCH 22/34] fix: set agent component during configure in deferred path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the deferred path, _set_agent_component was called from post_collection_warmups, after pytest_collection_modifyitems had already run. That hook builds the Manifest from context.scenario.components, and match_condition returns False for any rule whose component is absent — so all agent-version-gated skip/xfail markers were silently dropped, causing tests that should be skipped to run and fail. Since agent_version is already known from the image label at configure time (that's the condition for taking the deferred path), call _set_agent_component() directly during configure alongside _set_library_component(), and remove the now-redundant call from _defer_container_startup. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/_context/_scenarios/endtoend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 0f7275573f5..ee67c4e3989 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -341,6 +341,7 @@ def configure(self, config: pytest.Config): # Both versions known from image labels: defer container startup to post-collection # so containers are skipped entirely when no tests are selected self._set_library_component() + self._set_agent_component() self.warmups.append(self._log_agent_info) self.warmups.append(self._log_weblog_info) self._defer_container_startup() @@ -375,7 +376,6 @@ def _defer_container_startup(self): self._start_interfaces_watchdog, self._start_containers, *[c.post_start for c in self._containers], - self._set_agent_component, self._get_weblog_system_info, ] From 4243c8bed84c60719ef0eba48ed2c1a1087d5b8a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 13 May 2026 14:35:08 +0200 Subject: [PATCH 23/34] fix: watchdog inserted before network in fallback/legacy paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _log_starting_containers was added at position 0, shifting _create_network to position 1. The insert(1, watchdog) now placed the watchdog before network creation. Use insert(2, ...) to restore the correct order: log → network → watchdog → containers --- utils/_context/_scenarios/endtoend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index ee67c4e3989..d6e3727eb71 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -350,13 +350,13 @@ def configure(self, config: pytest.Config): self.warmups.append(self._log_weblog_info) self.warmups.append(self._set_agent_component) if not self.replay: - self.warmups.insert(1, self._start_interfaces_watchdog) + self.warmups.insert(2, self._start_interfaces_watchdog) self.warmups.append(self._get_weblog_system_info) else: self.warmups.append(self._set_library_component) self.warmups.append(self._set_agent_component) if not self.replay: - self.warmups.insert(1, self._start_interfaces_watchdog) + self.warmups.insert(2, self._start_interfaces_watchdog) self.warmups.append(self._get_weblog_system_info) def _defer_container_startup(self): From 644606798940345307bc77fc99a10159fde84304 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 13 May 2026 14:35:15 +0200 Subject: [PATCH 24/34] fix: collect dotnet library version when installing from .so Use GetAssemblyVersion (already used in parametric) in poc/uds Dockerfiles to read the version from Datadog.Trace.dll and write /system-tests-library-version when the install script could not determine the version (i.e. .so install path). --- utils/build/docker/dotnet/poc.Dockerfile | 17 +++++++++++++++++ utils/build/docker/dotnet/uds.Dockerfile | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/utils/build/docker/dotnet/poc.Dockerfile b/utils/build/docker/dotnet/poc.Dockerfile index 81ed8d0c22a..2d9ea1ed4ca 100644 --- a/utils/build/docker/dotnet/poc.Dockerfile +++ b/utils/build/docker/dotnet/poc.Dockerfile @@ -1,3 +1,11 @@ +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-version-tool +WORKDIR /app +ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 +COPY utils/build/docker/dotnet/GetAssemblyVersion ./ +RUN dotnet publish -c Release -o out + +######### + FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-app WORKDIR /app @@ -20,6 +28,15 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl COPY utils/build/docker/dotnet/install_ddtrace.sh binaries/ /binaries/ RUN --mount=type=secret,id=github_token /binaries/install_ddtrace.sh +# extract library version from installed assembly if install script could not determine it +COPY --from=build-version-tool /app/out /tmp/get-assembly-version/ +RUN if [ ! -f /system-tests-library-version ]; then \ + dll=$(ls /opt/datadog/net*/Datadog.Trace.dll 2>/dev/null | head -1); \ + if [ -n "$dll" ]; then \ + /tmp/get-assembly-version/GetAssemblyVersion "$dll" > /system-tests-library-version; \ + fi; \ + fi && rm -rf /tmp/get-assembly-version + # Enable Datadog .NET SDK ENV CORECLR_ENABLE_PROFILING=1 ENV CORECLR_PROFILER='{846F5F1C-F9AE-4B07-969E-05C26BC060D8}' diff --git a/utils/build/docker/dotnet/uds.Dockerfile b/utils/build/docker/dotnet/uds.Dockerfile index b0aaad82efe..f259ab330c9 100644 --- a/utils/build/docker/dotnet/uds.Dockerfile +++ b/utils/build/docker/dotnet/uds.Dockerfile @@ -1,3 +1,11 @@ +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-version-tool +WORKDIR /app +ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 +COPY utils/build/docker/dotnet/GetAssemblyVersion ./ +RUN dotnet publish -c Release -o out + +######### + FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-app WORKDIR /app @@ -20,6 +28,15 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl COPY utils/build/docker/dotnet/install_ddtrace.sh binaries/ /binaries/ RUN --mount=type=secret,id=github_token /binaries/install_ddtrace.sh +# extract library version from installed assembly if install script could not determine it +COPY --from=build-version-tool /app/out /tmp/get-assembly-version/ +RUN if [ ! -f /system-tests-library-version ]; then \ + dll=$(ls /opt/datadog/net*/Datadog.Trace.dll 2>/dev/null | head -1); \ + if [ -n "$dll" ]; then \ + /tmp/get-assembly-version/GetAssemblyVersion "$dll" > /system-tests-library-version; \ + fi; \ + fi && rm -rf /tmp/get-assembly-version + # Enable Datadog .NET SDK ENV CORECLR_ENABLE_PROFILING=1 ENV CORECLR_PROFILER='{846F5F1C-F9AE-4B07-969E-05C26BC060D8}' From 4fd297f16d7531cc8947a443ec82b5da45c8fa69 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 13 May 2026 15:02:44 +0200 Subject: [PATCH 25/34] test: add warmup ordering tests for ADR-002 (post-collection rework) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 8 tests covering: - defer path: container startup absent from warmups, present in post_collection_warmups in the right order (network→watchdog→containers→readiness) - fallback/legacy paths: watchdog at index 2 (after _create_network) - execute_post_collection_warmups: invokes all callables, calls close_targets() and re-raises on error --- .../test_the_test/test_collection_warmups.py | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 tests/test_the_test/test_collection_warmups.py diff --git a/tests/test_the_test/test_collection_warmups.py b/tests/test_the_test/test_collection_warmups.py new file mode 100644 index 00000000000..f62bb38b133 --- /dev/null +++ b/tests/test_the_test/test_collection_warmups.py @@ -0,0 +1,190 @@ +"""Tests for post-collection warmup ordering (ADR-002). + +Covers: +- Defer path: both versions known from image labels → container startup + moves to post_collection_warmups in correct order. +- Fallback/legacy paths: watchdog is inserted at index 2 (after network). +- execute_post_collection_warmups: calls all callables and propagates errors. +""" + +from contextlib import contextmanager +from threading import RLock +from unittest.mock import MagicMock, patch + +import pytest + +from utils import interfaces, scenarios +from utils._context._scenarios.endtoend import EndToEndScenario +from utils._context.component_version import ComponentVersion +from utils._context.containers import ProxyContainer, TestedContainer + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_config(*, replay: bool = False) -> MagicMock: + cfg = MagicMock() + cfg.option.replay = replay + cfg.option.force_dd_trace_debug = False + cfg.option.force_dd_iast_debug = False + return cfg + + +def _stub_base_configure(self, *, host_log_folder: str, replay: bool) -> None: # noqa: ARG001 + """Minimal TestedContainer.configure substitute: skips Docker and file I/O.""" + self.host_log_folder = host_log_folder + self._starting_lock = RLock() + # image.labels / image.env must be pre-populated by the test fixture + + +def _stub_proxy_configure(self, *, host_log_folder: str, replay: bool) -> None: # noqa: ARG001 + """ProxyContainer.configure substitute: skips JSON file writes.""" + self.host_log_folder = host_log_folder + self._starting_lock = RLock() + + +@contextmanager +def _configured_scenario(*, library_version: str | None, agent_version: str | None): + """Yield a fully configured EndToEndScenario with all Docker I/O patched out. + + Image labels are pre-populated so that label-based version detection works: + - library_version set → WeblogContainer._library is set during configure() + - agent_version set → AgentContainer.agent_version is set during configure() + """ + scenario = EndToEndScenario("FAKE_E2E", doc="test", github_workflow="endtoend") + + # Pre-populate image labels/env to satisfy label-reading logic in configure() + scenario.weblog_container.image.labels = { + "system-tests-library": "python", + "system-tests-weblog-variant": "flask", + } + if library_version: + scenario.weblog_container.image.labels["system-tests-library-version"] = library_version + scenario.weblog_container.image.env = {} + + scenario.agent_container.image.labels = {} + if agent_version: + scenario.agent_container.image.labels["org.opencontainers.image.version"] = agent_version + + with ( + patch("utils._context._scenarios.endtoend.get_docker_client") as mock_dc, + patch.object(TestedContainer, "configure", _stub_base_configure), + patch.object(ProxyContainer, "configure", _stub_proxy_configure), + patch.object(interfaces.agent, "configure"), + patch.object(interfaces.library, "configure"), + patch.object(interfaces.backend, "configure"), + patch.object(interfaces.library_dotnet_managed, "configure"), + patch.object(interfaces.library_stdout, "configure"), + patch.object(interfaces.agent_stdout, "configure"), + ): + mock_dc.return_value.info.return_value = {"CgroupVersion": "2"} + scenario.configure(_make_config()) + yield scenario + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@scenarios.test_the_test +class Test_WarmupOrdering: + """Warmup list invariants after EndToEndScenario.configure().""" + + # -- Defer path ---------------------------------------------------------- + + def test_defer_path_container_startup_not_in_warmups(self): + """Both versions from labels: container startup must be absent from warmups.""" + with _configured_scenario(library_version="1.2.3", agent_version="7.50.0") as s: + for fn in (s._create_network, s._start_containers): + assert fn not in s.warmups, f"{fn.__name__} must not be in warmups (defer path)" + for c in s._containers: + assert c.post_start not in s.warmups, f"{c.name}.post_start must not be in warmups (defer path)" + + def test_defer_path_post_collection_order(self): + """Defer path: post_collection_warmups order must be network→watchdog→containers→readiness.""" + with _configured_scenario(library_version="1.2.3", agent_version="7.50.0") as s: + pcw = s.post_collection_warmups + + idx_net = pcw.index(s._create_network) + idx_wdg = pcw.index(s._start_interfaces_watchdog) + idx_start = pcw.index(s._start_containers) + idx_readiness = pcw.index(s._wait_for_app_readiness) + + assert idx_net < idx_wdg, "network must precede watchdog" + assert idx_wdg < idx_start, "watchdog must precede _start_containers" + + for c in s._containers: + idx_ps = pcw.index(c.post_start) + assert idx_start < idx_ps, f"{c.name}.post_start must follow _start_containers" + assert idx_ps < idx_readiness, f"{c.name}.post_start must precede _wait_for_app_readiness" + + def test_defer_path_weblog_system_info_before_readiness(self): + """_get_weblog_system_info must appear in post_collection_warmups before _wait_for_app_readiness.""" + with _configured_scenario(library_version="1.2.3", agent_version="7.50.0") as s: + pcw = s.post_collection_warmups + assert s._get_weblog_system_info in pcw + assert pcw.index(s._get_weblog_system_info) < pcw.index(s._wait_for_app_readiness) + + # -- Fallback path (library known, agent unknown) ------------------------ + + def test_fallback_path_watchdog_position(self): + """Library version from label only: watchdog must be at index 2 (after network).""" + with _configured_scenario(library_version="1.2.3", agent_version=None) as s: + assert s.warmups[0] == s._log_starting_containers + assert s.warmups[1] == s._create_network + assert s.warmups[2] == s._start_interfaces_watchdog + assert s.warmups[3] == s._start_containers + + def test_fallback_path_container_startup_in_warmups(self): + """Library version from label only: container startup must stay in warmups.""" + with _configured_scenario(library_version="1.2.3", agent_version=None) as s: + assert s._create_network in s.warmups + assert s._start_containers in s.warmups + + # -- Legacy path (no labels) --------------------------------------------- + + def test_legacy_path_watchdog_position(self): + """No label versions: watchdog must be at index 2 (after network).""" + with _configured_scenario(library_version=None, agent_version=None) as s: + assert s.warmups[0] == s._log_starting_containers + assert s.warmups[1] == s._create_network + assert s.warmups[2] == s._start_interfaces_watchdog + assert s.warmups[3] == s._start_containers + + +@scenarios.test_the_test +class Test_ExecutePostCollectionWarmups: + """execute_post_collection_warmups behaviour.""" + + def test_all_callables_are_invoked(self): + """Every item in post_collection_warmups is called in order.""" + from utils._context._scenarios.core import Scenario + + calls: list[int] = [] + scenario = Scenario("FAKE", doc="", github_workflow="testthetest") + scenario.post_collection_warmups = [lambda i=i: calls.append(i) for i in range(3)] + + scenario.execute_post_collection_warmups() + + assert calls == [0, 1, 2] + + def test_error_calls_close_targets_and_reraises(self): + """An exception in a warmup must trigger close_targets() and propagate.""" + from utils._context._scenarios.core import Scenario + + closed: list[bool] = [] + + class FakeScenario(Scenario): + def close_targets(self): + closed.append(True) + + scenario = FakeScenario("FAKE", doc="", github_workflow="testthetest") + scenario.post_collection_warmups = [lambda: (_ for _ in ()).throw(RuntimeError("boom"))] + + with pytest.raises(RuntimeError, match="boom"): + scenario.execute_post_collection_warmups() + + assert closed == [True], "close_targets() must be called on warmup error" From d1671ffa410122070e30ccfdbad019cfdfa575a9 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 13 May 2026 16:47:58 +0200 Subject: [PATCH 26/34] refactor: dedupe Scenario warmup runner into _run_warmups helper --- utils/_context/_scenarios/core.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/utils/_context/_scenarios/core.py b/utils/_context/_scenarios/core.py index b5f2156a923..fef4959e0ed 100644 --- a/utils/_context/_scenarios/core.py +++ b/utils/_context/_scenarios/core.py @@ -2,12 +2,12 @@ import os from pathlib import Path import shutil +from collections.abc import Callable from typing import TYPE_CHECKING if TYPE_CHECKING: from utils._context.component_version import Version - from collections.abc import Callable import pytest from utils._logger import logger, get_log_formatter @@ -188,19 +188,15 @@ def pytest_sessionstart(self, session: pytest.Session): # noqa: ARG002 """Called at the very begining of the process""" logger.terminal.write_sep("=", "test context", bold=True) - - try: - for warmup in self.warmups: - logger.info(f"Executing warmup {warmup}") - warmup() - except: - self.close_targets() - raise + self._run_warmups(self.warmups, label="") def execute_post_collection_warmups(self): + self._run_warmups(self.post_collection_warmups, label="post-collection ") + + def _run_warmups(self, warmups: list[Callable], *, label: str): try: - for warmup in self.post_collection_warmups: - logger.info(f"Executing post-collection warmup {warmup}") + for warmup in warmups: + logger.info(f"Executing {label}warmup {warmup}") warmup() except: self.close_targets() From 26bdacf906d080e3b1e882e0813f2ad53c4399bc Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 13 May 2026 16:48:31 +0200 Subject: [PATCH 27/34] refactor: collapse warmup branches and centralize startup logging in scenario - EndToEndScenario.configure: replace 3-branch if/elif/else with two flat blocks for library_known / agent_known and a single defer-or-watchdog tail. - Container post_start methods stop emitting Library/Agent/Backend/UDS/variant log lines; the scenario warmup is now the sole owner of those logs (no more divergent ordering between label and healthcheck paths). - Track container-startup warmups on the scenario so the defer path can move them to post_collection_warmups by identity instead of rebuilding lambdas. --- utils/_context/_scenarios/endtoend.py | 95 ++++++++++++--------------- utils/_context/containers.py | 11 ---- 2 files changed, 42 insertions(+), 64 deletions(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index f6e7a338b73..b8366996faa 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -1,4 +1,6 @@ import os +from collections.abc import Callable # noqa: TC003 (used at runtime in annotation) + import pytest from docker.models.networks import Network @@ -89,15 +91,19 @@ def configure(self, config: pytest.Config): # noqa: ARG002 if not self.replay: docker_info = get_docker_client().info() self.components["docker.Cgroup"] = docker_info.get("CgroupVersion", None) - self.warmups.append(self._log_starting_containers) - self.warmups.append(self._create_network) - self.warmups.append(self._start_containers) for container in reversed(self._containers): container.configure(host_log_folder=self.host_log_folder, replay=self.replay) - for container in self._containers: - self.warmups.append(container.post_start) + self._container_warmups: list[Callable] = [ + lambda: logger.stdout("Starting containers..."), + self._create_network, + self._start_containers, + *(c.post_start for c in self._containers), + ] + + if not self.replay: + self.warmups.extend(self._container_warmups) def get_container_by_dd_integration_name(self, name: str): for container in self._containers: @@ -129,9 +135,6 @@ def _ingest(self, event: FileSystemEvent): observer.start() - def _log_starting_containers(self): - logger.stdout("Starting containers...") - def _create_network(self) -> None: name = "system-tests-ipv6" if self.enable_ipv6 else "system-tests-ipv4" @@ -335,55 +338,41 @@ def configure(self, config: pytest.Config): else: self.library_interface_timeout = self._library_interface_timeout - if not self.replay: - self.post_collection_warmups.append(self._wait_for_app_readiness) - self.post_collection_warmups.append(self._set_weblog_domain) - - if ( - not self.replay - and self.weblog_container._library is not None # noqa: SLF001 - and self.agent_container.agent_version is not None - ): - # Both versions known from image labels: defer container startup to post-collection - # so containers are skipped entirely when no tests are selected - self._set_library_component() - self._set_agent_component() - self.warmups.append(self._log_agent_info) - self.warmups.append(self._log_weblog_info) - self._defer_container_startup() - elif self.weblog_container._library is not None: # noqa: SLF001 + # Resolve component versions (from image labels when available, else as a warmup). + library_known = self.weblog_container._library is not None # noqa: SLF001 + agent_known = self.agent_container.agent_version is not None + + if library_known: self._set_library_component() - self.warmups.append(self._log_weblog_info) - self.warmups.append(self._set_agent_component) - if not self.replay: - self.warmups.insert(2, self._start_interfaces_watchdog) - self.warmups.append(self._get_weblog_system_info) else: self.warmups.append(self._set_library_component) + self.warmups.append(self._log_weblog_info) + + if agent_known: + self._set_agent_component() + else: self.warmups.append(self._set_agent_component) - if not self.replay: - self.warmups.insert(2, self._start_interfaces_watchdog) - self.warmups.append(self._get_weblog_system_info) - - def _defer_container_startup(self): - """Move container startup warmups to post_collection_warmups (inserted before interface warmups).""" - container_warmups = [ - self._log_starting_containers, - self._create_network, - self._start_containers, - *[c.post_start for c in self._containers], - ] - for w in container_warmups: - self.warmups.remove(w) - # Watchdog must start after network creation but before containers to capture early output - self.post_collection_warmups[0:0] = [ - self._log_starting_containers, - self._create_network, - self._start_interfaces_watchdog, - self._start_containers, - *[c.post_start for c in self._containers], - self._get_weblog_system_info, - ] + self.warmups.append(self._log_agent_info) + + if self.replay: + return + + self.post_collection_warmups.append(self._wait_for_app_readiness) + self.post_collection_warmups.append(self._set_weblog_domain) + + if library_known and agent_known: + # Both versions known upfront: defer container startup to post-collection so an empty + # test session skips Docker entirely. Watchdog must run after network/before containers. + for step in self._container_warmups: + self.warmups.remove(step) + steps = list(self._container_warmups) + steps.insert(2, self._start_interfaces_watchdog) + steps.append(self._get_weblog_system_info) + self.post_collection_warmups[0:0] = steps + else: + net_idx = self.warmups.index(self._create_network) + self.warmups.insert(net_idx + 1, self._start_interfaces_watchdog) + self.warmups.append(self._get_weblog_system_info) def _set_containers_dependancies(self) -> None: if self._use_proxy_for_agent: diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 3437195e118..9f9afb62af7 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -811,8 +811,6 @@ def post_start(self): data = json.load(f) self.agent_version = ComponentVersion("agent", data["version"]).version - logger.stdout(f"Agent: {self.agent_version}") - logger.stdout(f"Backend: {self.dd_site}") @property def dd_site(self): @@ -1128,15 +1126,6 @@ def post_start(self): logger.warning( "Library version from healthcheck — add system-tests-library-version label to speed up startup" ) - logger.stdout(f"Library: {self.library}") - - if self.appsec_rules_file: - logger.stdout("Using a custom appsec rules file") - - if self.uds_mode: - logger.stdout(f"UDS socket: {self.uds_socket}") - - logger.stdout(f"Weblog variant: {self.weblog_variant}") if self._container is not None: exit_code, output = self.exec_run("cat /binaries/metadata.txt") From c1df3705126be348aacabb3f8fa315d1bfe9a1d3 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 13 May 2026 16:48:57 +0200 Subject: [PATCH 28/34] refactor: simplify debugger/go_proxies/conftest control flow - DebuggerScenario: pick warmup target list with a ternary instead of branching. - GoProxiesScenario._set_components: drop defensive None guard; agent_version is always set by configure() (label) or post_start() (healthcheck) before this warmup runs. - conftest: use truthy check on session.items. --- conftest.py | 2 +- utils/_context/_scenarios/debugger.py | 8 ++++---- utils/_context/_scenarios/go_proxies.py | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/conftest.py b/conftest.py index 0480f6b4843..5f6c3d0d608 100644 --- a/conftest.py +++ b/conftest.py @@ -425,7 +425,7 @@ def pytest_collection_finish(session: pytest.Session) -> None: if session.config.option.replay: setup_properties.load(context.scenario.host_log_folder) - if len(session.items) == 0: + if not session.items: if not session.config.option.replay: setup_properties.dump(context.scenario.host_log_folder) return diff --git a/utils/_context/_scenarios/debugger.py b/utils/_context/_scenarios/debugger.py index 5e44e23b345..5742d2027ec 100644 --- a/utils/_context/_scenarios/debugger.py +++ b/utils/_context/_scenarios/debugger.py @@ -72,10 +72,10 @@ def configure(self, config: pytest.Config): self.agent_container.environment["DD_AGENT_HOST"] = weblog_env["DD_AGENT_HOST"] if not self.replay: - if self._start_containers in self.post_collection_warmups: - self.post_collection_warmups.append(self._wait_for_agent_debugging) - else: - self.warmups.append(self._wait_for_agent_debugging) + target = ( + self.post_collection_warmups if self._start_containers in self.post_collection_warmups else self.warmups + ) + target.append(self._wait_for_agent_debugging) def _wait_for_agent_debugging(self) -> None: logger.stdout("Wait for /debugger/v1/diagnostics endpoint on agent") diff --git a/utils/_context/_scenarios/go_proxies.py b/utils/_context/_scenarios/go_proxies.py index 352f14bc981..4df5f66d5f9 100644 --- a/utils/_context/_scenarios/go_proxies.py +++ b/utils/_context/_scenarios/go_proxies.py @@ -108,8 +108,7 @@ def _wait_for_app_readiness(self) -> None: logger.debug("Agent ready") def _set_components(self) -> None: - if self._agent_container.agent_version is not None: - self.components["agent"] = self._agent_container.agent_version + self.components["agent"] = self._agent_container.agent_version lib = self.library self.components["library"] = lib.version self.components[lib.name] = lib.version From ed0c5452940ab20aea7c1489c1e2771164f56f1f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 13 May 2026 16:51:53 +0200 Subject: [PATCH 29/34] test: tighten warmup ordering tests - Drop duplicate ProxyContainer stub (identical to TestedContainer one). - Yield-with-cleanup fixture pops the test scenario from the global group registry to avoid polluting subsequent tests. - Drop unused config attrs and ad-hoc replay parameter from helpers. - Replace exact-index assertions on warmups[0..3] (which broke when the 'Starting containers' log entry became an anonymous lambda) with ordering invariants via .index(). - Whitelist SLF001/ANN001 for tests/test_the_test/* (warmup tests need to inspect privates and stub internal interfaces); drop the now-unused per-line ANN001 noqa directives in two existing files. --- pyproject.toml | 2 + .../test_the_test/test_collection_warmups.py | 129 ++++++------------ tests/test_the_test/test_decorators.py | 2 +- tests/test_the_test/test_docker_scenario.py | 2 +- 4 files changed, 42 insertions(+), 93 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8f407a684b3..d57970275dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,6 +220,8 @@ ignore = [ ] "tests/test_the_test/*" = [ "S605", # start-process-with-a-shell: test the test needs to run shell commands + "ANN001", # missing-type-function-argument: stubs of internal interfaces + "SLF001", # private-member-access: tests of warmup ordering need to inspect privates ] "utils/_context/_scenarios/auto_injection.py" = [ "PLC0415", # import-outside-top-level TODO diff --git a/tests/test_the_test/test_collection_warmups.py b/tests/test_the_test/test_collection_warmups.py index f62bb38b133..f529b57a79e 100644 --- a/tests/test_the_test/test_collection_warmups.py +++ b/tests/test_the_test/test_collection_warmups.py @@ -1,11 +1,4 @@ -"""Tests for post-collection warmup ordering (ADR-002). - -Covers: -- Defer path: both versions known from image labels → container startup - moves to post_collection_warmups in correct order. -- Fallback/legacy paths: watchdog is inserted at index 2 (after network). -- execute_post_collection_warmups: calls all callables and propagates errors. -""" +"""Tests for post-collection warmup ordering (ADR-002).""" from contextlib import contextmanager from threading import RLock @@ -14,48 +7,25 @@ import pytest from utils import interfaces, scenarios +from utils._context._scenarios.core import Scenario from utils._context._scenarios.endtoend import EndToEndScenario -from utils._context.component_version import ComponentVersion from utils._context.containers import ProxyContainer, TestedContainer -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_config(*, replay: bool = False) -> MagicMock: - cfg = MagicMock() - cfg.option.replay = replay - cfg.option.force_dd_trace_debug = False - cfg.option.force_dd_iast_debug = False - return cfg - - -def _stub_base_configure(self, *, host_log_folder: str, replay: bool) -> None: # noqa: ARG001 - """Minimal TestedContainer.configure substitute: skips Docker and file I/O.""" - self.host_log_folder = host_log_folder - self._starting_lock = RLock() - # image.labels / image.env must be pre-populated by the test fixture - - -def _stub_proxy_configure(self, *, host_log_folder: str, replay: bool) -> None: # noqa: ARG001 - """ProxyContainer.configure substitute: skips JSON file writes.""" +def _stub_configure(self, *, host_log_folder: str, replay: bool) -> None: # noqa: ARG001 + """Stand-in for *Container.configure that skips Docker and file I/O.""" self.host_log_folder = host_log_folder self._starting_lock = RLock() @contextmanager def _configured_scenario(*, library_version: str | None, agent_version: str | None): - """Yield a fully configured EndToEndScenario with all Docker I/O patched out. + """Yield a configured EndToEndScenario with all Docker / interface I/O patched out. - Image labels are pre-populated so that label-based version detection works: - - library_version set → WeblogContainer._library is set during configure() - - agent_version set → AgentContainer.agent_version is set during configure() + Image labels drive label-based version detection in container.configure(). """ - scenario = EndToEndScenario("FAKE_E2E", doc="test", github_workflow="endtoend") + scenario = EndToEndScenario("FAKE_E2E", doc="test") - # Pre-populate image labels/env to satisfy label-reading logic in configure() scenario.weblog_container.image.labels = { "system-tests-library": "python", "system-tests-weblog-variant": "flask", @@ -68,10 +38,13 @@ def _configured_scenario(*, library_version: str | None, agent_version: str | No if agent_version: scenario.agent_container.image.labels["org.opencontainers.image.version"] = agent_version + cfg = MagicMock() + cfg.option.replay = False + with ( patch("utils._context._scenarios.endtoend.get_docker_client") as mock_dc, - patch.object(TestedContainer, "configure", _stub_base_configure), - patch.object(ProxyContainer, "configure", _stub_proxy_configure), + patch.object(TestedContainer, "configure", _stub_configure), + patch.object(ProxyContainer, "configure", _stub_configure), patch.object(interfaces.agent, "configure"), patch.object(interfaces.library, "configure"), patch.object(interfaces.backend, "configure"), @@ -80,89 +53,62 @@ def _configured_scenario(*, library_version: str | None, agent_version: str | No patch.object(interfaces.agent_stdout, "configure"), ): mock_dc.return_value.info.return_value = {"CgroupVersion": "2"} - scenario.configure(_make_config()) - yield scenario - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- + scenario.configure(cfg) + try: + yield scenario + finally: + # Pop from the global scenario group registry to avoid polluting other tests. + for group in scenario.scenario_groups: + if scenario in group.scenarios: + group.scenarios.remove(scenario) @scenarios.test_the_test class Test_WarmupOrdering: """Warmup list invariants after EndToEndScenario.configure().""" - # -- Defer path ---------------------------------------------------------- - def test_defer_path_container_startup_not_in_warmups(self): - """Both versions from labels: container startup must be absent from warmups.""" with _configured_scenario(library_version="1.2.3", agent_version="7.50.0") as s: for fn in (s._create_network, s._start_containers): - assert fn not in s.warmups, f"{fn.__name__} must not be in warmups (defer path)" + assert fn not in s.warmups for c in s._containers: - assert c.post_start not in s.warmups, f"{c.name}.post_start must not be in warmups (defer path)" + assert c.post_start not in s.warmups def test_defer_path_post_collection_order(self): - """Defer path: post_collection_warmups order must be network→watchdog→containers→readiness.""" with _configured_scenario(library_version="1.2.3", agent_version="7.50.0") as s: pcw = s.post_collection_warmups - idx_net = pcw.index(s._create_network) idx_wdg = pcw.index(s._start_interfaces_watchdog) idx_start = pcw.index(s._start_containers) idx_readiness = pcw.index(s._wait_for_app_readiness) - assert idx_net < idx_wdg, "network must precede watchdog" - assert idx_wdg < idx_start, "watchdog must precede _start_containers" - + assert idx_net < idx_wdg < idx_start < idx_readiness for c in s._containers: idx_ps = pcw.index(c.post_start) - assert idx_start < idx_ps, f"{c.name}.post_start must follow _start_containers" - assert idx_ps < idx_readiness, f"{c.name}.post_start must precede _wait_for_app_readiness" + assert idx_start < idx_ps < idx_readiness def test_defer_path_weblog_system_info_before_readiness(self): - """_get_weblog_system_info must appear in post_collection_warmups before _wait_for_app_readiness.""" with _configured_scenario(library_version="1.2.3", agent_version="7.50.0") as s: pcw = s.post_collection_warmups - assert s._get_weblog_system_info in pcw assert pcw.index(s._get_weblog_system_info) < pcw.index(s._wait_for_app_readiness) - # -- Fallback path (library known, agent unknown) ------------------------ - - def test_fallback_path_watchdog_position(self): - """Library version from label only: watchdog must be at index 2 (after network).""" - with _configured_scenario(library_version="1.2.3", agent_version=None) as s: - assert s.warmups[0] == s._log_starting_containers - assert s.warmups[1] == s._create_network - assert s.warmups[2] == s._start_interfaces_watchdog - assert s.warmups[3] == s._start_containers - - def test_fallback_path_container_startup_in_warmups(self): - """Library version from label only: container startup must stay in warmups.""" + def test_fallback_path_watchdog_after_network(self): + """Library version from label only: watchdog must follow network creation.""" with _configured_scenario(library_version="1.2.3", agent_version=None) as s: assert s._create_network in s.warmups assert s._start_containers in s.warmups + assert s.warmups.index(s._create_network) < s.warmups.index(s._start_interfaces_watchdog) + assert s.warmups.index(s._start_interfaces_watchdog) < s.warmups.index(s._start_containers) - # -- Legacy path (no labels) --------------------------------------------- - - def test_legacy_path_watchdog_position(self): - """No label versions: watchdog must be at index 2 (after network).""" + def test_legacy_path_watchdog_after_network(self): with _configured_scenario(library_version=None, agent_version=None) as s: - assert s.warmups[0] == s._log_starting_containers - assert s.warmups[1] == s._create_network - assert s.warmups[2] == s._start_interfaces_watchdog - assert s.warmups[3] == s._start_containers + assert s.warmups.index(s._create_network) < s.warmups.index(s._start_interfaces_watchdog) + assert s.warmups.index(s._start_interfaces_watchdog) < s.warmups.index(s._start_containers) @scenarios.test_the_test class Test_ExecutePostCollectionWarmups: - """execute_post_collection_warmups behaviour.""" - def test_all_callables_are_invoked(self): - """Every item in post_collection_warmups is called in order.""" - from utils._context._scenarios.core import Scenario - calls: list[int] = [] scenario = Scenario("FAKE", doc="", github_workflow="testthetest") scenario.post_collection_warmups = [lambda i=i: calls.append(i) for i in range(3)] @@ -172,19 +118,20 @@ def test_all_callables_are_invoked(self): assert calls == [0, 1, 2] def test_error_calls_close_targets_and_reraises(self): - """An exception in a warmup must trigger close_targets() and propagate.""" - from utils._context._scenarios.core import Scenario - closed: list[bool] = [] class FakeScenario(Scenario): def close_targets(self): closed.append(True) - scenario = FakeScenario("FAKE", doc="", github_workflow="testthetest") - scenario.post_collection_warmups = [lambda: (_ for _ in ()).throw(RuntimeError("boom"))] + scenario = FakeScenario("FAKE2", doc="", github_workflow="testthetest") + + def boom(): + raise RuntimeError("boom") + + scenario.post_collection_warmups = [boom] with pytest.raises(RuntimeError, match="boom"): scenario.execute_post_collection_warmups() - assert closed == [True], "close_targets() must be called on warmup error" + assert closed == [True] diff --git a/tests/test_the_test/test_decorators.py b/tests/test_the_test/test_decorators.py index 4c5e200ef8f..8f2ecb8d466 100644 --- a/tests/test_the_test/test_decorators.py +++ b/tests/test_the_test/test_decorators.py @@ -27,7 +27,7 @@ def is_skipped(item: Any, reason: str): # noqa: ANN401 raise Exception(f"{item} is not skipped, or not with the good reason") -def is_not_skipped(item): # noqa: ANN001 +def is_not_skipped(item): if hasattr(item, "pytestmark"): for mark in item.pytestmark: if mark.name == ("skip", "xfail"): diff --git a/tests/test_the_test/test_docker_scenario.py b/tests/test_the_test/test_docker_scenario.py index ef3241b9cb9..c0d16e9475e 100644 --- a/tests/test_the_test/test_docker_scenario.py +++ b/tests/test_the_test/test_docker_scenario.py @@ -14,7 +14,7 @@ def __init__(self, name: str, events: list | None = None) -> None: def configure(self, *, host_log_folder: str, replay: bool): # noqa: ARG002 self._starting_lock = RLock() - def start(self, network): # noqa: ARG002, ANN001 + def start(self, network): # noqa: ARG002 self._test_events.append(f"start {self.name}") self.healthy = True From 5085fe6b27986e5b4de56e93fdabfadfafe5a5c6 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 13 May 2026 16:51:55 +0200 Subject: [PATCH 30/34] build: simplify library-version extraction and dedupe dotnet version-tool stage - build.sh: drop the multi-path lookup loop. Every install_ddtrace.sh on this branch writes /system-tests-library-version, so reading the canonical path is sufficient. - Pre-build the .NET assembly-version helper image once (system_tests/dotnet-version-tool) and have both poc.Dockerfile and uds.Dockerfile COPY --from=that tag, removing the duplicated build-version-tool stage from each Dockerfile. --- utils/build/build.sh | 20 ++++++++++++++----- utils/build/docker/dotnet/poc.Dockerfile | 10 +--------- utils/build/docker/dotnet/uds.Dockerfile | 10 +--------- .../docker/dotnet/version-tool.Dockerfile | 5 +++++ 4 files changed, 22 insertions(+), 23 deletions(-) create mode 100644 utils/build/docker/dotnet/version-tool.Dockerfile diff --git a/utils/build/build.sh b/utils/build/build.sh index ccdef7c507c..ac08548afda 100755 --- a/utils/build/build.sh +++ b/utils/build/build.sh @@ -291,6 +291,17 @@ build() { DOCKERFILE=utils/build/docker/${TEST_LIBRARY}/${WEBLOG_VARIANT}.Dockerfile + # Pre-build the .NET assembly-version tool image so both poc/uds Dockerfiles can + # COPY --from=system_tests/dotnet-version-tool without each duplicating the stage. + if [[ $TEST_LIBRARY == dotnet ]]; then + run_build_command docker buildx build \ + --load \ + ${DOCKER_PLATFORM_ARGS} \ + -f utils/build/docker/dotnet/version-tool.Dockerfile \ + -t system_tests/dotnet-version-tool \ + . + fi + GITHUB_TOKEN_SECRET_ARG="" if [ -n "${GITHUB_TOKEN_FILE:-}" ]; then @@ -318,12 +329,11 @@ build() { $EXTRA_DOCKER_ARGS \ . + # Read library version baked into the image by install_ddtrace.sh and re-tag + # with a system-tests-library-version label so the scenario can skip the + # post-start healthcheck round-trip when no tests are selected. CID=$(docker create system_tests/weblog) - LIBRARY_VERSION="" - for _path in /system-tests-library-version /app/SYSTEM_TESTS_LIBRARY_VERSION /binaries/SYSTEM_TESTS_LIBRARY_VERSION /builds/SYSTEM_TESTS_LIBRARY_VERSION /SYSTEM_TESTS_LIBRARY_VERSION; do - LIBRARY_VERSION=$(docker cp "${CID}:${_path}" - 2>/dev/null | tar -xO 2>/dev/null | tr -d '[:space:]') - [ -n "${LIBRARY_VERSION}" ] && break - done + LIBRARY_VERSION=$(docker cp "${CID}:/system-tests-library-version" - 2>/dev/null | tar -xO 2>/dev/null | tr -d '[:space:]' || true) docker rm "${CID}" > /dev/null if [ -n "${LIBRARY_VERSION}" ]; then docker build \ diff --git a/utils/build/docker/dotnet/poc.Dockerfile b/utils/build/docker/dotnet/poc.Dockerfile index e8b7519226a..8bbd4328b65 100644 --- a/utils/build/docker/dotnet/poc.Dockerfile +++ b/utils/build/docker/dotnet/poc.Dockerfile @@ -1,11 +1,3 @@ -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-version-tool -WORKDIR /app -ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 -COPY utils/build/docker/dotnet/GetAssemblyVersion ./ -RUN dotnet publish -c Release -o out - -######### - FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-app WORKDIR /app @@ -35,7 +27,7 @@ COPY utils/build/docker/dotnet/install_ddtrace.sh binaries/ /binaries/ RUN --mount=type=secret,id=github_token /binaries/install_ddtrace.sh # extract library version from installed assembly if install script could not determine it -COPY --from=build-version-tool /app/out /tmp/get-assembly-version/ +COPY --from=system_tests/dotnet-version-tool /out /tmp/get-assembly-version/ RUN if [ ! -f /system-tests-library-version ]; then \ dll=$(ls /opt/datadog/net*/Datadog.Trace.dll 2>/dev/null | head -1); \ if [ -n "$dll" ]; then \ diff --git a/utils/build/docker/dotnet/uds.Dockerfile b/utils/build/docker/dotnet/uds.Dockerfile index f259ab330c9..4c24c0fbc93 100644 --- a/utils/build/docker/dotnet/uds.Dockerfile +++ b/utils/build/docker/dotnet/uds.Dockerfile @@ -1,11 +1,3 @@ -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-version-tool -WORKDIR /app -ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 -COPY utils/build/docker/dotnet/GetAssemblyVersion ./ -RUN dotnet publish -c Release -o out - -######### - FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-app WORKDIR /app @@ -29,7 +21,7 @@ COPY utils/build/docker/dotnet/install_ddtrace.sh binaries/ /binaries/ RUN --mount=type=secret,id=github_token /binaries/install_ddtrace.sh # extract library version from installed assembly if install script could not determine it -COPY --from=build-version-tool /app/out /tmp/get-assembly-version/ +COPY --from=system_tests/dotnet-version-tool /out /tmp/get-assembly-version/ RUN if [ ! -f /system-tests-library-version ]; then \ dll=$(ls /opt/datadog/net*/Datadog.Trace.dll 2>/dev/null | head -1); \ if [ -n "$dll" ]; then \ diff --git a/utils/build/docker/dotnet/version-tool.Dockerfile b/utils/build/docker/dotnet/version-tool.Dockerfile new file mode 100644 index 00000000000..13720500fb3 --- /dev/null +++ b/utils/build/docker/dotnet/version-tool.Dockerfile @@ -0,0 +1,5 @@ +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-version-tool +WORKDIR /app +ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 +COPY utils/build/docker/dotnet/GetAssemblyVersion ./ +RUN dotnet publish -c Release -o /out From 456b679e0d87a7d7d93eb39a725535f3640d8bdb Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 25 Jun 2026 17:58:08 +0200 Subject: [PATCH 31/34] format --- utils/_context/_scenarios/go_proxies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/_context/_scenarios/go_proxies.py b/utils/_context/_scenarios/go_proxies.py index 4df5f66d5f9..fff39c35cff 100644 --- a/utils/_context/_scenarios/go_proxies.py +++ b/utils/_context/_scenarios/go_proxies.py @@ -108,7 +108,7 @@ def _wait_for_app_readiness(self) -> None: logger.debug("Agent ready") def _set_components(self) -> None: - self.components["agent"] = self._agent_container.agent_version + self.components["agent"] = self._agent_container.agent_version or Version("0.0.0") lib = self.library self.components["library"] = lib.version self.components[lib.name] = lib.version From 4e1abe9c804a3cd205bbd6de48f86bbf84bcaaeb Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 25 Jun 2026 18:24:52 +0200 Subject: [PATCH 32/34] fix: load library version from healthcheck log in replay mode for Lambda --- utils/_context/containers.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 2e3f4bb9da2..68cfcce9771 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -1212,6 +1212,20 @@ def __init__( # Remove port bindings, as only the LambdaProxyContainer needs to expose a server self.ports = {} + def configure(self, *, host_log_folder: str, replay: bool): + super().configure(host_log_folder=host_log_folder, replay=replay) + + if replay and self._library is None: + # In replay mode, containers do not start and post_start is never called. + # Load the library version from the healthcheck log saved during the previous run. + try: + with open(self.healthcheck_log_file, encoding="utf-8") as f: + data = json.load(f) + lib = data["library"] + self._library = ComponentVersion(lib["name"], lib["version"]) + except Exception as e: + logger.warning(f"Could not load library version from healthcheck log in replay mode: {e}") + class PostgresContainer(SqlDbTestedContainer): def __init__(self) -> None: From 4125cc5f52feb461c56a880599192240c508255e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 26 Jun 2026 18:00:53 +0200 Subject: [PATCH 33/34] fix: restore library version and log patterns loading in replay mode --- utils/_context/containers.py | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 68cfcce9771..96d6cac81cb 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -1017,6 +1017,21 @@ def configure(self, *, host_log_folder: str, replay: bool): version_from_label = self.image.labels.get("system-tests-library-version") if version_from_label: self._library = ComponentVersion(library, version_from_label) + elif replay: + # In replay mode, post_start is never called (containers do not start). + # Load the library version from the saved healthcheck log so that + # init_patterns() below can install the library-specific log-filtering + # rules (e.g. PHP env-var skip patterns that prevent SOME_SECRET_ENV leaks). + try: + with open(self.healthcheck_log_file, encoding="utf-8") as f: + data = json.load(f) + lib = data["library"] + self._library = ComponentVersion(lib["name"], lib["version"]) + except Exception as e: + logger.warning(f"Could not load library version in configure (replay): {e}") + + if self._library is not None: + self.stdout_interface.init_patterns(self._library) header_tags = "" if library in ("cpp_nginx", "cpp_httpd", "dotnet", "java", "python"): @@ -1697,6 +1712,17 @@ def __init__( }, ) + def configure(self, *, host_log_folder: str, replay: bool): + super().configure(host_log_folder=host_log_folder, replay=replay) + if replay: + try: + with open(self.healthcheck_log_file, encoding="utf-8") as f: + data = json.load(f) + lib = data["library"] + self.library = ComponentVersion("envoy", lib["version"]) + except Exception as e: + logger.warning(f"Could not load library version from healthcheck log in replay mode: {e}") + def post_start(self): with open(self.healthcheck_log_file, encoding="utf-8") as f: data = json.load(f) @@ -1773,6 +1799,17 @@ def __init__( }, ) + def configure(self, *, host_log_folder: str, replay: bool): + super().configure(host_log_folder=host_log_folder, replay=replay) + if replay: + try: + with open(self.healthcheck_log_file, encoding="utf-8") as f: + data = json.load(f) + lib = data["library"] + self.library = ComponentVersion("haproxy", lib["version"]) + except Exception as e: + logger.warning(f"Could not load library version from healthcheck log in replay mode: {e}") + def post_start(self): with open(self.healthcheck_log_file, encoding="utf-8") as f: data = json.load(f) From e472ab30f57d9800cfaefa607f769454d0d466ab Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 26 Jun 2026 18:01:03 +0200 Subject: [PATCH 34/34] fix: move version-tool.Dockerfile out of weblog discovery path --- utils/build/build.sh | 2 +- .../docker/dotnet/{ => build-helpers}/version-tool.Dockerfile | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename utils/build/docker/dotnet/{ => build-helpers}/version-tool.Dockerfile (100%) diff --git a/utils/build/build.sh b/utils/build/build.sh index 68bf3a3eb97..79dff8895eb 100755 --- a/utils/build/build.sh +++ b/utils/build/build.sh @@ -309,7 +309,7 @@ build() { run_build_command docker buildx build \ --load \ ${DOCKER_PLATFORM_ARGS} \ - -f utils/build/docker/dotnet/version-tool.Dockerfile \ + -f utils/build/docker/dotnet/build-helpers/version-tool.Dockerfile \ -t system_tests/dotnet-version-tool \ . fi diff --git a/utils/build/docker/dotnet/version-tool.Dockerfile b/utils/build/docker/dotnet/build-helpers/version-tool.Dockerfile similarity index 100% rename from utils/build/docker/dotnet/version-tool.Dockerfile rename to utils/build/docker/dotnet/build-helpers/version-tool.Dockerfile