Skip container startup for empty scenarios

conftest.py

@@ -425,6 +425,13 @@ def pytest_collection_finish(session: pytest.Session) -> None:
     if session.config.option.replay:
         setup_properties.load(context.scenario.host_log_folder)
 
+    if not session.items:
+        if not session.config.option.replay:
+            setup_properties.dump(context.scenario.host_log_folder)
+        return
+
+    context.scenario.execute_post_collection_warmups()
+
     last_item_file = ""
     for item in session.items:
         if _item_is_skipped(item):

pyproject.toml

@@ -220,6 +220,8 @@ ignore = [
 ]
 "tests/test_the_test/*" = [
     "S605",      # start-process-with-a-shell: test the test needs to run shell commands
+    "ANN001",    # missing-type-function-argument: stubs of internal interfaces
+    "SLF001",    # private-member-access: tests of warmup ordering need to inspect privates
 ]
 "utils/_context/_scenarios/auto_injection.py" = [
     "PLC0415",  # import-outside-top-level TODO

tests/test_the_test/test_collection_warmups.py

@@ -0,0 +1,137 @@
+"""Tests for post-collection warmup ordering (ADR-002)."""
+
+from contextlib import contextmanager
+from threading import RLock
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from utils import interfaces, scenarios
+from utils._context._scenarios.core import Scenario
+from utils._context._scenarios.endtoend import EndToEndScenario
+from utils._context.containers import ProxyContainer, TestedContainer
+
+
+def _stub_configure(self, *, host_log_folder: str, replay: bool) -> None:  # noqa: ARG001
+    """Stand-in for *Container.configure that skips Docker and file I/O."""
+    self.host_log_folder = host_log_folder
+    self._starting_lock = RLock()
+
+
+@contextmanager
+def _configured_scenario(*, library_version: str | None, agent_version: str | None):
+    """Yield a configured EndToEndScenario with all Docker / interface I/O patched out.
+
+    Image labels drive label-based version detection in container.configure().
+    """
+    scenario = EndToEndScenario("FAKE_E2E", doc="test")
+
+    scenario.weblog_container.image.labels = {
+        "system-tests-library": "python",
+        "system-tests-weblog-variant": "flask",
+    }
+    if library_version:
+        scenario.weblog_container.image.labels["system-tests-library-version"] = library_version
+    scenario.weblog_container.image.env = {}
+
+    scenario.agent_container.image.labels = {}
+    if agent_version:
+        scenario.agent_container.image.labels["org.opencontainers.image.version"] = agent_version
+
+    cfg = MagicMock()
+    cfg.option.replay = False
+
+    with (
+        patch("utils._context._scenarios.endtoend.get_docker_client") as mock_dc,
+        patch.object(TestedContainer, "configure", _stub_configure),
+        patch.object(ProxyContainer, "configure", _stub_configure),
+        patch.object(interfaces.agent, "configure"),
+        patch.object(interfaces.library, "configure"),
+        patch.object(interfaces.backend, "configure"),
+        patch.object(interfaces.library_dotnet_managed, "configure"),
+        patch.object(interfaces.library_stdout, "configure"),
+        patch.object(interfaces.agent_stdout, "configure"),
+    ):
+        mock_dc.return_value.info.return_value = {"CgroupVersion": "2"}
+        scenario.configure(cfg)
+        try:
+            yield scenario
+        finally:
+            # Pop from the global scenario group registry to avoid polluting other tests.
+            for group in scenario.scenario_groups:
+                if scenario in group.scenarios:
+                    group.scenarios.remove(scenario)
+
+
+@scenarios.test_the_test
+class Test_WarmupOrdering:
+    """Warmup list invariants after EndToEndScenario.configure()."""
+
+    def test_defer_path_container_startup_not_in_warmups(self):
+        with _configured_scenario(library_version="1.2.3", agent_version="7.50.0") as s:
+            for fn in (s._create_network, s._start_containers):
+                assert fn not in s.warmups
+            for c in s._containers:
+                assert c.post_start not in s.warmups
+
+    def test_defer_path_post_collection_order(self):
+        with _configured_scenario(library_version="1.2.3", agent_version="7.50.0") as s:
+            pcw = s.post_collection_warmups
+            idx_net = pcw.index(s._create_network)
+            idx_wdg = pcw.index(s._start_interfaces_watchdog)
+            idx_start = pcw.index(s._start_containers)
+            idx_readiness = pcw.index(s._wait_for_app_readiness)
+
+            assert idx_net < idx_wdg < idx_start < idx_readiness
+            for c in s._containers:
+                idx_ps = pcw.index(c.post_start)
+                assert idx_start < idx_ps < idx_readiness
+
+    def test_defer_path_weblog_system_info_before_readiness(self):
+        with _configured_scenario(library_version="1.2.3", agent_version="7.50.0") as s:
+            pcw = s.post_collection_warmups
+            assert pcw.index(s._get_weblog_system_info) < pcw.index(s._wait_for_app_readiness)
+
+    def test_fallback_path_watchdog_after_network(self):
+        """Library version from label only: watchdog must follow network creation."""
+        with _configured_scenario(library_version="1.2.3", agent_version=None) as s:
+            assert s._create_network in s.warmups
+            assert s._start_containers in s.warmups
+            assert s.warmups.index(s._create_network) < s.warmups.index(s._start_interfaces_watchdog)
+            assert s.warmups.index(s._start_interfaces_watchdog) < s.warmups.index(s._start_containers)
+
+    def test_legacy_path_watchdog_after_network(self):
+        with _configured_scenario(library_version=None, agent_version=None) as s:
+            assert s.warmups.index(s._create_network) < s.warmups.index(s._start_interfaces_watchdog)
+            assert s.warmups.index(s._start_interfaces_watchdog) < s.warmups.index(s._start_containers)
+
+
+@scenarios.test_the_test
+class Test_ExecutePostCollectionWarmups:
+    def test_all_callables_are_invoked(self):
+        calls: list[int] = []
+        scenario = Scenario("FAKE", doc="", github_workflow="testthetest")
+        scenario.post_collection_warmups = [lambda i=i: calls.append(i) for i in range(3)]
+
+        scenario.execute_post_collection_warmups()
+
+        assert calls == [0, 1, 2]
+
+    def test_error_calls_close_targets_and_reraises(self):
+        closed: list[bool] = []
+
+        class FakeScenario(Scenario):
+            def close_targets(self):
+                closed.append(True)
+
+        scenario = FakeScenario("FAKE2", doc="", github_workflow="testthetest")
+
+        def boom():
+            raise RuntimeError("boom")
+
+        scenario.post_collection_warmups = [boom]
+
+        with pytest.raises(RuntimeError, match="boom"):
+            scenario.execute_post_collection_warmups()
+
+        assert closed == [True]

tests/test_the_test/test_decorators.py

@@ -27,7 +27,7 @@ def is_skipped(item: Any, reason: str):  # noqa: ANN401
     raise Exception(f"{item} is not skipped, or not with the good reason")
 
 
-def is_not_skipped(item):  # noqa: ANN001
+def is_not_skipped(item):
     if hasattr(item, "pytestmark"):
         for mark in item.pytestmark:
             if mark.name == ("skip", "xfail"):

tests/test_the_test/test_docker_scenario.py

@@ -14,7 +14,7 @@ def __init__(self, name: str, events: list | None = None) -> None:
     def configure(self, *, host_log_folder: str, replay: bool):  # noqa: ARG002
         self._starting_lock = RLock()
 
-    def start(self, network):  # noqa: ARG002, ANN001
+    def start(self, network):  # noqa: ARG002
         self._test_events.append(f"start {self.name}")
         self.healthy = True
 

utils/_context/_scenarios/core.py

@@ -2,12 +2,12 @@
 import os
 from pathlib import Path
 import shutil
+from collections.abc import Callable
 from typing import TYPE_CHECKING
 
 
 if TYPE_CHECKING:
     from utils._context.component_version import Version
-    from collections.abc import Callable
 
 import pytest
 from utils._logger import logger, get_log_formatter
@@ -128,6 +128,7 @@ def __init__(
             group.scenarios.append(self)
 
         self.warmups: list[Callable] = []
+        self.post_collection_warmups: list[Callable] = []
         self.collect_only: bool = False
 
     def _create_log_subfolder(self, subfolder: str, *, remove_if_exists: bool = False):
@@ -187,10 +188,15 @@ def pytest_sessionstart(self, session: pytest.Session):  # noqa: ARG002
         """Called at the very begining of the process"""
 
         logger.terminal.write_sep("=", "test context", bold=True)
+        self._run_warmups(self.warmups, label="")
 
+    def execute_post_collection_warmups(self):
+        self._run_warmups(self.post_collection_warmups, label="post-collection ")
+
+    def _run_warmups(self, warmups: list[Callable], *, label: str):
         try:
-            for warmup in self.warmups:
-                logger.info(f"Executing warmup {warmup}")
+            for warmup in warmups:
+                logger.info(f"Executing {label}warmup {warmup}")
                 warmup()
         except:
             self.close_targets()

utils/_context/_scenarios/debugger.py

@@ -76,7 +76,10 @@ def configure(self, config: pytest.Config):
             self.agent_container.environment["DD_AGENT_HOST"] = weblog_env["DD_AGENT_HOST"]
 
         if not self.replay:
-            self.warmups.append(self._wait_for_agent_debugging)
+            target = (
+                self.post_collection_warmups if self._start_containers in self.post_collection_warmups else self.warmups
+            )
+            target.append(self._wait_for_agent_debugging)
 
     def _wait_for_agent_debugging(self) -> None:
         logger.stdout("Wait for /debugger/v1/diagnostics endpoint on agent")

utils/_context/_scenarios/endtoend.py

@@ -1,5 +1,7 @@
 from typing import Literal
 import os
+from collections.abc import Callable  # noqa: TC003 (used at runtime in annotation)
+
 import pytest
 
 from docker.models.networks import Network
@@ -93,14 +95,19 @@ def configure(self, config: pytest.Config):  # noqa: ARG002
         if not self.replay:
             docker_info = get_docker_client().info()
             self.components["docker.Cgroup"] = docker_info.get("CgroupVersion", None)
-            self.warmups.append(self._create_network)
-            self.warmups.append(self._start_containers)
 
         for container in reversed(self._containers):
             container.configure(host_log_folder=self.host_log_folder, replay=self.replay)
 
-        for container in self._containers:
-            self.warmups.append(container.post_start)
+        self._container_warmups: list[Callable] = [
+            lambda: logger.stdout("Starting containers..."),
+            self._create_network,
+            self._start_containers,
+            *(c.post_start for c in self._containers),
+        ]
+
+        if not self.replay:
+            self.warmups.extend(self._container_warmups)
 
     def get_container_by_dd_integration_name(self, name: str):
         for container in self._containers:
@@ -157,7 +164,6 @@ def _create_network(self) -> None:
             self._network = get_docker_client().networks.create(name, check_duplicate=True)
 
     def _start_containers(self):
-        logger.stdout("Starting containers...")
         threads = []
 
         for container in self._containers:
@@ -338,12 +344,41 @@ def configure(self, config: pytest.Config):
         else:
             self.library_interface_timeout = self._library_interface_timeout
 
-        if not self.replay:
-            self.warmups.insert(1, self._start_interfaces_watchdog)
+        # Resolve component versions (from image labels when available, else as a warmup).
+        library_known = self.weblog_container._library is not None  # noqa: SLF001
+        agent_known = self.agent_container.agent_version is not None
+
+        if library_known:
+            self._set_library_component()
+        else:
+            self.warmups.append(self._set_library_component)
+        self.warmups.append(self._log_weblog_info)
+
+        if agent_known:
+            self._set_agent_component()
+        else:
+            self.warmups.append(self._set_agent_component)
+        self.warmups.append(self._log_agent_info)
+
+        if self.replay:
+            return
+
+        self.post_collection_warmups.append(self._wait_for_app_readiness)
+        self.post_collection_warmups.append(self._set_weblog_domain)
+
+        if library_known and agent_known:
+            # Both versions known upfront: defer container startup to post-collection so an empty
+            # test session skips Docker entirely. Watchdog must run after network/before containers.
+            for step in self._container_warmups:
+                self.warmups.remove(step)
+            steps = list(self._container_warmups)
+            steps.insert(2, self._start_interfaces_watchdog)
+            steps.append(self._get_weblog_system_info)
+            self.post_collection_warmups[0:0] = steps
+        else:
+            net_idx = self.warmups.index(self._create_network)
+            self.warmups.insert(net_idx + 1, self._start_interfaces_watchdog)
             self.warmups.append(self._get_weblog_system_info)
-            self.warmups.append(self._wait_for_app_readiness)
-            self.warmups.append(self._set_weblog_domain)
-        self.warmups.append(self._set_components)
 
     def _set_containers_dependancies(self) -> None:
         if self._use_proxy_for_agent:
@@ -389,11 +424,25 @@ def _set_weblog_domain(self):
         if self.enable_ipv6:
             self.weblog_container.set_weblog_domain_for_ipv6(self._network)
 
-    def _set_components(self):
-        self.components["agent"] = self.agent_version
+    def _log_agent_info(self):
+        logger.stdout(f"Agent: {self.agent_container.agent_version}")
+        logger.stdout(f"Backend: {self.agent_container.dd_site}")
+
+    def _log_weblog_info(self):
+        logger.stdout(f"Library: {self.library}")
+        if self.weblog_container.appsec_rules_file:
+            logger.stdout("Using a custom appsec rules file")
+        if self.weblog_container.uds_mode:
+            logger.stdout(f"UDS socket: {self.weblog_container.uds_socket}")
+        logger.stdout(f"Weblog variant: {self.weblog_variant}")
+
+    def _set_library_component(self):
         self.components["library"] = self.library.version
         self.components[self.library.name] = self.library.version
 
+    def _set_agent_component(self):
+        self.components["agent"] = self.agent_version
+
     def _wait_for_app_readiness(self):
         if self._use_proxy_for_weblog:
             logger.debug("Wait for app readiness")

utils/_context/_scenarios/go_proxies.py

@@ -108,7 +108,7 @@ def _wait_for_app_readiness(self) -> None:
         logger.debug("Agent ready")
 
     def _set_components(self) -> None:
-        self.components["agent"] = self._agent_container.agent_version
+        self.components["agent"] = self._agent_container.agent_version or Version("0.0.0")
         lib = self.library
         self.components["library"] = lib.version
         self.components[lib.name] = lib.version