From 79be4fd613d709f91b1f12b24eecbe37429838d1 Mon Sep 17 00:00:00 2001
From: Vlada Dusek <v.dusek96@gmail.com>
Date: Thu, 27 Aug 2026 12:14:08 +0200
Subject: [PATCH] test: Rebalance parallel test scheduling and trim fixture
 overhead

---
 pyproject.toml                                |  6 +--
 tests/unit/conftest.py                        | 38 +++++++++++++------
 tests/unit/crawlers/_playwright/test_utils.py | 11 +++---
 3 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 997fe988c5..c22a116aa6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -263,7 +263,7 @@ builtins-ignorelist = ["id"]
 known-first-party = ["crawlee"]
 
 [tool.pytest.ini_options]
-addopts = "-r a --verbose"
+addopts = "-r a --verbose --dist worksteal"
 asyncio_default_fixture_loop_scope = "function"
 asyncio_mode = "auto"
 timeout = 1800
@@ -339,7 +339,6 @@ shell = "uv run ruff check --fix && uv run ruff format"
 [tool.poe.tasks.unit-tests]
 shell = """
 uv run pytest \
-    --numprocesses=1 \
     -m "run_alone" \
     tests/unit && \
 uv run pytest \
@@ -349,9 +348,10 @@ uv run pytest \
 """
 
 [tool.poe.tasks.unit-tests-cov]
+# Falls back to the default core with a warning on Python < 3.12.
+env = { COVERAGE_CORE = "sysmon" }
 shell = """
 uv run pytest \
-    --numprocesses=1 \
     -m "run_alone" \
     --cov=src/crawlee \
     --cov-report=xml:coverage-unit.xml \
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
index 1b2365056d..281fe8c9b6 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/conftest.py
@@ -103,23 +103,37 @@ def _set_crawler_log_level(pytestconfig: pytest.Config, monkeypatch: pytest.Monk
         monkeypatch.setattr(_log_config, 'get_configured_log_level', lambda: getattr(logging, loglevel.upper()))
 
 
-@pytest.fixture
-async def proxy_info(unused_tcp_port: int) -> ProxyInfo:
+def _proxy_info(port: int) -> ProxyInfo:
+    """Describe a local proxy listening on `port`, authenticated with fixed throwaway credentials."""
     username = 'user'
     password = 'pass'
 
     return ProxyInfo(
-        url=f'http://{username}:{password}@127.0.0.1:{unused_tcp_port}',
+        url=f'http://{username}:{password}@127.0.0.1:{port}',
         scheme='http',
         hostname='127.0.0.1',
-        port=unused_tcp_port,
+        port=port,
         username=username,
         password=password,
     )
 
 
-@pytest.fixture
-async def proxy(proxy_info: ProxyInfo) -> AsyncGenerator[ProxyInfo, None]:
+# Session-scoped because a `Proxy` teardown blocks for up to a second on its acceptor's
+# `selector.select(timeout=1)`. Sync because a session-scoped async fixture is incompatible with
+# `asyncio_default_fixture_loop_scope = "function"`.
+@pytest.fixture(scope='session')
+def proxy_info(unused_tcp_port_factory: Callable[[], int]) -> ProxyInfo:
+    return _proxy_info(unused_tcp_port_factory())
+
+
+@pytest.fixture(scope='session')
+def disabled_proxy_info(unused_tcp_port_factory: Callable[[], int]) -> ProxyInfo:
+    """Describe the disabled proxy, which needs a port of its own now that both servers outlive a test."""
+    return _proxy_info(unused_tcp_port_factory())
+
+
+@pytest.fixture(scope='session')
+def proxy(proxy_info: ProxyInfo) -> Iterator[ProxyInfo]:
     with Proxy(
         [
             '--hostname',
@@ -139,16 +153,16 @@ async def proxy(proxy_info: ProxyInfo) -> AsyncGenerator[ProxyInfo, None]:
         yield proxy_info
 
 
-@pytest.fixture
-async def disabled_proxy(proxy_info: ProxyInfo) -> AsyncGenerator[ProxyInfo, None]:
+@pytest.fixture(scope='session')
+def disabled_proxy(disabled_proxy_info: ProxyInfo) -> Iterator[ProxyInfo]:
     with Proxy(
         [
             '--hostname',
-            proxy_info.hostname,
+            disabled_proxy_info.hostname,
             '--port',
-            str(proxy_info.port),
+            str(disabled_proxy_info.port),
             '--basic-auth',
-            f'{proxy_info.username}:{proxy_info.password}',
+            f'{disabled_proxy_info.username}:{disabled_proxy_info.password}',
             '--disable-http-proxy',
             '--num-workers',
             '1',
@@ -156,7 +170,7 @@ async def disabled_proxy(proxy_info: ProxyInfo) -> AsyncGenerator[ProxyInfo, Non
             '1',
         ]
     ):
-        yield proxy_info
+        yield disabled_proxy_info
 
 
 @pytest.fixture(scope='session')
diff --git a/tests/unit/crawlers/_playwright/test_utils.py b/tests/unit/crawlers/_playwright/test_utils.py
index ea59f610db..acd56cb39a 100644
--- a/tests/unit/crawlers/_playwright/test_utils.py
+++ b/tests/unit/crawlers/_playwright/test_utils.py
@@ -20,12 +20,13 @@ async def test_infinite_scroll_on_dynamic_page(server_url: URL) -> None:
         # Get data with manual scrolling
         await page.goto(target_url)
 
-        manual_items = []
-        for _ in range(4):
-            items = await page.query_selector_all('.item')
-            manual_items = items
+        # The page appends one batch of items per scroll, for three batches. Wait for each batch to land
+        # instead of sleeping, so the baseline costs as long as the page takes rather than a fixed 4 seconds.
+        manual_items = await page.query_selector_all('.item')
+        for _ in range(3):
             await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
-            await page.wait_for_timeout(1000)
+            await page.wait_for_function(f'document.querySelectorAll(".item").length > {len(manual_items)}')
+            manual_items = await page.query_selector_all('.item')
 
         # Reset page
         await page.close()