test: Rebalance parallel test scheduling and trim fixture overhead

pyproject.toml

@@ -263,7 +263,7 @@ builtins-ignorelist = ["id"]
 known-first-party = ["crawlee"]
 
 [tool.pytest.ini_options]
-addopts = "-r a --verbose"
+addopts = "-r a --verbose --dist worksteal"
 asyncio_default_fixture_loop_scope = "function"
 asyncio_mode = "auto"
 timeout = 1800
@@ -339,7 +339,6 @@ shell = "uv run ruff check --fix && uv run ruff format"
 [tool.poe.tasks.unit-tests]
 shell = """
 uv run pytest \
-    --numprocesses=1 \
     -m "run_alone" \
     tests/unit && \
 uv run pytest \
@@ -349,9 +348,10 @@ uv run pytest \
 """
 
 [tool.poe.tasks.unit-tests-cov]
+# Falls back to the default core with a warning on Python < 3.12.
+env = { COVERAGE_CORE = "sysmon" }
 shell = """
 uv run pytest \
-    --numprocesses=1 \
     -m "run_alone" \
     --cov=src/crawlee \
     --cov-report=xml:coverage-unit.xml \

tests/unit/conftest.py

@@ -103,23 +103,37 @@ def _set_crawler_log_level(pytestconfig: pytest.Config, monkeypatch: pytest.Monk
         monkeypatch.setattr(_log_config, 'get_configured_log_level', lambda: getattr(logging, loglevel.upper()))
 
 
-@pytest.fixture
-async def proxy_info(unused_tcp_port: int) -> ProxyInfo:
+def _proxy_info(port: int) -> ProxyInfo:
+    """Describe a local proxy listening on `port`, authenticated with fixed throwaway credentials."""
     username = 'user'
     password = 'pass'
 
     return ProxyInfo(
-        url=f'http://{username}:{password}@127.0.0.1:{unused_tcp_port}',
+        url=f'http://{username}:{password}@127.0.0.1:{port}',
         scheme='http',
         hostname='127.0.0.1',
-        port=unused_tcp_port,
+        port=port,
         username=username,
         password=password,
     )
 
 
-@pytest.fixture
-async def proxy(proxy_info: ProxyInfo) -> AsyncGenerator[ProxyInfo, None]:
+# Session-scoped because a `Proxy` teardown blocks for up to a second on its acceptor's
+# `selector.select(timeout=1)`. Sync because a session-scoped async fixture is incompatible with
+# `asyncio_default_fixture_loop_scope = "function"`.
+@pytest.fixture(scope='session')
+def proxy_info(unused_tcp_port_factory: Callable[[], int]) -> ProxyInfo:
+    return _proxy_info(unused_tcp_port_factory())
+
+
+@pytest.fixture(scope='session')
+def disabled_proxy_info(unused_tcp_port_factory: Callable[[], int]) -> ProxyInfo:
+    """Describe the disabled proxy, which needs a port of its own now that both servers outlive a test."""
+    return _proxy_info(unused_tcp_port_factory())
+
+
+@pytest.fixture(scope='session')
+def proxy(proxy_info: ProxyInfo) -> Iterator[ProxyInfo]:
     with Proxy(
         [
             '--hostname',
@@ -139,24 +153,24 @@ async def proxy(proxy_info: ProxyInfo) -> AsyncGenerator[ProxyInfo, None]:
         yield proxy_info
 
 
-@pytest.fixture
-async def disabled_proxy(proxy_info: ProxyInfo) -> AsyncGenerator[ProxyInfo, None]:
+@pytest.fixture(scope='session')
+def disabled_proxy(disabled_proxy_info: ProxyInfo) -> Iterator[ProxyInfo]:
     with Proxy(
         [
             '--hostname',
-            proxy_info.hostname,
+            disabled_proxy_info.hostname,
             '--port',
-            str(proxy_info.port),
+            str(disabled_proxy_info.port),
             '--basic-auth',
-            f'{proxy_info.username}:{proxy_info.password}',
+            f'{disabled_proxy_info.username}:{disabled_proxy_info.password}',
             '--disable-http-proxy',
             '--num-workers',
             '1',
             '--num-acceptors',
             '1',
         ]
     ):
-        yield proxy_info
+        yield disabled_proxy_info
 
 
 @pytest.fixture(scope='session')

tests/unit/crawlers/_playwright/test_utils.py

@@ -20,12 +20,13 @@ async def test_infinite_scroll_on_dynamic_page(server_url: URL) -> None:
         # Get data with manual scrolling
         await page.goto(target_url)
 
-        manual_items = []
-        for _ in range(4):
-            items = await page.query_selector_all('.item')
-            manual_items = items
+        # The page appends one batch of items per scroll, for three batches. Wait for each batch to land
+        # instead of sleeping, so the baseline costs as long as the page takes rather than a fixed 4 seconds.
+        manual_items = await page.query_selector_all('.item')
+        for _ in range(3):
             await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
-            await page.wait_for_timeout(1000)
+            await page.wait_for_function(f'document.querySelectorAll(".item").length > {len(manual_items)}')
+            manual_items = await page.query_selector_all('.item')
 
         # Reset page
         await page.close()