diff --git a/src/providers/__init__.py b/src/providers/__init__.py index fce92636..d8f769f7 100644 --- a/src/providers/__init__.py +++ b/src/providers/__init__.py @@ -360,7 +360,42 @@ class is what carries the wire contract. from .anthropic_provider import AnthropicProvider from .minimax_provider import MinimaxProvider - return isinstance(provider, (AnthropicProvider, MinimaxProvider)) + return isinstance(unwrap_provider(provider), (AnthropicProvider, MinimaxProvider)) + + +def unwrap_provider(provider: Any) -> Any: + """Follow a delegating wrapper down to the provider that owns the wire. + + ``FusionProvider`` wraps a base provider to substitute image blocks; its + MRO is ``(FusionProvider, object)``, so it is not an instance of anything + the wire tests check. An ``isinstance`` test therefore reports + "OpenAI-compatible" for EVERY fusion model — including one whose base is + Anthropic or Minimax. + + That is not cosmetic. Both things :func:`is_anthropic_wire` decides are + hard failures when decided wrongly: reasoning effort would go out as a + top-level ``reasoning_effort`` field, which the Anthropic API rejects with + ``400 invalid_request_error — reasoning_effort: Extra inputs are not + permitted``, and the system prompt would be prepended as a + ``{"role": "system"}`` message instead of passed as the ``system`` kwarg. + A fusion over ``anthropic:claude-opus-5`` is expressible today + (``fusion=anthropic:claude-opus-5+openai:gpt-5.6-luna``) and would fail on + its first request. + + Bounded rather than a ``while True``: a cycle in ``inner`` would otherwise + hang the wire check. Depth 8 is far beyond any real nesting (one wrapper + today) and any excess simply falls back to the outermost object, which is + the pre-existing behaviour. + """ + seen: set[int] = set() + current = provider + for _ in range(8): + inner = getattr(current, "inner", None) + if inner is None or inner is current or id(inner) in seen: + return current + seen.add(id(current)) + current = inner + return current #: The provider ids whose classes :func:`is_anthropic_wire` matches. Kept @@ -468,6 +503,7 @@ def resolve_api_key( "get_provider_class", "get_provider_info", "is_anthropic_wire", + "unwrap_provider", "canonical_provider_name", "provider_env_vars", "provider_has_credentials", diff --git a/src/providers/base.py b/src/providers/base.py index 7c8ab95d..bbf7c282 100644 --- a/src/providers/base.py +++ b/src/providers/base.py @@ -62,6 +62,44 @@ class BaseProvider(ABC): #: OpenRouter is intentionally NOT covered. is_deepseek: bool = False + #: Reasoning-effort levels this provider's API actually accepts, or + #: ``None`` for "the whole clawcodex ladder passes through untouched" + #: (the default, and correct for OpenAI/OpenRouter, which take all five). + #: See :meth:`normalize_reasoning_effort`. + supported_reasoning_efforts: tuple[str, ...] | None = None + + #: Maps a clawcodex level this provider does NOT accept onto the nearest + #: one it does. Only consulted for levels absent from + #: ``supported_reasoning_efforts``. + reasoning_effort_aliases: dict[str, str] = {} + + def normalize_reasoning_effort(self, effort: str | None) -> str | None: + """Translate a clawcodex effort level into this provider's vocabulary. + + clawcodex exposes ``low | medium | high | xhigh | max``, but that + ladder is Anthropic's and not every API shares it. Sending a level a + provider does not know is not obviously harmful — nobody 400s on it — + which is exactly why it needs handling: the provider ignores the field + and silently applies its own default, so the user gets a level they + did not ask for and no diagnostic saying so. + + The damaging direction is DOWNWARD. A user who selects ``xhigh`` + against a provider whose ladder tops out differently is asking for + more than ``high``; if the value is dropped they get the default, + which is typically ``high`` — strictly less than requested, on the + setting people reach for precisely when a task is hard. + + Default is identity: providers that accept the full ladder are + unaffected, and a provider that has not declared a vocabulary keeps + the pass-through behaviour it had before this hook existed. + """ + if effort is None: + return None + supported = self.supported_reasoning_efforts + if not supported or effort in supported: + return effort + return self.reasoning_effort_aliases.get(effort, effort) + def __init__( self, api_key: str, base_url: Optional[str] = None, model: Optional[str] = None ): diff --git a/src/providers/deepseek_provider.py b/src/providers/deepseek_provider.py index eca51bdd..a4f9783b 100644 --- a/src/providers/deepseek_provider.py +++ b/src/providers/deepseek_provider.py @@ -30,6 +30,25 @@ class DeepSeekProvider(OpenAICompatibleProvider): #: cache (see ``query._split_system_prompt_blocks``). is_deepseek = True + #: DeepSeek's OpenAI-format thinking vocabulary is ``low | high | max`` + #: (api-docs.deepseek.com/guides/thinking_mode). Thinking is ON by + #: default at ``high``. + #: + #: The API does not VALIDATE this field — probed 2026-08-03 against + #: ``deepseek-v4-flash``, every one of ``low / medium / high / xhigh / + #: max / minimal`` returned 200, and so did a value the docs never list. + #: So an unsupported level is not an error, it is silently discarded and + #: the default (``high``) applies. Without the mapping below, ``xhigh`` + #: — chosen precisely when a task is hard — quietly delivered LESS + #: reasoning than ``max``, which DeepSeek does support. + supported_reasoning_efforts = ("low", "high", "max") + + #: ``medium`` has no DeepSeek equivalent and already behaved as ``high`` + #: (unknown → default), so this makes the existing behaviour explicit + #: rather than changing it. ``xhigh`` is the real fix: it means "above + #: high", and ``max`` is the only DeepSeek level that is. + reasoning_effort_aliases = {"medium": "high", "xhigh": "max", "minimal": "low"} + def __init__( self, api_key: str, base_url: Optional[str] = None, model: Optional[str] = None ): diff --git a/src/query/query.py b/src/query/query.py index ab0c90e5..39f0f443 100644 --- a/src/query/query.py +++ b/src/query/query.py @@ -1217,6 +1217,37 @@ async def _call_model_sync( getattr(provider, "model", None) or call_kwargs.get("model"), clamp_xhigh=False, ) + # Translate onto the provider's own vocabulary before it hits the + # wire. The ladder above is Anthropic's; a provider that does not + # share it silently DISCARDS the unknown level and applies its own + # default, so the request goes out looking fine and the user gets a + # level they did not choose. Identity for providers that take the + # full ladder (OpenAI, OpenRouter) — see + # ``BaseProvider.normalize_reasoning_effort``. + # + # No explicit unwrap here, deliberately: ``FusionProvider.__getattr__`` + # already delegates unknown attributes to its base, so this lookup + # lands on the BASE provider's method and its vocabulary. (Contrast + # ``is_anthropic_wire``, which cannot delegate — an ``isinstance`` + # test sees the wrapper's class and needs ``unwrap_provider``.) The + # delegation is pinned by a test rather than left incidental. + # The result is VALIDATED before use, not trusted. This is a duck-typed + # ``getattr`` on whatever object the caller passed, and providers are + # not all real ``BaseProvider`` subclasses — mocks, gateway shims and + # third-party wrappers all reach here. A ``MagicMock`` in particular + # answers every attribute with a callable returning another Mock, so an + # unguarded assignment writes a ```` repr into the + # request body. Anything that is not a plain string on the known ladder + # is discarded in favour of the level we already resolved. + if resolved_effort is not None: + _normalize = getattr(provider, "normalize_reasoning_effort", None) + if callable(_normalize): + try: + _mapped = _normalize(resolved_effort) + except Exception: # noqa: BLE001 — never fail a request on this + _mapped = None + if isinstance(_mapped, str) and _mapped in VALID_THINKING_EFFORT_LEVELS: + resolved_effort = _mapped if resolved_effort is not None: extra_body = dict(call_kwargs.get("extra_body") or {}) extra_body.setdefault("reasoning_effort", resolved_effort) diff --git a/tests/test_deepseek_effort_vocabulary.py b/tests/test_deepseek_effort_vocabulary.py new file mode 100644 index 00000000..839505d0 --- /dev/null +++ b/tests/test_deepseek_effort_vocabulary.py @@ -0,0 +1,230 @@ +"""Reasoning effort must reach the wire in the PROVIDER's own vocabulary. + +clawcodex exposes ``low | medium | high | xhigh | max`` — Anthropic's ladder. +DeepSeek's OpenAI-format thinking mode accepts only ``low | high | max`` +(api-docs.deepseek.com/guides/thinking_mode), and it does NOT validate the +field: probed 2026-08-03 against ``deepseek-v4-flash``, every one of +low/medium/high/xhigh/max/minimal returned 200. An unsupported level is +therefore silently discarded and the provider's default (``high``) applies — +the request looks fine and the user gets a level they did not ask for. + +The damaging direction is downward: ``xhigh`` means "more than high", and +dropping it delivers ``high`` when ``max`` was available — on the setting +people reach for precisely when a task is hard. + +Also covers the wrapper case, because that is how the fusion models are +built: a ``FusionProvider`` must consult its BASE provider's vocabulary and +its base's wire family, not the wrapper's own (empty) defaults. +""" + +from __future__ import annotations + +import asyncio +import types + +import pytest + +from src.providers import is_anthropic_wire, unwrap_provider +from src.providers.anthropic_provider import AnthropicProvider +from src.providers.deepseek_provider import DeepSeekProvider +from src.providers.fusion_models import FusionModel, ModelRef +from src.providers.fusion_provider import FusionProvider +from src.providers.openrouter_provider import OpenRouterProvider + + +def _fusion(base): + return FusionProvider( + base, + FusionModel( + name="deepseek-v4-flash-luna", + base=ModelRef("deepseek", "deepseek-v4-flash"), + vision=ModelRef("openai", "gpt-5.6-luna"), + ), + ) + + +def _deepseek(): + return DeepSeekProvider(api_key="k", model="deepseek-v4-flash") + + +# --- vocabulary translation ------------------------------------------------ + + +@pytest.mark.parametrize( + "requested,on_the_wire", + [ + ("low", "low"), + ("high", "high"), + ("max", "max"), + # No DeepSeek equivalent. Already behaved as `high` (unknown -> + # default), so this makes the existing behaviour explicit. + ("medium", "high"), + # The real fix: "above high" must not silently become "high". + ("xhigh", "max"), + ], +) +def test_deepseek_translates_onto_its_own_ladder(requested, on_the_wire): + assert _deepseek().normalize_reasoning_effort(requested) == on_the_wire + + +@pytest.mark.parametrize("effort", ["low", "medium", "high", "xhigh", "max"]) +def test_providers_taking_the_full_ladder_are_untouched(effort): + """The default must be identity. OpenRouter accepts all five, and + rewriting them there would be inventing a restriction that does not + exist — the failure this fix is meant to prevent, inverted.""" + p = OpenRouterProvider(api_key="k", model="openai/gpt-5.6-luna") + assert p.normalize_reasoning_effort(effort) == effort + + +def test_a_provider_with_no_declared_vocabulary_passes_through(): + """Pre-existing behaviour for anything that has not opted in.""" + + class _Bare: + supported_reasoning_efforts = None + reasoning_effort_aliases: dict[str, str] = {} + normalize_reasoning_effort = ( + DeepSeekProvider.normalize_reasoning_effort # unbound, shared impl + ) + + assert _Bare().normalize_reasoning_effort("xhigh") == "xhigh" + assert _Bare().normalize_reasoning_effort(None) is None + + +# --- wrapper transparency -------------------------------------------------- + + +def test_unwrap_reaches_the_base_provider(): + base = _deepseek() + assert unwrap_provider(_fusion(base)) is base + assert unwrap_provider(base) is base + + +def test_wire_family_follows_the_fusion_BASE(): + """`is_anthropic_wire` used an isinstance test, and FusionProvider's MRO + is (FusionProvider, object) — so EVERY fusion model reported + OpenAI-compatible, including one whose base is Anthropic. That sends a + top-level `reasoning_effort` to the Anthropic API, which rejects it with + `400 ... Extra inputs are not permitted`, and prepends the system prompt + as a message instead of passing the `system` kwarg. + """ + assert is_anthropic_wire(_fusion(_deepseek())) is False + anthropic_base = AnthropicProvider(api_key="k", model="claude-opus-5") + assert is_anthropic_wire(_fusion(anthropic_base)) is True + assert is_anthropic_wire(anthropic_base) is True + + +def test_fusion_delegates_the_vocabulary_lookup_to_its_base(): + """The wire boundary looks `normalize_reasoning_effort` up on whatever + provider it was handed, with no explicit unwrap — that is only correct + because `FusionProvider.__getattr__` delegates to the base. Pinned here + so the delegation is a tested property rather than an incidental one: if + the wrapper ever grows its own attribute (or the `__getattr__` guard list + changes), a fusion model would silently start sending the untranslated + clawcodex level. + """ + fused = _fusion(_deepseek()) + assert fused.normalize_reasoning_effort("xhigh") == "max" + assert fused.supported_reasoning_efforts == ("low", "high", "max") + + +def test_unwrap_terminates_on_a_self_referential_wrapper(): + """A cycle must not hang the wire check.""" + + class _Loop: + pass + + a = _Loop() + a.inner = a + assert unwrap_provider(a) is a + + b, c = _Loop(), _Loop() + b.inner, c.inner = c, b + assert unwrap_provider(b) in (b, c) # terminates; which end is arbitrary + + +# --- the whole chain, at the wire boundary --------------------------------- + + +def _wire_kwargs(provider, effort): + """Drive the real `_call_model_sync` and capture what it would send.""" + import importlib + import sys + + importlib.import_module("src.query.query") + Q = sys.modules["src.query.query"] + + captured: dict = {} + + def _fake_stream(*args, **kwargs): + captured.update(kwargs) + return types.SimpleNamespace( + content="ok", model="m", usage={}, finish_reason="stop", + tool_uses=None, reasoning_content=None, raw_content_blocks=None, + ) + + target = unwrap_provider(provider) + target.chat_stream_response = _fake_stream # type: ignore[method-assign] + try: + asyncio.run( + Q._call_model_sync( + provider=provider, + messages=[{"role": "user", "content": "hi"}], + system_prompt="s", + tools=[], + thinking_effort=effort, + ) + ) + except Exception: # noqa: BLE001 — only the captured kwargs matter + pass + return captured + + +@pytest.mark.parametrize( + "requested,on_the_wire", + [("low", "low"), ("medium", "high"), ("high", "high"), + ("xhigh", "max"), ("max", "max")], +) +def test_effort_reaches_the_deepseek_wire_translated(requested, on_the_wire): + kwargs = _wire_kwargs(_deepseek(), requested) + assert (kwargs.get("extra_body") or {}).get("reasoning_effort") == on_the_wire + + +@pytest.mark.parametrize( + "requested,on_the_wire", + [("medium", "high"), ("xhigh", "max"), ("max", "max")], +) +def test_a_fusion_model_uses_its_BASE_providers_vocabulary(requested, on_the_wire): + """The fusion wrapper declares no vocabulary of its own; without + unwrapping, every fusion model would send the raw clawcodex level.""" + kwargs = _wire_kwargs(_fusion(_deepseek()), requested) + assert (kwargs.get("extra_body") or {}).get("reasoning_effort") == on_the_wire + + +def test_a_bogus_normalize_hook_cannot_corrupt_the_wire(): + """The wire boundary duck-types this hook via `getattr`, and not every + provider-shaped object is a real BaseProvider — mocks, gateway shims and + third-party wrappers all reach it. A MagicMock answers EVERY attribute + with a callable returning another Mock, which without validation writes a + `` repr into the request body as the effort level. + """ + from unittest.mock import MagicMock + + provider = MagicMock() + provider.model = "some-model" + # A bare MagicMock also answers `.inner` with a Mock, which would send + # `unwrap_provider` walking a chain that never ends in a real object. + # Pin it so this test exercises the normalize guard, not the unwrap loop. + provider.inner = None + kwargs = _wire_kwargs(provider, "xhigh") + assert (kwargs.get("extra_body") or {}).get("reasoning_effort") == "xhigh" + + +def test_a_raising_normalize_hook_falls_back_instead_of_failing_the_turn(): + base = _deepseek() + + def _boom(_effort): + raise RuntimeError("provider bug") + + base.normalize_reasoning_effort = _boom # type: ignore[method-assign] + kwargs = _wire_kwargs(base, "max") + assert (kwargs.get("extra_body") or {}).get("reasoning_effort") == "max"