From 7795b294268dfe3f04bc71900570f4a5ae50affa Mon Sep 17 00:00:00 2001 From: AbdullahFageeh Date: Fri, 31 Jul 2026 21:35:51 +0300 Subject: [PATCH] fix: support groq compound defaults Route groq-prefixed models through the OpenAI-compatible client, apply Groq token-limit metadata, and skip local tool injection for compound models. Add regression coverage for routing, token limits, and compound local-tool behavior. Co-Authored-By: Oz --- rust/crates/api/src/providers/mod.rs | 58 ++++++++++++++++++++++++ rust/crates/rusty-claude-cli/src/main.rs | 24 ++++++++-- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/rust/crates/api/src/providers/mod.rs b/rust/crates/api/src/providers/mod.rs index 2524e5520a..d07564c910 100644 --- a/rust/crates/api/src/providers/mod.rs +++ b/rust/crates/api/src/providers/mod.rs @@ -270,6 +270,16 @@ pub fn metadata_for_model(model: &str) -> Option { default_base_url: openai_compat::DEFAULT_OPENAI_BASE_URL, }); } + // Groq compound systems and Groq-namespaced model ids are served through + // the OpenAI-compatible transport selected by OPENAI_BASE_URL. + if canonical.starts_with("groq/") { + return Some(ProviderMetadata { + provider: ProviderKind::OpenAi, + auth_env: "OPENAI_API_KEY", + base_url_env: "OPENAI_BASE_URL", + default_base_url: openai_compat::DEFAULT_OPENAI_BASE_URL, + }); + } // Alibaba DashScope compatible-mode endpoint. Routes qwen/* and bare // qwen-* model names (qwen-max, qwen-plus, qwen-turbo, qwen-qwq, etc.) // to the OpenAI-compat client pointed at DashScope's /compatible-mode/v1. @@ -666,6 +676,13 @@ pub fn model_token_limit(model: &str) -> Option { max_output_tokens: 16_384, context_window_tokens: 256_000, }), + // Groq Llama 4 models accept tool calling but cap completions below + // Claw's 64k fallback heuristic. Encode the provider limit so prompts + // fit inside Groq's request validation and TPM budget. + "llama-4-scout-17b-16e-instruct" => Some(ModelTokenLimit { + max_output_tokens: 8_192, + context_window_tokens: 131_072, + }), "qwen-max" => Some(ModelTokenLimit { max_output_tokens: 8_192, context_window_tokens: 131_072, @@ -674,6 +691,14 @@ pub fn model_token_limit(model: &str) -> Option { max_output_tokens: 8_192, context_window_tokens: 131_072, }), + // Groq compound systems are exposed through an OpenAI-compatible API + // but cap completions at 8,192 output tokens. Without this metadata + // Claw uses its 64k fallback heuristic and the provider rejects the + // request before generation starts. + "compound" | "compound-mini" => Some(ModelTokenLimit { + max_output_tokens: 8_192, + context_window_tokens: 131_072, + }), _ => None, } } @@ -1080,6 +1105,18 @@ mod tests { assert_eq!(kind, ProviderKind::OpenAi); } + #[test] + fn groq_prefix_routes_to_openai_not_anthropic() { + let meta = super::metadata_for_model("groq/compound") + .expect("groq/ prefix must resolve to OpenAI-compatible metadata"); + assert_eq!(meta.provider, ProviderKind::OpenAi); + assert_eq!(meta.auth_env, "OPENAI_API_KEY"); + assert_eq!(meta.base_url_env, "OPENAI_BASE_URL"); + + let kind = detect_provider_kind("groq/compound"); + assert_eq!(kind, ProviderKind::OpenAi); + } + #[test] fn qwen_prefix_routes_to_dashscope_not_anthropic() { // User request from Discord #clawcode-get-help: web3g wants to use @@ -1379,6 +1416,27 @@ mod tests { ); } + #[test] + fn returns_context_window_metadata_for_groq_compound_models() { + let compound = + model_token_limit("groq/compound").expect("groq/compound should have token limits"); + assert_eq!(compound.max_output_tokens, 8_192); + assert_eq!(compound.context_window_tokens, 131_072); + + let compound_mini = model_token_limit("groq/compound-mini") + .expect("groq/compound-mini should have token limits"); + assert_eq!(compound_mini.max_output_tokens, 8_192); + assert_eq!(compound_mini.context_window_tokens, 131_072); + } + + #[test] + fn returns_context_window_metadata_for_groq_llama4_scout() { + let scout = model_token_limit("groq/meta-llama/llama-4-scout-17b-16e-instruct") + .expect("groq llama 4 scout should have token limits"); + assert_eq!(scout.max_output_tokens, 8_192); + assert_eq!(scout.context_window_tokens, 131_072); + } + #[test] fn preflight_blocks_oversized_requests_for_kimi_models() { let request = MessageRequest { diff --git a/rust/crates/rusty-claude-cli/src/main.rs b/rust/crates/rusty-claude-cli/src/main.rs index 665ce632cf..ab719b3e72 100644 --- a/rust/crates/rusty-claude-cli/src/main.rs +++ b/rust/crates/rusty-claude-cli/src/main.rs @@ -12624,6 +12624,16 @@ fn resolve_cli_auth_source_for_cwd() -> Result { resolve_startup_auth_source(|| Ok(None)) } +fn model_supports_local_tools(model: &str) -> bool { + !matches!( + api::resolve_model_alias(model) + .rsplit('/') + .next() + .unwrap_or(model), + "compound" | "compound-mini" + ) +} + impl ApiClient for AnthropicRuntimeClient { #[allow(clippy::too_many_lines)] fn stream(&mut self, request: ApiRequest) -> Result, RuntimeError> { @@ -12631,15 +12641,15 @@ impl ApiClient for AnthropicRuntimeClient { progress_reporter.mark_model_phase(); } let is_post_tool = request_ends_with_tool_result(&request); + let enable_local_tools = self.enable_tools && model_supports_local_tools(&self.model); let message_request = MessageRequest { model: self.model.clone(), max_tokens: max_tokens_for_model(&self.model), messages: convert_messages(&request.messages), system: (!request.system_prompt.is_empty()).then(|| request.system_prompt.join("\n\n")), - tools: self - .enable_tools + tools: enable_local_tools .then(|| filter_tool_specs(&self.tool_registry, self.allowed_tools.as_ref())), - tool_choice: self.enable_tools.then_some(ToolChoice::Auto), + tool_choice: enable_local_tools.then_some(ToolChoice::Auto), stream: true, reasoning_effort: self.reasoning_effort.clone(), ..Default::default() @@ -19437,6 +19447,14 @@ UU conflicted.rs", } } + #[test] + fn groq_compound_disables_local_tools() { + assert!(!super::model_supports_local_tools("groq/compound")); + assert!(!super::model_supports_local_tools("groq/compound-mini")); + assert!(super::model_supports_local_tools("sonnet")); + assert!(super::model_supports_local_tools("llama-3.1-8b-instant")); + } + #[test] fn stub_commands_absent_from_repl_completions() { let candidates =