llm/providers/
generic.rs

1#![doc = include_str!(concat!(env!("OUT_DIR"), "/docs/generic.md"))]
2
3use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderName, HeaderValue};
4use serde::Serialize;
5use std::borrow::Cow;
6use std::future::Future;
7use std::time::Duration;
8use utils::variables::Vars;
9
10use crate::catalog::Provider;
11use crate::provider::{display_name, validate_reasoning};
12use crate::providers::anthropic::mappers::build_request as build_messages_request;
13use crate::providers::anthropic::streaming::decode_messages;
14use crate::providers::anthropic::types::StreamingRequest;
15use crate::providers::http::{SseData, anthropic_code, http_client, post_sse, responses_code};
16pub use crate::providers::openai::chat::ChatRequestPolicy;
17use crate::providers::openai::chat::build_chat_request;
18use crate::providers::openai::chat::streaming::decode_sse_chunk;
19use crate::providers::openai::chat::types::CompatibleChatRequest;
20use crate::providers::openai::responses::mappers::build_request as build_responses_request;
21use crate::providers::openai::responses::streaming::decode_responses;
22use crate::providers::openai::wire::PromptCacheKeySource;
23use crate::providers::response_stream::{OpenedStream, error_stream, response_stream};
24use crate::tool_schema::normalize_for_moonshot;
25use crate::{
26    Context, LlmError, LlmModel, LlmResponseStream, ProviderAuthMode, ProviderConnectionConfig, Result,
27    StreamingModelProvider,
28};
29
30pub use crate::providers::openai::responses::mappers::ResponsesRequestPolicy;
31
32pub struct ProviderConfig {
33    pub provider: Provider,
34    pub api_base: Option<&'static str>,
35    pub api_base_env: Option<&'static str>,
36    pub default_model: &'static str,
37    pub api: Api,
38}
39
40pub enum Api {
41    ChatCompletions(ChatRequestPolicy),
42    Responses(ResponsesRequestPolicy),
43    Messages,
44}
45
46pub const ANTHROPIC: ProviderConfig = ProviderConfig {
47    provider: Provider::Anthropic,
48    api_base: Some("https://api.anthropic.com"),
49    api_base_env: None,
50    default_model: "claude-sonnet-4-5-20250929",
51    api: Api::Messages,
52};
53
54pub const OPENAI: ProviderConfig = ProviderConfig {
55    provider: Provider::Openai,
56    api_base: Some("https://api.openai.com/v1"),
57    api_base_env: None,
58    default_model: "gpt-4.1",
59    api: Api::Responses(ResponsesRequestPolicy::OPENAI),
60};
61
62pub const XIAOMI: ProviderConfig = ProviderConfig {
63    provider: Provider::Xiaomi,
64    api_base: Some("https://api.xiaomimimo.com/v1"),
65    api_base_env: None,
66    default_model: "mimo-v2.6-pro",
67    api: Api::Responses(ResponsesRequestPolicy::XIAOMI),
68};
69
70pub const DEEPSEEK: ProviderConfig = ProviderConfig {
71    provider: Provider::DeepSeek,
72    api_base: Some("https://api.deepseek.com"),
73    api_base_env: None,
74    default_model: "deepseek-v4-flash",
75    api: Api::ChatCompletions(ChatRequestPolicy::DEFAULT),
76};
77
78pub const MOONSHOT: ProviderConfig = ProviderConfig {
79    provider: Provider::Moonshot,
80    api_base: Some("https://api.moonshot.ai/v1"),
81    api_base_env: None,
82    default_model: "moonshot-v1-8k",
83    api: Api::ChatCompletions(ChatRequestPolicy {
84        tool_schema_transform: Some(normalize_for_moonshot),
85        ..ChatRequestPolicy::DEFAULT
86    }),
87};
88
89pub const ZAI: ProviderConfig = ProviderConfig {
90    provider: Provider::ZAi,
91    api_base: Some("https://api.z.ai/api/coding/paas/v4"),
92    api_base_env: None,
93    default_model: "GLM-4.6",
94    api: Api::ChatCompletions(ChatRequestPolicy::DEFAULT),
95};
96
97pub const AZURE_FOUNDRY: ProviderConfig = ProviderConfig {
98    provider: Provider::AzureFoundry,
99    api_base: None,
100    api_base_env: None,
101    default_model: "gpt-5.5",
102    api: Api::ChatCompletions(ChatRequestPolicy {
103        prompt_cache_key: PromptCacheKeySource::Prefix,
104        ..ChatRequestPolicy::DEFAULT
105    }),
106};
107
108pub const FIREWORKS: ProviderConfig = ProviderConfig {
109    provider: Provider::Fireworks,
110    api_base: Some("https://api.fireworks.ai/inference/v1"),
111    api_base_env: None,
112    default_model: "accounts/fireworks/models/glm-5p1",
113    api: Api::ChatCompletions(ChatRequestPolicy {
114        prompt_cache_key: PromptCacheKeySource::SessionAffinity,
115        ..ChatRequestPolicy::DEFAULT
116    }),
117};
118
119pub const GEMINI: ProviderConfig = ProviderConfig {
120    provider: Provider::Gemini,
121    api_base: Some("https://generativelanguage.googleapis.com/v1beta/openai"),
122    api_base_env: None,
123    default_model: "",
124    api: Api::ChatCompletions(ChatRequestPolicy::DEFAULT),
125};
126
127pub const OPENROUTER: ProviderConfig = ProviderConfig {
128    provider: Provider::OpenRouter,
129    api_base: Some("https://openrouter.ai/api/v1"),
130    api_base_env: None,
131    default_model: "",
132    api: Api::ChatCompletions(ChatRequestPolicy {
133        prompt_cache_key: PromptCacheKeySource::Prefix,
134        customize: Some(openrouter_request),
135        ..ChatRequestPolicy::DEFAULT
136    }),
137};
138
139pub const OLLAMA: ProviderConfig = ProviderConfig {
140    provider: Provider::Ollama,
141    api_base: Some("http://localhost:11434/v1"),
142    api_base_env: Some("OLLAMA_HOST"),
143    default_model: "",
144    api: Api::ChatCompletions(LOCAL_CHAT),
145};
146
147pub const LLAMACPP: ProviderConfig = ProviderConfig {
148    provider: Provider::LlamaCpp,
149    api_base: Some("http://localhost:8080/v1"),
150    api_base_env: Some("LLAMA_CPP_HOST"),
151    default_model: "",
152    api: Api::ChatCompletions(LOCAL_CHAT),
153};
154
155const LOCAL_CHAT: ChatRequestPolicy =
156    ChatRequestPolicy { tool_call_reasoning_placeholder: false, ..ChatRequestPolicy::DEFAULT };
157
158pub(crate) const BUILT_INS: &[&ProviderConfig] = &[
159    &ANTHROPIC,
160    &OPENAI,
161    &XIAOMI,
162    &DEEPSEEK,
163    &MOONSHOT,
164    &ZAI,
165    &AZURE_FOUNDRY,
166    &FIREWORKS,
167    &GEMINI,
168    &OPENROUTER,
169    &OLLAMA,
170    &LLAMACPP,
171];
172
173/// A provider whose behavior is fully described by a [`ProviderConfig`].
174#[derive(Clone)]
175pub struct GenericProvider {
176    config: &'static ProviderConfig,
177    http: reqwest::Client,
178    api_base: String,
179    headers: HeaderMap,
180    model: String,
181    request_model: Option<String>,
182    idle_timeout: Duration,
183}
184
185impl GenericProvider {
186    pub fn from_env(config: &'static ProviderConfig) -> Result<Self> {
187        Self::from_env_with_connection(config, ProviderConnectionConfig::default())
188    }
189
190    pub fn from_env_with_connection(
191        config: &'static ProviderConfig,
192        connection: ProviderConnectionConfig,
193    ) -> Result<Self> {
194        Self::from_vars(config, connection, &Vars::new())
195    }
196
197    pub fn from_vars(
198        config: &'static ProviderConfig,
199        mut connection: ProviderConnectionConfig,
200        vars: &Vars,
201    ) -> Result<Self> {
202        let api_key = match (connection.auth_mode, config.provider.required_env_var()) {
203            (ProviderAuthMode::Default, Some(env_var)) => {
204                vars.get(env_var).ok_or_else(|| LlmError::MissingApiKey(env_var.to_string()))?.into_owned()
205            }
206            _ => String::new(),
207        };
208        connection.base_url = connection
209            .base_url
210            .or_else(|| config.api_base_env.and_then(|env_var| vars.get(env_var)).map(Cow::into_owned));
211        Self::new_with_connection(&api_key, config, connection)
212    }
213
214    pub fn new(api_key: &str, config: &'static ProviderConfig) -> Result<Self> {
215        Self::new_with_connection(api_key, config, ProviderConnectionConfig::default())
216    }
217
218    pub fn new_with_connection(
219        api_key: &str,
220        config: &'static ProviderConfig,
221        connection: ProviderConnectionConfig,
222    ) -> Result<Self> {
223        let api_base = connection
224            .base_url
225            .or_else(|| config.api_base.map(str::to_string))
226            .ok_or_else(|| LlmError::MissingProviderUrl { provider: config.provider.parser_name().to_string() })?;
227        let mut api_base = api_base.trim_end_matches('/').to_string();
228        if config.provider.is_local() && !api_base.ends_with("/v1") {
229            api_base.push_str("/v1");
230        }
231        Ok(Self {
232            config,
233            http: http_client(),
234            api_base,
235            headers: api_headers(&config.api, api_key, connection.auth_mode)?,
236            model: config.default_model.to_string(),
237            request_model: connection.request_model,
238            idle_timeout: connection.idle_timeout,
239        })
240    }
241
242    pub fn with_model(mut self, model: &str) -> Self {
243        self.model = model.to_string();
244        self
245    }
246}
247
248impl StreamingModelProvider for GenericProvider {
249    fn stream_response(&self, context: &Context) -> LlmResponseStream {
250        let identity = self.model();
251        let wire_model = self.request_model.as_deref().unwrap_or(&self.model);
252        let stream = validate_reasoning(context, identity.as_ref()).and_then(|()| match &self.config.api {
253            Api::ChatCompletions(policy) => self.stream_chat_completions(wire_model, context, policy),
254            Api::Responses(policy) => self.stream_responses(identity.as_ref(), wire_model, context, policy),
255            Api::Messages => self.stream_messages(identity.as_ref(), wire_model, context),
256        });
257        stream.unwrap_or_else(error_stream)
258    }
259
260    fn display_name(&self) -> String {
261        display_name(self.config.provider, &self.model)
262    }
263
264    fn model(&self) -> Option<LlmModel> {
265        LlmModel::parse_for(self.config.provider, &self.model).ok()
266    }
267}
268
269impl GenericProvider {
270    fn stream_chat_completions(
271        &self,
272        wire_model: &str,
273        context: &Context,
274        policy: &ChatRequestPolicy,
275    ) -> Result<LlmResponseStream> {
276        let request = build_chat_request(wire_model, context, policy)?;
277        let opened = self.post("chat/completions", &request, responses_code);
278        Ok(response_stream(opened, |data, turn| decode_sse_chunk(&data, turn), self.idle_timeout))
279    }
280
281    fn stream_responses(
282        &self,
283        identity: Option<&LlmModel>,
284        wire_model: &str,
285        context: &Context,
286        policy: &ResponsesRequestPolicy,
287    ) -> Result<LlmResponseStream> {
288        let request = build_responses_request(identity, wire_model, context, policy)?;
289        Ok(response_stream(self.post("responses", &request, responses_code), decode_responses(), self.idle_timeout))
290    }
291
292    fn stream_messages(
293        &self,
294        identity: Option<&LlmModel>,
295        wire_model: &str,
296        context: &Context,
297    ) -> Result<LlmResponseStream> {
298        let request = StreamingRequest::new(wire_model, build_messages_request(identity, context)?);
299        let opened = self.post("v1/messages", &request, anthropic_code);
300        Ok(response_stream(opened, decode_messages(), self.idle_timeout))
301    }
302
303    fn post<T: Serialize>(
304        &self,
305        path: &str,
306        body: &T,
307        code: fn(&str) -> Option<String>,
308    ) -> impl Future<Output = Result<OpenedStream<SseData>>> + Send + use<T> {
309        post_sse(&self.http, &format!("{}/{path}", self.api_base), self.headers.clone(), body, code)
310    }
311}
312
313fn api_headers(api: &Api, api_key: &str, auth_mode: ProviderAuthMode) -> Result<HeaderMap> {
314    let mut headers = HeaderMap::new();
315    if let Api::Messages = api {
316        headers.insert("anthropic-version", HeaderValue::from_static("2023-06-01"));
317    }
318    if auth_mode == ProviderAuthMode::Default && !api_key.is_empty() {
319        let (name, value) = match api {
320            Api::ChatCompletions(_) | Api::Responses(_) => (AUTHORIZATION, format!("Bearer {api_key}")),
321            Api::Messages => (HeaderName::from_static("x-api-key"), api_key.to_string()),
322        };
323        let mut value = HeaderValue::from_str(&value)?;
324        value.set_sensitive(true);
325        headers.insert(name, value);
326    }
327    Ok(headers)
328}
329
330fn openrouter_request(request: &mut CompatibleChatRequest, context: &Context) {
331    request.extra.insert("usage".into(), serde_json::json!({ "include": true }));
332    request.extra.insert("cache_control".into(), serde_json::json!({ "type": "ephemeral" }));
333    if let Some(session_id) = context.session_affinity_key() {
334        request.extra.insert("session_id".into(), session_id.into());
335    }
336    if request.reasoning_effort == Some("none") {
337        request.reasoning_effort = None;
338        request.extra.insert("reasoning".into(), serde_json::json!({ "effort": "none" }));
339    }
340}
341
342#[cfg(test)]
343mod tests {
344    use futures::StreamExt;
345    use serde_json::json;
346
347    use super::*;
348    use crate::providers::anthropic::test_stream::{AnthropicStreamBuilder, anthropic_stream, usage};
349    use crate::providers::test_capture_server::{CaptureServer, ProviderFixture, ResponseSpec, hello_context};
350    use crate::reasoning::ReasoningControl;
351    use crate::testing::{assistant_message, llm_response, tool_error, tool_result};
352    use crate::types::IsoString;
353    use crate::{
354        AssistantReply, ChatMessage, ContentBlock, LlmResponse, MessageId, ModelSettings, ProviderErrorKind,
355        ReasoningEffort, StopReason, ToolCallRequest, ToolCallResult, ToolDefinition,
356    };
357
358    #[tokio::test]
359    async fn gemini_disables_reasoning_with_none() {
360        let model = reasoning_off_model(Provider::Gemini);
361        let mut fixture = ProviderFixture::builder(&GEMINI).model(&model.model_id()).build().await;
362        let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
363        context.set_reasoning_effort(ReasoningEffort::Disabled);
364
365        let (captured, responses) = fixture.run(&context).await;
366
367        assert_successful_stream(&responses);
368        assert_eq!(captured.body["reasoning_effort"], "none");
369    }
370
371    #[tokio::test]
372    async fn moonshot_normalizes_tool_schemas() {
373        let parameters = json!({
374            "$schema": "https://json-schema.org/draft/2020-12/schema",
375            "title": "MySchema",
376            "type": "object",
377            "properties": { "q": { "type": "string", "format": "uri" } }
378        });
379        let mut fixture = ProviderFixture::builder(&MOONSHOT).build().await;
380        let context =
381            Context::new(vec![ChatMessage::user("hi")], vec![ToolDefinition::new("search", "Search", parameters)]);
382
383        let (captured, responses) = fixture.run(&context).await;
384        let body = captured.body;
385
386        assert_successful_stream(&responses);
387        let params = &body["tools"][0]["function"]["parameters"];
388        assert!(params.get("$schema").is_none());
389        assert!(params.get("title").is_none());
390        assert!(params["properties"]["q"].get("format").is_none());
391        assert_eq!(params["type"], "object");
392    }
393
394    #[tokio::test]
395    async fn chat_completions_send_errors_and_summaries_as_user_notes() {
396        let mut fixture = ProviderFixture::builder(&DEEPSEEK).build().await;
397        let context = Context::new(
398            vec![
399                ChatMessage::Error {
400                    message_id: MessageId::new(),
401                    message: "boom".to_string(),
402                    timestamp: IsoString::now(),
403                },
404                ChatMessage::Summary {
405                    message_id: MessageId::new(),
406                    content: "earlier work".to_string(),
407                    timestamp: IsoString::now(),
408                    messages_compacted: 2,
409                },
410            ],
411            vec![],
412        );
413
414        let (captured, responses) = fixture.run(&context).await;
415        let body = captured.body;
416
417        assert_successful_stream(&responses);
418        assert_eq!(body["messages"][0], json!({ "role": "user", "content": "Error: boom" }));
419        assert_eq!(
420            body["messages"][1],
421            json!({ "role": "user", "content": "[Previous conversation handoff]\n\nearlier work" })
422        );
423    }
424
425    #[tokio::test]
426    async fn openrouter_requests_usage_caching_and_session_routing() {
427        let mut fixture = ProviderFixture::builder(&OPENROUTER).model("anthropic/claude-haiku-4.5").build().await;
428        let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
429        context.set_prompt_cache_key(Some("prefix-abc".to_string()));
430        context.set_session_affinity_key(Some("conversation-abc".to_string()));
431        context.set_reasoning_effort(ReasoningEffort::High);
432
433        let (captured, responses) = fixture.run(&context).await;
434        let body = captured.body;
435
436        assert_successful_stream(&responses);
437        assert_eq!(body["model"], "anthropic/claude-haiku-4.5");
438        assert_eq!(body["prompt_cache_key"], "prefix-abc");
439        assert_eq!(body["session_id"], "conversation-abc");
440        assert_eq!(body["reasoning_effort"], "high");
441        assert_eq!(body["cache_control"]["type"], "ephemeral");
442        assert_eq!(body["usage"]["include"], true);
443        assert_eq!(fixture.provider.display_name(), "OpenRouter (anthropic/claude-haiku-4.5)");
444    }
445
446    #[tokio::test]
447    async fn openrouter_disables_reasoning_through_its_unified_reasoning_object() {
448        let model = reasoning_off_model(Provider::OpenRouter);
449        let mut fixture = ProviderFixture::builder(&OPENROUTER).model(&model.model_id()).build().await;
450        let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
451        context.set_reasoning_effort(ReasoningEffort::Disabled);
452
453        let (captured, responses) = fixture.run(&context).await;
454        let body = captured.body;
455
456        assert_successful_stream(&responses);
457        assert_eq!(body["reasoning"], json!({ "effort": "none" }));
458        assert!(body.get("reasoning_effort").is_none());
459    }
460
461    #[tokio::test]
462    async fn chat_rate_limit_rejection_is_a_retryable_provider_error() {
463        let body = r#"{"error":{"message":"Rate limit exceeded: free-models-per-day","code":429}}"#;
464        let error = rejected_chat_request(ResponseSpec::sse(body).with_status(429)).await;
465        let provider_error = error.provider().expect("a 429 must classify as a provider error");
466
467        assert_eq!(provider_error.kind, ProviderErrorKind::RateLimit);
468        assert_eq!(provider_error.http_status, Some(429));
469        assert_eq!(provider_error.code.as_deref(), Some("429"));
470        assert_eq!(provider_error.request_id.as_deref(), Some("request-123"));
471        assert!(provider_error.message.contains("free-models-per-day"));
472        assert!(error.is_retryable(), "429 must be retryable, got {error}");
473    }
474
475    #[tokio::test]
476    async fn chat_rejection_preserves_string_error_codes() {
477        let body = r#"{"error":{"code":"rate_limit_exceeded","message":"slow down"}}"#;
478        let error = rejected_chat_request(ResponseSpec::sse(body).with_status(429)).await;
479        let provider_error = error.provider().unwrap();
480
481        assert_eq!(provider_error.code.as_deref(), Some("rate_limit_exceeded"));
482        assert!(provider_error.message.contains("slow down"));
483        assert!(error.is_retryable());
484    }
485
486    #[tokio::test]
487    async fn chat_bad_request_is_not_retryable() {
488        let body = r#"{"error":{"message":"invalid request"}}"#;
489        let error = rejected_chat_request(ResponseSpec::sse(body).with_status(400)).await;
490
491        assert_eq!(error.provider().map(|provider| provider.kind), Some(ProviderErrorKind::Api));
492        assert!(!error.is_retryable());
493    }
494
495    #[tokio::test]
496    async fn chat_rejection_keeps_http_status_when_the_error_body_fails_to_read() {
497        let error = rejected_chat_request(ResponseSpec::sse("").with_status(429).interrupted()).await;
498        let provider_error = error.provider().unwrap();
499
500        assert!(provider_error.message.ends_with("Unknown error"), "{provider_error:?}");
501        assert_eq!(provider_error.http_status, Some(429));
502        assert_eq!(provider_error.request_id.as_deref(), Some("request-123"));
503        assert_eq!(provider_error.kind, ProviderErrorKind::RateLimit);
504        assert!(error.is_retryable());
505    }
506
507    #[tokio::test]
508    async fn chat_chunk_that_is_not_json_interrupts_the_stream() {
509        let responses = streamed_chat("data: invalid json\n\n").await;
510
511        assert!(matches!(responses[..], [Ok(LlmResponse::Start), Err(_)]), "{responses:?}");
512        assert_interrupted(&responses);
513    }
514
515    #[tokio::test]
516    async fn chat_stream_ends_at_done() {
517        let responses = streamed_chat(&format!("{CHAT_FIXTURE}data: invalid json\n\n")).await;
518
519        assert_successful_stream(&responses);
520    }
521
522    #[tokio::test]
523    async fn chat_stream_skips_keepalive_events() {
524        let responses = streamed_chat(&format!("event: keepalive\ndata: ping\n\n{CHAT_FIXTURE}")).await;
525
526        assert_successful_stream(&responses);
527    }
528
529    #[tokio::test]
530    async fn chat_stream_cut_off_mid_tool_call_is_interrupted() {
531        let tool_call_start = json!({ "choices": [{ "index": 0, "delta": { "tool_calls": [{
532            "index": 0, "id": "call_1", "type": "function",
533            "function": { "name": "bash", "arguments": r#"{"command":"# }
534        }] } }] });
535
536        let responses = streamed_chat(&format!("data: {tool_call_start}\n\n")).await;
537
538        assert!(
539            !responses.iter().any(|response| matches!(
540                response,
541                Ok(LlmResponse::ToolRequestComplete { .. } | LlmResponse::Done { .. })
542            )),
543            "{responses:?}"
544        );
545        assert_interrupted(&responses);
546    }
547
548    #[tokio::test]
549    async fn disabled_toggle_and_unknown_models_never_send_requests() {
550        let mut fixture = ProviderFixture::builder(&DEEPSEEK).build().await;
551        for model in ["deepseek-v4-flash", "unknown"] {
552            fixture.provider = fixture.provider.with_model(model);
553            let mut context = Context::new(vec![], vec![]);
554            context.set_reasoning_effort(ReasoningEffort::Disabled);
555            let responses = fixture.provider.stream_response(&context).collect::<Vec<_>>().await;
556            assert_eq!(responses.len(), 1, "{responses:?}");
557            let error = responses[0].as_ref().unwrap_err();
558            if model == "unknown" {
559                assert!(matches!(error, LlmError::ReasoningValidation(_)));
560            } else {
561                assert!(matches!(error, LlmError::UnsupportedDisableTransport { .. }));
562            }
563            assert!(!error.is_retryable());
564        }
565    }
566
567    #[tokio::test]
568    async fn sampling_settings_are_sent_as_configured() {
569        for config in [&DEEPSEEK, &OPENAI, &ANTHROPIC] {
570            let mut fixture = ProviderFixture::builder(config).build().await;
571            let mut context = hello_context();
572            context.set_model_settings(ModelSettings { temperature: Some(0.2), top_p: Some(0.9), max_tokens: None });
573
574            let (captured, responses) = fixture.run(&context).await;
575            let body = captured.body;
576
577            assert!(responses.iter().all(Result::is_ok), "{responses:?}");
578            assert_eq!(body["temperature"], 0.2, "{}", config.provider);
579            assert_eq!(body["top_p"], 0.9, "{}", config.provider);
580        }
581    }
582
583    #[tokio::test]
584    async fn every_api_validates_reasoning_before_sending() {
585        for config in [&DEEPSEEK, &OPENAI, &ANTHROPIC] {
586            let fixture = ProviderFixture::builder(config).model("unknown").build().await;
587            let mut context = hello_context();
588            context.set_reasoning_effort(ReasoningEffort::Disabled);
589
590            let responses = fixture.provider.stream_response(&context).collect::<Vec<_>>().await;
591
592            assert!(matches!(responses[..], [Err(LlmError::ReasoningValidation(_))]), "{responses:?}");
593        }
594    }
595
596    #[test]
597    fn display_name_names_models_outside_the_catalog() {
598        let provider = GenericProvider::new("key", &DEEPSEEK).unwrap().with_model("my-custom-model");
599        assert_eq!(provider.display_name(), "DeepSeek (my-custom-model)");
600
601        let provider = GenericProvider::new("key", &DEEPSEEK).unwrap().with_model("");
602        assert_eq!(provider.display_name(), "DeepSeek");
603    }
604
605    #[test]
606    fn azure_foundry_requires_a_configured_url() {
607        let Err(error) = GenericProvider::new("key", &AZURE_FOUNDRY) else {
608            panic!("Azure Foundry must require a URL");
609        };
610        assert!(matches!(error, LlmError::MissingProviderUrl { provider } if provider == "azure-foundry"));
611    }
612
613    #[tokio::test]
614    async fn request_model_routes_each_api_without_changing_catalog_identity() {
615        for (config, model, deployment, identity, display_name, auth_mode) in [
616            (
617                &AZURE_FOUNDRY,
618                "gpt-5.5",
619                "production-coding",
620                "azure-foundry:gpt-5.5",
621                "Microsoft Foundry (gpt-5.5)",
622                ProviderAuthMode::None,
623            ),
624            (
625                &XIAOMI,
626                "mimo-v2.6-pro",
627                "mimo-deployment",
628                "xiaomi:mimo-v2.6-pro",
629                "Xiaomi (mimo-v2.6-pro)",
630                ProviderAuthMode::None,
631            ),
632            (
633                &ANTHROPIC,
634                "claude-sonnet-4-5-20250929",
635                "claude-deployment",
636                "anthropic:claude-sonnet-4-5-20250929",
637                "Anthropic (claude-sonnet-4-5-20250929)",
638                ProviderAuthMode::Default,
639            ),
640        ] {
641            let mut fixture = ProviderFixture::builder(config)
642                .model(model)
643                .connection(ProviderConnectionConfig {
644                    request_model: Some(deployment.into()),
645                    auth_mode,
646                    ..Default::default()
647                })
648                .build()
649                .await;
650
651            let (captured, responses) = fixture.run(&hello_context()).await;
652
653            assert_successful_stream(&responses);
654            assert_eq!(captured.body["model"], deployment);
655            assert_eq!(captured.body["stream"], true);
656            assert!(captured.headers.get("authorization").is_none());
657            assert_eq!(fixture.provider.model().unwrap().to_string(), identity);
658            assert_eq!(fixture.provider.display_name(), display_name);
659            match config.api {
660                Api::ChatCompletions(_) => {
661                    assert_eq!(captured.path, "/chat/completions");
662                    assert_eq!(captured.body["stream_options"]["include_usage"], true);
663                }
664                Api::Responses(_) => assert_eq!(captured.path, "/responses"),
665                Api::Messages => {
666                    assert_eq!(captured.path, "/v1/messages");
667                    assert_eq!(captured.headers["anthropic-version"], "2023-06-01");
668                    assert_eq!(captured.headers["x-api-key"], "key");
669                }
670            }
671        }
672    }
673
674    #[tokio::test]
675    async fn default_auth_sends_the_api_key_as_a_bearer_token() {
676        let mut fixture = ProviderFixture::builder(&DEEPSEEK)
677            .api_key("secret")
678            .connection(ProviderConnectionConfig::default())
679            .build()
680            .await;
681        let (captured, responses) = fixture.run(&hello_context()).await;
682
683        assert_successful_stream(&responses);
684        assert_eq!(captured.headers["authorization"], "Bearer secret");
685    }
686
687    #[tokio::test]
688    async fn chat_providers_apply_their_declared_prompt_cache_policy() {
689        for (config, expected_key) in [
690            (&AZURE_FOUNDRY, Some("prefix-abc")),
691            (&FIREWORKS, Some("conversation-abc")),
692            (&DEEPSEEK, None),
693            (&MOONSHOT, None),
694            (&ZAI, None),
695        ] {
696            for keys_set in [true, false] {
697                let mut fixture = ProviderFixture::builder(config).build().await;
698                let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
699                if keys_set {
700                    context.set_prompt_cache_key(Some("prefix-abc".to_string()));
701                    context.set_session_affinity_key(Some("conversation-abc".to_string()));
702                }
703
704                let (captured, responses) = fixture.run(&context).await;
705
706                assert_successful_stream(&responses);
707                let sent_key = captured.body.get("prompt_cache_key").and_then(serde_json::Value::as_str);
708                assert_eq!(sent_key, expected_key.filter(|_| keys_set), "{} keys_set={keys_set}", config.provider);
709                assert!(captured.body.get("user").is_none());
710                assert!(captured.body.get("session_id").is_none());
711            }
712        }
713    }
714
715    #[tokio::test]
716    async fn local_providers_send_bare_chat_requests_under_v1() {
717        for (config, display_name) in [(&OLLAMA, "Ollama (test-model)"), (&LLAMACPP, "LlamaCpp (test-model)")] {
718            let mut server = CaptureServer::start_chat_completions().await;
719            let connection = ProviderConnectionConfig { base_url: Some(server.base_url.clone()), ..Default::default() };
720            let provider =
721                GenericProvider::from_env_with_connection(config, connection).unwrap().with_model("test-model");
722            let tool_call = assistant_message().tool_call("call_1", "tool", "{}").build();
723            let mut context = Context::new(vec![ChatMessage::user("Hello"), tool_call], vec![]);
724            context.set_prompt_cache_key(Some("prefix-abc".to_string()));
725            context.set_session_affinity_key(Some("conversation-abc".to_string()));
726
727            let responses = provider.stream_response(&context).collect::<Vec<_>>().await;
728            let captured = server.captured().await;
729
730            assert_successful_stream(&responses);
731            assert_eq!(captured.path, "/v1/chat/completions");
732            assert_eq!(captured.body["model"], "test-model");
733            assert!(captured.headers.get("authorization").is_none());
734            assert!(captured.body.get("prompt_cache_key").is_none());
735            assert!(captured.body["messages"][1].get("reasoning_content").is_none());
736            assert_eq!(provider.display_name(), display_name);
737        }
738    }
739
740    #[tokio::test]
741    async fn local_providers_send_requests_to_their_host_variable() {
742        for (config, host_var) in [(&OLLAMA, "OLLAMA_HOST"), (&LLAMACPP, "LLAMA_CPP_HOST")] {
743            let mut server = CaptureServer::start_chat_completions().await;
744            let vars = Vars::new().with_env_lookup(|_| None).with(host_var, server.base_url.clone());
745            let provider = GenericProvider::from_vars(config, ProviderConnectionConfig::default(), &vars)
746                .unwrap()
747                .with_model("test-model");
748
749            let responses = provider.stream_response(&hello_context()).collect::<Vec<_>>().await;
750
751            assert_successful_stream(&responses);
752            assert_eq!(server.captured().await.path, "/v1/chat/completions", "{host_var}");
753        }
754    }
755
756    #[tokio::test]
757    async fn connection_url_takes_precedence_over_the_host_variable() {
758        let mut server = CaptureServer::start_chat_completions().await;
759        let vars = Vars::new().with_env_lookup(|_| None).with("OLLAMA_HOST", "http://127.0.0.1:1");
760        let connection = ProviderConnectionConfig { base_url: Some(server.base_url.clone()), ..Default::default() };
761        let provider = GenericProvider::from_vars(&OLLAMA, connection, &vars).unwrap().with_model("test-model");
762
763        let responses = provider.stream_response(&hello_context()).collect::<Vec<_>>().await;
764
765        assert_successful_stream(&responses);
766        assert_eq!(server.captured().await.path, "/v1/chat/completions");
767    }
768
769    #[tokio::test]
770    async fn providers_honor_configured_idle_timeout() {
771        for (config, fixture) in [
772            (&OLLAMA, include_str!("../../tests/fixtures/openai/01_minimal.sse")),
773            (&ANTHROPIC, include_str!("../../tests/fixtures/anthropic/01_minimal.sse")),
774        ] {
775            let spec = ResponseSpec::sse(fixture).paced(Duration::from_mins(2));
776            let mut fixture = ProviderFixture::builder(config)
777                .response(spec)
778                .connection(ProviderConnectionConfig {
779                    auth_mode: ProviderAuthMode::None,
780                    idle_timeout: Duration::from_mins(1),
781                    ..Default::default()
782                })
783                .build()
784                .await;
785
786            let responses =
787                fixture.server.collect_on_paused_clock(fixture.provider.stream_response(&hello_context())).await;
788
789            let error = responses.last().and_then(|response| response.as_ref().err()).and_then(LlmError::provider);
790            assert_eq!(error.map(|error| error.kind), Some(ProviderErrorKind::Timeout), "{responses:?}");
791        }
792    }
793
794    #[tokio::test]
795    async fn responses_send_user_input_without_unset_options() {
796        let body = openai_request_body("gpt-4.1", &hello_context()).await;
797
798        assert_eq!(body["model"], "gpt-4.1");
799        assert_eq!(
800            body["input"],
801            json!([{
802                "type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]
803            }])
804        );
805        for field in ["instructions", "tools", "reasoning", "temperature", "top_p", "max_output_tokens"] {
806            assert!(body.get(field).is_none(), "{field}: {body}");
807        }
808    }
809
810    #[tokio::test]
811    async fn responses_lift_system_instructions_and_apply_model_settings() {
812        let mut context = Context::new(vec![ChatMessage::system("You are helpful."), ChatMessage::user("Hi")], vec![]);
813        context.set_model_settings(ModelSettings { temperature: Some(0.0), top_p: Some(0.5), max_tokens: Some(128) });
814
815        let body = openai_request_body("gpt-4.1", &context).await;
816
817        assert_eq!(body["instructions"], "You are helpful.");
818        assert_eq!(body["input"].as_array().unwrap().len(), 1);
819        assert_eq!(body["input"][0]["role"], "user");
820        assert_eq!(body["temperature"], 0.0);
821        assert_eq!(body["top_p"], 0.5);
822        assert_eq!(body["max_output_tokens"], 128);
823    }
824
825    #[tokio::test]
826    async fn responses_preserve_tool_conversations_and_definitions() {
827        let arguments = r#"{"path":"foo.rs"}"#;
828        let context = Context::new(
829            vec![
830                ChatMessage::user("Read foo.rs"),
831                assistant_message().text("I'll read that file.").tool_call("call_1", "read_file", arguments).build(),
832                tool_result("call_1", "read_file", arguments, "fn main() {}"),
833                assistant_message().text("Here's the file content.").build(),
834            ],
835            vec![ToolDefinition::new(
836                "read_file",
837                "Read a file from disk",
838                json!({
839                    "type": "object", "properties": { "path": { "type": "string" } }
840                }),
841            )],
842        );
843
844        let body = openai_request_body("gpt-4.1", &context).await;
845
846        assert_eq!(
847            body["input"],
848            json!([
849                { "type": "message", "role": "user", "content": [{"type": "input_text", "text": "Read foo.rs"}] },
850                { "type": "message", "role": "assistant", "content": "I'll read that file." },
851                { "type": "function_call", "arguments": arguments, "call_id": "call_1", "name": "read_file" },
852                { "type": "function_call_output", "call_id": "call_1", "output": "fn main() {}" },
853                { "type": "message", "role": "assistant", "content": "Here's the file content." },
854            ])
855        );
856        let tools = body["tools"].as_array().unwrap();
857        assert_eq!(tools.len(), 1);
858        assert_eq!(tools[0]["type"], "function");
859        assert_eq!(tools[0]["name"], "read_file");
860        assert_eq!(tools[0]["description"], "Read a file from disk");
861        assert_eq!(tools[0]["parameters"]["properties"]["path"]["type"], "string");
862        assert_eq!(tools[0]["strict"], false);
863    }
864
865    #[tokio::test]
866    async fn responses_preserve_tool_calls_without_an_assistant_text_message() {
867        let context = Context::new(vec![assistant_message().tool_call("call_1", "search", "{}").build()], vec![]);
868
869        let body = openai_request_body("gpt-4.1", &context).await;
870
871        assert_eq!(
872            body["input"],
873            json!([{
874                "type": "function_call", "arguments": "{}", "call_id": "call_1", "name": "search"
875            }])
876        );
877    }
878
879    #[tokio::test]
880    async fn responses_preserve_tool_errors_and_summary_notes() {
881        let context = Context::new(
882            vec![
883                tool_error("call_2", "bash", "command failed"),
884                ChatMessage::Summary {
885                    message_id: MessageId::new(),
886                    content: "User asked about Rust.".into(),
887                    timestamp: IsoString::now(),
888                    messages_compacted: 5,
889                },
890            ],
891            vec![],
892        );
893
894        let body = openai_request_body("gpt-4.1", &context).await;
895        let input = body["input"].as_array().unwrap();
896
897        assert_eq!(input.len(), 2);
898        assert_eq!(
899            input[0],
900            json!({ "type": "function_call_output", "call_id": "call_2", "output": "Error: command failed" })
901        );
902        assert_eq!(input[1]["type"], "message");
903        assert_eq!(input[1]["role"], "user");
904        let note = input[1]["content"].as_str().unwrap();
905        assert!(note.starts_with("[Previous conversation handoff]"));
906        assert!(note.contains("User asked about Rust."));
907    }
908
909    #[tokio::test]
910    async fn responses_replay_encrypted_reasoning_only_for_the_original_model() {
911        for source_model in [Some("openai:gpt-5.6"), Some("openai:gpt-5.5"), None] {
912            let mut assistant = assistant_message().reasoning("summary").encrypted_reasoning("r_1", "encrypted-blob");
913            if let Some(model) = source_model {
914                assistant = assistant.model(model);
915            }
916            let context = Context::new(vec![assistant.text("thinking done").build()], vec![]);
917
918            let body = openai_request_body("gpt-5.6", &context).await;
919            let input = body["input"].as_array().unwrap();
920
921            let answer = json!({ "type": "message", "role": "assistant", "content": "thinking done" });
922            if source_model == Some("openai:gpt-5.6") {
923                let reasoning =
924                    json!({ "type": "reasoning", "id": "r_1", "summary": [], "encrypted_content": "encrypted-blob" });
925                assert_eq!(input, &[reasoning, answer]);
926            } else {
927                assert_eq!(input, &[answer], "{source_model:?}");
928            }
929        }
930    }
931
932    #[tokio::test]
933    async fn responses_replay_each_reasoning_item_where_the_model_wrote_it() {
934        let context = Context::new(
935            vec![
936                assistant_message()
937                    .model("openai:gpt-5.6")
938                    .reasoning("plan")
939                    .encrypted_reasoning("r_1", "first-blob")
940                    .tool_call("call_1", "search", "{}")
941                    .reasoning("check")
942                    .encrypted_reasoning("r_2", "second-blob")
943                    .tool_call("call_2", "search", "{}")
944                    .build(),
945            ],
946            vec![],
947        );
948
949        let body = openai_request_body("gpt-5.6", &context).await;
950        let items: Vec<_> = body["input"]
951            .as_array()
952            .unwrap()
953            .iter()
954            .map(|item| item["encrypted_content"].as_str().or(item["call_id"].as_str()).unwrap())
955            .collect();
956
957        assert_eq!(items, ["first-blob", "call_1", "second-blob", "call_2"]);
958    }
959
960    #[tokio::test]
961    async fn responses_send_each_enabled_effort_with_an_automatic_summary() {
962        for effort in ReasoningEffort::all().iter().filter(|effort| effort.is_enabled()) {
963            let model = LlmModel::all()
964                .iter()
965                .find(|model| {
966                    model.provider_enum() == Provider::Openai && model.effective_reasoning_levels().contains(effort)
967                })
968                .expect("OpenAI has a model supporting this effort");
969            let mut context = hello_context();
970            context.set_reasoning_effort(*effort);
971
972            let body = openai_request_body(&model.model_id(), &context).await;
973
974            assert_eq!(body["reasoning"], json!({ "effort": effort.as_str(), "summary": "auto" }));
975        }
976    }
977
978    #[tokio::test]
979    async fn openai_distinguishes_default_disabled_and_low_effort() {
980        for (effort, expected) in [
981            (ReasoningEffort::Default, None),
982            (ReasoningEffort::Disabled, Some("none")),
983            (ReasoningEffort::Low, Some("low")),
984        ] {
985            let mut fixture = ProviderFixture::builder(&OPENAI).model("gpt-5.4").build().await;
986            let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
987            context.set_reasoning_effort(effort);
988
989            let (captured, responses) = fixture.run(&context).await;
990            let body = captured.body;
991
992            assert!(responses.iter().all(Result::is_ok), "{responses:?}");
993            assert_eq!(body["reasoning"]["effort"].as_str(), expected);
994            if effort == ReasoningEffort::Disabled {
995                assert!(body["reasoning"]["summary"].is_null());
996            }
997        }
998    }
999
1000    #[tokio::test]
1001    async fn openai_sends_max_effort_and_prompt_cache_key() {
1002        let mut fixture = ProviderFixture::builder(&OPENAI).model("gpt-5.6").build().await;
1003        let mut context = Context::new(vec![ChatMessage::user("Think harder")], vec![]);
1004        context.set_reasoning_effort(ReasoningEffort::Max);
1005        context.set_prompt_cache_key(Some("cache-key".to_string()));
1006
1007        let (captured, responses) = fixture.run(&context).await;
1008
1009        assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1010        assert_eq!(captured.body["reasoning"]["effort"], "max");
1011        assert_eq!(captured.body["model"], "gpt-5.6");
1012        assert_eq!(captured.body["prompt_cache_key"], "cache-key");
1013        assert_eq!(captured.body["include"], json!(["reasoning.encrypted_content"]));
1014        assert_eq!(captured.body["stream"], true);
1015        assert_eq!(fixture.provider.display_name(), "OpenAI (gpt-5.6)");
1016    }
1017
1018    #[tokio::test]
1019    async fn responses_http_200_failed_server_error_is_retryable_with_request_id() {
1020        let spec = ResponseSpec::sse(include_str!("../../tests/fixtures/openai_responses/04_failed_server.sse"))
1021            .with_header("x-request-id", "req-openai-1");
1022        let mut fixture = ProviderFixture::builder(&OPENAI).response(spec).build().await;
1023        let context = Context::new(vec![ChatMessage::user("hi")], vec![]);
1024
1025        let (_, responses) = fixture.run(&context).await;
1026
1027        assert!(!responses.iter().any(|r| matches!(r, Ok(LlmResponse::Done { .. }))));
1028        let err = responses.iter().find_map(|r| r.as_ref().err()).expect("expected a failure");
1029        assert!(err.is_retryable(), "server_error must be retryable: {err:?}");
1030        let provider_error = err.provider().expect("expected provider error");
1031        assert_eq!(provider_error.kind, ProviderErrorKind::Server);
1032        assert_eq!(provider_error.http_status, Some(200));
1033        assert_eq!(provider_error.request_id.as_deref(), Some("req-openai-1"));
1034        assert_eq!(provider_error.code.as_deref(), Some("server_error"));
1035    }
1036
1037    #[tokio::test]
1038    async fn responses_surface_a_mapping_failure_as_the_only_item() {
1039        let connection = ProviderConnectionConfig { auth_mode: ProviderAuthMode::None, ..Default::default() };
1040        let provider = GenericProvider::from_env_with_connection(&OPENAI, connection).unwrap();
1041        let context = Context::new(
1042            vec![ChatMessage::User {
1043                message_id: MessageId::new(),
1044                content: vec![ContentBlock::Audio { data: "YXVkaW8=".to_string(), mime_type: "audio/wav".to_string() }],
1045                timestamp: IsoString::now(),
1046            }],
1047            vec![],
1048        );
1049
1050        let responses = provider.stream_response(&context).collect::<Vec<_>>().await;
1051
1052        assert_eq!(responses.len(), 1);
1053        assert!(matches!(responses[0], Err(LlmError::UnsupportedContent(_))), "{responses:?}");
1054    }
1055
1056    #[tokio::test]
1057    async fn xiaomi_omits_encrypted_reasoning_summaries_and_prompt_cache_key() {
1058        let mut fixture = ProviderFixture::builder(&XIAOMI).build().await;
1059        let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
1060        context.set_reasoning_effort(ReasoningEffort::High);
1061        context.set_prompt_cache_key(Some("cache-key".to_string()));
1062
1063        let (captured, responses) = fixture.run(&context).await;
1064
1065        assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1066        assert_eq!(captured.path, "/responses");
1067        assert_eq!(captured.body["model"], "mimo-v2.6-pro");
1068        assert_eq!(captured.body["reasoning"], json!({ "effort": "high" }));
1069        assert!(captured.body.get("include").is_none());
1070        assert!(captured.body.get("prompt_cache_key").is_none());
1071        assert_eq!(fixture.provider.display_name(), "Xiaomi (mimo-v2.6-pro)");
1072    }
1073
1074    #[tokio::test]
1075    async fn xiaomi_replays_prior_reasoning_as_plain_text() {
1076        let mut fixture = ProviderFixture::builder(&XIAOMI).build().await;
1077        let context = Context::new(
1078            vec![
1079                ChatMessage::user("Hello"),
1080                assistant_message().reasoning("greeting the user").text("Hi").build(),
1081                ChatMessage::user("Again"),
1082            ],
1083            vec![],
1084        );
1085
1086        let (captured, responses) = fixture.run(&context).await;
1087
1088        assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1089        let reasoning = captured.body["input"]
1090            .as_array()
1091            .unwrap()
1092            .iter()
1093            .find(|item| item["type"] == "reasoning")
1094            .expect("reasoning item should be replayed");
1095        assert_eq!(reasoning["content"], json!([{ "type": "reasoning_text", "text": "greeting the user" }]));
1096        assert!(reasoning.get("encrypted_content").is_none_or(serde_json::Value::is_null));
1097    }
1098
1099    #[tokio::test]
1100    async fn xiaomi_drops_null_from_optional_tool_parameters() {
1101        let mut fixture = ProviderFixture::builder(&XIAOMI).build().await;
1102        let tool = ToolDefinition::new(
1103            "bash",
1104            "Run a command",
1105            json!({
1106                "type": "object",
1107                "properties": {
1108                    "command": { "type": "string" },
1109                    "description": { "type": ["string", "null"] }
1110                },
1111                "required": ["command"]
1112            }),
1113        );
1114        let context = Context::new(vec![ChatMessage::user("Hello")], vec![tool]);
1115
1116        let (captured, responses) = fixture.run(&context).await;
1117
1118        assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1119        assert_eq!(captured.body["tools"][0]["parameters"]["properties"]["description"], json!({ "type": "string" }));
1120    }
1121
1122    #[tokio::test]
1123    async fn anthropic_sends_its_version_and_api_key_unless_auth_is_none() {
1124        for (auth_mode, api_key) in [(ProviderAuthMode::Default, Some("secret")), (ProviderAuthMode::None, None)] {
1125            let mut fixture = ProviderFixture::builder(&ANTHROPIC)
1126                .api_key("secret")
1127                .connection(ProviderConnectionConfig { auth_mode, ..Default::default() })
1128                .build()
1129                .await;
1130            let (captured, responses) = fixture.run(&hello_context()).await;
1131
1132            assert_successful_stream(&responses);
1133            assert_eq!(captured.path, "/v1/messages");
1134            assert_eq!(captured.headers["anthropic-version"], "2023-06-01");
1135            assert_eq!(captured.headers.get("x-api-key").map(|key| key.to_str().unwrap()), api_key);
1136            assert!(captured.headers.get("authorization").is_none());
1137        }
1138    }
1139
1140    #[tokio::test]
1141    async fn anthropic_caches_the_system_prompt_and_sends_tools() {
1142        let mut fixture = ProviderFixture::builder(&ANTHROPIC).build().await;
1143        let context = Context::new(
1144            vec![ChatMessage::system("You are helpful"), ChatMessage::user("Hello")],
1145            vec![ToolDefinition::new("search", "Search", json!({ "type": "object" }))],
1146        );
1147
1148        let (captured, responses) = fixture.run(&context).await;
1149        let body = captured.body;
1150
1151        assert_successful_stream(&responses);
1152        assert_eq!(body["model"], "claude-sonnet-4-5-20250929");
1153        assert_eq!(body["max_tokens"], 16_384);
1154        assert_eq!(body["stream"], true);
1155        assert!(body.get("cache_control").is_none(), "{body}");
1156        assert_eq!(
1157            body["system"],
1158            json!([{ "type": "text", "text": "You are helpful", "cache_control": { "type": "ephemeral" } }])
1159        );
1160        assert_eq!(
1161            body["messages"],
1162            json!([{
1163                "role": "user",
1164                "content": [{ "type": "text", "text": "Hello", "cache_control": { "type": "ephemeral" } }]
1165            }])
1166        );
1167        assert_eq!(
1168            body["tools"],
1169            json!([{ "name": "search", "description": "Search", "input_schema": { "type": "object" } }])
1170        );
1171        assert_eq!(fixture.provider.display_name(), "Anthropic (claude-sonnet-4-5-20250929)");
1172    }
1173
1174    #[tokio::test]
1175    async fn anthropic_default_and_disabled_thinking_keep_sampling_and_token_limit() {
1176        for (model, effort, thinking) in [
1177            (anthropic_model(ReasoningControl::Budget, &[]), ReasoningEffort::Default, None),
1178            (anthropic_model(ReasoningControl::Effort, &[]), ReasoningEffort::Default, None),
1179            (reasoning_off_model(Provider::Anthropic), ReasoningEffort::Disabled, Some(json!({ "type": "disabled" }))),
1180        ] {
1181            let body = anthropic_request_body(model, effort).await;
1182
1183            assert_eq!(body.get("thinking"), thinking.as_ref());
1184            assert!(body.get("output_config").is_none());
1185            assert_eq!(body["max_tokens"], 500);
1186            assert_eq!(body["temperature"], 0.5);
1187            assert_eq!(body["top_p"], 0.5);
1188        }
1189    }
1190
1191    #[tokio::test]
1192    async fn anthropic_budget_models_clear_sampling_and_raise_token_limit() {
1193        let cases = [(ReasoningEffort::Low, 1024), (ReasoningEffort::Medium, 4096), (ReasoningEffort::High, 10240)];
1194        let model = anthropic_model(ReasoningControl::Budget, &cases.map(|(effort, _)| effort));
1195        for (effort, budget_tokens) in cases {
1196            let body = anthropic_request_body(model, effort).await;
1197
1198            assert_eq!(body["thinking"], json!({ "type": "enabled", "budget_tokens": budget_tokens }));
1199            assert!(body.get("output_config").is_none());
1200            assert_eq!(body["max_tokens"], budget_tokens + 1024);
1201            assert!(body.get("temperature").is_none());
1202            assert!(body.get("top_p").is_none());
1203        }
1204    }
1205
1206    #[tokio::test]
1207    async fn anthropic_adaptive_models_send_effort_with_summarized_thinking() {
1208        let cases = [(ReasoningEffort::Low, "low"), (ReasoningEffort::Xhigh, "xhigh"), (ReasoningEffort::Max, "max")];
1209        let model = anthropic_model(ReasoningControl::Effort, &cases.map(|(effort, _)| effort));
1210        for (effort, wire) in cases {
1211            let body = anthropic_request_body(model, effort).await;
1212
1213            assert_eq!(body["thinking"], json!({ "type": "adaptive", "display": "summarized" }));
1214            assert_eq!(body["output_config"], json!({ "effort": wire }));
1215            assert_eq!(body["max_tokens"], 500);
1216            assert!(body.get("temperature").is_none());
1217            assert!(body.get("top_p").is_none());
1218        }
1219    }
1220
1221    #[tokio::test]
1222    async fn anthropic_budget_and_effort_models_send_a_budget_refined_by_effort() {
1223        let cases = [(ReasoningEffort::Low, 1024, "low"), (ReasoningEffort::Max, 32768, "max")];
1224        let model = anthropic_model(ReasoningControl::BudgetAndEffort, &cases.map(|(effort, ..)| effort));
1225        for (effort, budget_tokens, wire) in cases {
1226            let body = anthropic_request_body(model, effort).await;
1227
1228            assert_eq!(body["thinking"], json!({ "type": "enabled", "budget_tokens": budget_tokens }));
1229            assert_eq!(body["output_config"], json!({ "effort": wire }));
1230            assert_eq!(body["max_tokens"], budget_tokens + 1024);
1231            assert!(body.get("temperature").is_none());
1232        }
1233    }
1234
1235    #[tokio::test]
1236    async fn anthropic_replays_signed_thinking_only_to_the_model_that_wrote_it() {
1237        let thinking = json!([
1238            { "type": "thinking", "thinking": "Check the weather", "signature": "sig-1" },
1239            { "type": "redacted_thinking", "data": "opaque" }
1240        ]);
1241        let tool_use = json!({ "type": "tool_use", "id": "call_1", "name": "weather", "input": { "city": "Paris" } });
1242        let mut anthropic_models = LlmModel::all().iter().filter(|model| model.provider_enum() == Provider::Anthropic);
1243        let (model, other_model) = (anthropic_models.next().unwrap(), anthropic_models.next().unwrap());
1244        for (author, expected_content) in [
1245            (model, json!([thinking[0], thinking[1], { "type": "text", "text": "Checking" }, tool_use])),
1246            (other_model, json!([{ "type": "text", "text": "Checking" }, tool_use])),
1247        ] {
1248            let mut fixture = ProviderFixture::builder(&ANTHROPIC).model(&model.model_id()).build().await;
1249            let context = Context::new(
1250                vec![
1251                    ChatMessage::user("Weather in Paris?"),
1252                    assistant_message()
1253                        .model(&author.to_string())
1254                        .reasoning("Check the weather")
1255                        .encrypted_reasoning("msg_1", &thinking[0].to_string())
1256                        .encrypted_reasoning("msg_1", &thinking[1].to_string())
1257                        .text("Checking")
1258                        .tool_call("call_1", "weather", r#"{"city":"Paris"}"#)
1259                        .build(),
1260                    tool_result("call_1", "weather", r#"{"city":"Paris"}"#, "Sunny"),
1261                ],
1262                vec![],
1263            );
1264
1265            let (captured, responses) = fixture.run(&context).await;
1266
1267            assert_successful_stream(&responses);
1268            assert_eq!(captured.body["messages"][1]["content"], expected_content, "{author}");
1269        }
1270    }
1271
1272    #[tokio::test]
1273    async fn anthropic_groups_parallel_tool_results_into_one_user_message() {
1274        let mut fixture = ProviderFixture::builder(&ANTHROPIC).build().await;
1275        let context = Context::new(
1276            vec![
1277                ChatMessage::user("Find things"),
1278                assistant_message().tool_call("call_1", "find", "{}").tool_call("call_2", "find", "{}").build(),
1279                tool_result("call_1", "find", "{}", "call_1 done"),
1280                tool_result("call_2", "find", "{}", "call_2 done"),
1281            ],
1282            vec![],
1283        );
1284
1285        let (captured, responses) = fixture.run(&context).await;
1286
1287        assert_successful_stream(&responses);
1288        let messages = captured.body["messages"].as_array().unwrap();
1289        assert_eq!(messages.len(), 3, "{messages:?}");
1290        assert_eq!(messages[2]["role"], "user");
1291        assert_eq!(
1292            messages[2]["content"],
1293            json!([
1294                { "type": "tool_result", "tool_use_id": "call_1", "content": "call_1 done", "is_error": false },
1295                {
1296                    "type": "tool_result",
1297                    "tool_use_id": "call_2",
1298                    "content": "call_2 done",
1299                    "is_error": false,
1300                    "cache_control": { "type": "ephemeral" }
1301                }
1302            ])
1303        );
1304    }
1305
1306    #[tokio::test]
1307    async fn anthropic_replays_signed_and_redacted_thinking() {
1308        let content = anthropic_replay(
1309            anthropic_stream()
1310                .signed_thinking(0, &["Let me think"], "sig-1")
1311                .redacted_thinking(1, "opaque")
1312                .signed_thinking(2, &[], "sig-2")
1313                .text(3, &["Answer"])
1314                .message_delta("end_turn", &usage(10, 50))
1315                .message_stop(),
1316        )
1317        .await;
1318
1319        assert_eq!(
1320            content,
1321            json!([
1322                { "type": "thinking", "thinking": "Let me think", "signature": "sig-1" },
1323                { "type": "redacted_thinking", "data": "opaque" },
1324                { "type": "thinking", "thinking": "", "signature": "sig-2" },
1325                { "type": "text", "text": "Answer" }
1326            ])
1327        );
1328    }
1329
1330    #[tokio::test]
1331    async fn anthropic_replays_a_reply_in_the_order_the_model_wrote_it() {
1332        let content = anthropic_replay(
1333            anthropic_stream()
1334                .signed_thinking(0, &["Plan"], "sig-1")
1335                .text(1, &["Checking"])
1336                .tool_call(2, "call_1", "first", &["{}"])
1337                .signed_thinking(3, &["Next"], "sig-2")
1338                .text(4, &["Then"])
1339                .tool_call(5, "call_2", "second", &["{}"])
1340                .message_delta("tool_use", &usage(10, 50))
1341                .message_stop(),
1342        )
1343        .await;
1344
1345        assert_eq!(
1346            content,
1347            json!([
1348                { "type": "thinking", "thinking": "Plan", "signature": "sig-1" },
1349                { "type": "text", "text": "Checking" },
1350                { "type": "tool_use", "id": "call_1", "name": "first", "input": {} },
1351                { "type": "thinking", "thinking": "Next", "signature": "sig-2" },
1352                { "type": "text", "text": "Then" },
1353                { "type": "tool_use", "id": "call_2", "name": "second", "input": {} }
1354            ])
1355        );
1356    }
1357
1358    #[tokio::test]
1359    async fn anthropic_unsigned_thinking_is_not_replayable() {
1360        let responses =
1361            streamed_anthropic(anthropic_stream().thinking(0, &["musing"]).text(1, &["Answer"]).message_stop()).await;
1362
1363        assert_eq!(responses, llm_response().reasoning(&["musing"]).text(&["Answer"]).build());
1364    }
1365
1366    #[tokio::test]
1367    async fn anthropic_usage_combines_message_start_input_with_message_delta_output() {
1368        let responses = streamed_anthropic(
1369            anthropic_stream()
1370                .text(0, &["ok"])
1371                .message_delta("end_turn", &json!({ "output_tokens": 25 }))
1372                .message_stop(),
1373        )
1374        .await;
1375
1376        assert_eq!(responses, llm_response().text(&["ok"]).usage(10, 25).build_with_stop_reason(StopReason::EndTurn));
1377    }
1378
1379    const CHAT_FIXTURE: &str = include_str!("../../tests/fixtures/openai/01_minimal.sse");
1380
1381    async fn streamed_chat(body: &str) -> Vec<Result<LlmResponse>> {
1382        let mut fixture = ProviderFixture::builder(&OPENROUTER).response(ResponseSpec::sse(body)).build().await;
1383
1384        let (_, responses) = fixture.run(&hello_context()).await;
1385        responses
1386    }
1387
1388    async fn streamed_anthropic(stream: AnthropicStreamBuilder) -> Vec<LlmResponse> {
1389        let mut fixture = ProviderFixture::builder(&ANTHROPIC).response(ResponseSpec::sse(&stream.sse())).build().await;
1390
1391        let (_, responses) = fixture.run(&hello_context()).await;
1392        responses.into_iter().map(Result::unwrap).collect()
1393    }
1394
1395    async fn anthropic_replay(stream: AnthropicStreamBuilder) -> serde_json::Value {
1396        let model = LlmModel::all().iter().find(|model| model.provider_enum() == Provider::Anthropic).unwrap();
1397        let mut fixture = ProviderFixture::builder(&ANTHROPIC)
1398            .model(&model.model_id())
1399            .response(ResponseSpec::sse(&stream.sse()))
1400            .build()
1401            .await;
1402
1403        let (_, responses) = fixture.run(&hello_context()).await;
1404        let (mut reply, mut results) = (AssistantReply::default(), Vec::new());
1405        for response in responses.into_iter().map(Result::unwrap) {
1406            reply.push(&response);
1407            if let LlmResponse::ToolRequestComplete { tool_call } = response {
1408                let ToolCallRequest { id, name, arguments } = tool_call;
1409                results.push(Ok(ToolCallResult { id, name, arguments, result: "ok".to_string() }));
1410            }
1411        }
1412        let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
1413        context.push_assistant_turn(MessageId::new(), Some(model.clone()), reply, results);
1414        context.add_message(ChatMessage::user("Continue"));
1415
1416        let (captured, _) = fixture.run(&context).await;
1417        captured.body["messages"][1]["content"].clone()
1418    }
1419
1420    fn assert_successful_stream(responses: &[Result<LlmResponse>]) {
1421        assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1422        assert!(responses.iter().any(|response| matches!(response, Ok(LlmResponse::Done { .. }))));
1423    }
1424
1425    fn assert_interrupted(responses: &[Result<LlmResponse>]) {
1426        let error = responses.last().and_then(|response| response.as_ref().err()).expect("the stream must fail");
1427        assert_eq!(error.provider().map(|provider| provider.kind), Some(ProviderErrorKind::StreamInterrupted));
1428        assert!(error.is_retryable());
1429    }
1430
1431    async fn openai_request_body(model: &str, context: &Context) -> serde_json::Value {
1432        let mut fixture = ProviderFixture::builder(&OPENAI).model(model).build().await;
1433
1434        let (captured, responses) = fixture.run(context).await;
1435
1436        assert_successful_stream(&responses);
1437        captured.body
1438    }
1439
1440    async fn anthropic_request_body(model: &LlmModel, effort: ReasoningEffort) -> serde_json::Value {
1441        let mut fixture = ProviderFixture::builder(&ANTHROPIC).model(&model.model_id()).build().await;
1442        let mut context = hello_context();
1443        context.set_reasoning_effort(effort);
1444        context.set_model_settings(ModelSettings { temperature: Some(0.5), top_p: Some(0.5), max_tokens: Some(500) });
1445
1446        let (captured, responses) = fixture.run(&context).await;
1447
1448        assert_successful_stream(&responses);
1449        captured.body
1450    }
1451
1452    fn reasoning_off_model(provider: Provider) -> &'static LlmModel {
1453        LlmModel::all()
1454            .iter()
1455            .find(|model| model.provider_enum() == provider && model.supports_reasoning_off())
1456            .expect("the catalog has a model that can turn reasoning off")
1457    }
1458
1459    fn anthropic_model(control: ReasoningControl, efforts: &[ReasoningEffort]) -> &'static LlmModel {
1460        LlmModel::all()
1461            .iter()
1462            .find(|model| {
1463                model.provider_enum() == Provider::Anthropic
1464                    && model.reasoning_control() == control
1465                    && efforts.iter().all(|effort| model.validate_reasoning_effort(*effort).is_ok())
1466            })
1467            .unwrap_or_else(|| panic!("the catalog has no Anthropic model with {control:?} reasoning and {efforts:?}"))
1468    }
1469
1470    async fn rejected_chat_request(spec: ResponseSpec) -> LlmError {
1471        let mut fixture = ProviderFixture::builder(&OPENROUTER)
1472            .response(spec.with_header("x-request-id", "request-123"))
1473            .build()
1474            .await;
1475
1476        let (_, mut events) = fixture.run(&hello_context()).await;
1477
1478        assert_eq!(events.len(), 1, "a rejected request must yield exactly one error: {events:?}");
1479        events.pop().unwrap().expect_err("request must fail")
1480    }
1481}
llm/providers/generic.rs

llm/providers/
generic.rs
