llm/providers/
generic.rs

1#![doc = include_str!(concat!(env!("OUT_DIR"), "/docs/generic.md"))]
2
3use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderName, HeaderValue};
4use serde::Serialize;
5use std::borrow::Cow;
6use std::future::Future;
7use std::time::Duration;
8use utils::variables::Vars;
9
10use crate::catalog::Provider;
11use crate::provider::{display_name, validate_reasoning};
12use crate::providers::anthropic::mappers::build_request as build_messages_request;
13use crate::providers::anthropic::streaming::decode_line;
14use crate::providers::http::{SseData, anthropic_code, http_client, post_sse, responses_code};
15pub use crate::providers::openai::chat::ChatRequestPolicy;
16use crate::providers::openai::chat::build_chat_request;
17use crate::providers::openai::chat::streaming::decode_sse_chunk;
18use crate::providers::openai::chat::types::CompatibleChatRequest;
19use crate::providers::openai::responses::mappers::build_request as build_responses_request;
20use crate::providers::openai::responses::streaming::decode_responses;
21use crate::providers::openai::wire::PromptCacheKeySource;
22use crate::providers::response_stream::{OpenedStream, error_stream, response_stream};
23use crate::tool_schema::normalize_for_moonshot;
24use crate::{
25    Context, LlmError, LlmModel, LlmResponseStream, ProviderAuthMode, ProviderConnectionConfig, Result,
26    StreamingModelProvider,
27};
28
29pub use crate::providers::openai::responses::mappers::ResponsesRequestPolicy;
30
31pub struct ProviderConfig {
32    pub provider: Provider,
33    pub api_base: Option<&'static str>,
34    pub api_base_env: Option<&'static str>,
35    pub default_model: &'static str,
36    pub api: Api,
37}
38
39pub enum Api {
40    ChatCompletions(ChatRequestPolicy),
41    Responses(ResponsesRequestPolicy),
42    Messages,
43}
44
45pub const ANTHROPIC: ProviderConfig = ProviderConfig {
46    provider: Provider::Anthropic,
47    api_base: Some("https://api.anthropic.com"),
48    api_base_env: None,
49    default_model: "claude-sonnet-4-5-20250929",
50    api: Api::Messages,
51};
52
53pub const OPENAI: ProviderConfig = ProviderConfig {
54    provider: Provider::Openai,
55    api_base: Some("https://api.openai.com/v1"),
56    api_base_env: None,
57    default_model: "gpt-4.1",
58    api: Api::Responses(ResponsesRequestPolicy::OPENAI),
59};
60
61pub const XIAOMI: ProviderConfig = ProviderConfig {
62    provider: Provider::Xiaomi,
63    api_base: Some("https://api.xiaomimimo.com/v1"),
64    api_base_env: None,
65    default_model: "mimo-v2.6-pro",
66    api: Api::Responses(ResponsesRequestPolicy::XIAOMI),
67};
68
69pub const DEEPSEEK: ProviderConfig = ProviderConfig {
70    provider: Provider::DeepSeek,
71    api_base: Some("https://api.deepseek.com"),
72    api_base_env: None,
73    default_model: "deepseek-v4-flash",
74    api: Api::ChatCompletions(ChatRequestPolicy::DEFAULT),
75};
76
77pub const MOONSHOT: ProviderConfig = ProviderConfig {
78    provider: Provider::Moonshot,
79    api_base: Some("https://api.moonshot.ai/v1"),
80    api_base_env: None,
81    default_model: "moonshot-v1-8k",
82    api: Api::ChatCompletions(ChatRequestPolicy {
83        tool_schema_transform: Some(normalize_for_moonshot),
84        ..ChatRequestPolicy::DEFAULT
85    }),
86};
87
88pub const ZAI: ProviderConfig = ProviderConfig {
89    provider: Provider::ZAi,
90    api_base: Some("https://api.z.ai/api/coding/paas/v4"),
91    api_base_env: None,
92    default_model: "GLM-4.6",
93    api: Api::ChatCompletions(ChatRequestPolicy::DEFAULT),
94};
95
96pub const AZURE_FOUNDRY: ProviderConfig = ProviderConfig {
97    provider: Provider::AzureFoundry,
98    api_base: None,
99    api_base_env: None,
100    default_model: "gpt-5.5",
101    api: Api::ChatCompletions(ChatRequestPolicy {
102        prompt_cache_key: PromptCacheKeySource::Prefix,
103        ..ChatRequestPolicy::DEFAULT
104    }),
105};
106
107pub const FIREWORKS: ProviderConfig = ProviderConfig {
108    provider: Provider::Fireworks,
109    api_base: Some("https://api.fireworks.ai/inference/v1"),
110    api_base_env: None,
111    default_model: "accounts/fireworks/models/glm-5p1",
112    api: Api::ChatCompletions(ChatRequestPolicy {
113        prompt_cache_key: PromptCacheKeySource::SessionAffinity,
114        ..ChatRequestPolicy::DEFAULT
115    }),
116};
117
118pub const GEMINI: ProviderConfig = ProviderConfig {
119    provider: Provider::Gemini,
120    api_base: Some("https://generativelanguage.googleapis.com/v1beta/openai"),
121    api_base_env: None,
122    default_model: "",
123    api: Api::ChatCompletions(ChatRequestPolicy::DEFAULT),
124};
125
126pub const OPENROUTER: ProviderConfig = ProviderConfig {
127    provider: Provider::OpenRouter,
128    api_base: Some("https://openrouter.ai/api/v1"),
129    api_base_env: None,
130    default_model: "",
131    api: Api::ChatCompletions(ChatRequestPolicy {
132        prompt_cache_key: PromptCacheKeySource::Prefix,
133        customize: Some(openrouter_request),
134        ..ChatRequestPolicy::DEFAULT
135    }),
136};
137
138pub const OLLAMA: ProviderConfig = ProviderConfig {
139    provider: Provider::Ollama,
140    api_base: Some("http://localhost:11434/v1"),
141    api_base_env: Some("OLLAMA_HOST"),
142    default_model: "",
143    api: Api::ChatCompletions(LOCAL_CHAT),
144};
145
146pub const LLAMACPP: ProviderConfig = ProviderConfig {
147    provider: Provider::LlamaCpp,
148    api_base: Some("http://localhost:8080/v1"),
149    api_base_env: Some("LLAMA_CPP_HOST"),
150    default_model: "",
151    api: Api::ChatCompletions(LOCAL_CHAT),
152};
153
154const LOCAL_CHAT: ChatRequestPolicy =
155    ChatRequestPolicy { tool_call_reasoning_placeholder: false, ..ChatRequestPolicy::DEFAULT };
156
157pub(crate) const BUILT_INS: &[&ProviderConfig] = &[
158    &ANTHROPIC,
159    &OPENAI,
160    &XIAOMI,
161    &DEEPSEEK,
162    &MOONSHOT,
163    &ZAI,
164    &AZURE_FOUNDRY,
165    &FIREWORKS,
166    &GEMINI,
167    &OPENROUTER,
168    &OLLAMA,
169    &LLAMACPP,
170];
171
172/// A provider whose behavior is fully described by a [`ProviderConfig`].
173#[derive(Clone)]
174pub struct GenericProvider {
175    config: &'static ProviderConfig,
176    http: reqwest::Client,
177    api_base: String,
178    headers: HeaderMap,
179    model: String,
180    request_model: Option<String>,
181    idle_timeout: Duration,
182}
183
184impl GenericProvider {
185    pub fn from_env(config: &'static ProviderConfig) -> Result<Self> {
186        Self::from_env_with_connection(config, ProviderConnectionConfig::default())
187    }
188
189    pub fn from_env_with_connection(
190        config: &'static ProviderConfig,
191        connection: ProviderConnectionConfig,
192    ) -> Result<Self> {
193        Self::from_vars(config, connection, &Vars::new())
194    }
195
196    pub fn from_vars(
197        config: &'static ProviderConfig,
198        mut connection: ProviderConnectionConfig,
199        vars: &Vars,
200    ) -> Result<Self> {
201        let api_key = match (connection.auth_mode, config.provider.required_env_var()) {
202            (ProviderAuthMode::Default, Some(env_var)) => {
203                vars.get(env_var).ok_or_else(|| LlmError::MissingApiKey(env_var.to_string()))?.into_owned()
204            }
205            _ => String::new(),
206        };
207        connection.base_url = connection
208            .base_url
209            .or_else(|| config.api_base_env.and_then(|env_var| vars.get(env_var)).map(Cow::into_owned));
210        Self::new_with_connection(&api_key, config, connection)
211    }
212
213    pub fn new(api_key: &str, config: &'static ProviderConfig) -> Result<Self> {
214        Self::new_with_connection(api_key, config, ProviderConnectionConfig::default())
215    }
216
217    pub fn new_with_connection(
218        api_key: &str,
219        config: &'static ProviderConfig,
220        connection: ProviderConnectionConfig,
221    ) -> Result<Self> {
222        let api_base = connection
223            .base_url
224            .or_else(|| config.api_base.map(str::to_string))
225            .ok_or_else(|| LlmError::MissingProviderUrl { provider: config.provider.parser_name().to_string() })?;
226        let mut api_base = api_base.trim_end_matches('/').to_string();
227        if config.provider.is_local() && !api_base.ends_with("/v1") {
228            api_base.push_str("/v1");
229        }
230        Ok(Self {
231            config,
232            http: http_client(),
233            api_base,
234            headers: api_headers(&config.api, api_key, connection.auth_mode)?,
235            model: config.default_model.to_string(),
236            request_model: connection.request_model,
237            idle_timeout: connection.idle_timeout,
238        })
239    }
240
241    pub fn with_model(mut self, model: &str) -> Self {
242        self.model = model.to_string();
243        self
244    }
245}
246
247impl StreamingModelProvider for GenericProvider {
248    fn stream_response(&self, context: &Context) -> LlmResponseStream {
249        let identity = self.model();
250        let wire_model = self.request_model.as_deref().unwrap_or(&self.model);
251        let stream = validate_reasoning(context, identity.as_ref()).and_then(|()| match &self.config.api {
252            Api::ChatCompletions(policy) => self.stream_chat_completions(wire_model, context, policy),
253            Api::Responses(policy) => self.stream_responses(identity.as_ref(), wire_model, context, policy),
254            Api::Messages => self.stream_messages(wire_model, context),
255        });
256        stream.unwrap_or_else(error_stream)
257    }
258
259    fn display_name(&self) -> String {
260        display_name(self.config.provider, &self.model)
261    }
262
263    fn model(&self) -> Option<LlmModel> {
264        LlmModel::parse_for(self.config.provider, &self.model).ok()
265    }
266}
267
268impl GenericProvider {
269    fn stream_chat_completions(
270        &self,
271        wire_model: &str,
272        context: &Context,
273        policy: &ChatRequestPolicy,
274    ) -> Result<LlmResponseStream> {
275        let request = build_chat_request(wire_model, context, policy)?;
276        let opened = self.post("chat/completions", &request, responses_code);
277        Ok(response_stream(opened, |data, turn| decode_sse_chunk(&data, turn), self.idle_timeout))
278    }
279
280    fn stream_responses(
281        &self,
282        identity: Option<&LlmModel>,
283        wire_model: &str,
284        context: &Context,
285        policy: &ResponsesRequestPolicy,
286    ) -> Result<LlmResponseStream> {
287        let request = build_responses_request(identity, wire_model, context, policy)?;
288        Ok(response_stream(self.post("responses", &request, responses_code), decode_responses(), self.idle_timeout))
289    }
290
291    fn stream_messages(&self, wire_model: &str, context: &Context) -> Result<LlmResponseStream> {
292        let request = build_messages_request(wire_model, context)?;
293        let opened = self.post("v1/messages", &request, anthropic_code);
294        Ok(response_stream(opened, |data, turn| decode_line(&data, turn), self.idle_timeout))
295    }
296
297    fn post<T: Serialize>(
298        &self,
299        path: &str,
300        body: &T,
301        code: fn(&str) -> Option<String>,
302    ) -> impl Future<Output = Result<OpenedStream<SseData>>> + Send + use<T> {
303        post_sse(&self.http, &format!("{}/{path}", self.api_base), self.headers.clone(), body, code)
304    }
305}
306
307fn api_headers(api: &Api, api_key: &str, auth_mode: ProviderAuthMode) -> Result<HeaderMap> {
308    let mut headers = HeaderMap::new();
309    if let Api::Messages = api {
310        headers.insert("anthropic-version", HeaderValue::from_static("2023-06-01"));
311    }
312    if auth_mode == ProviderAuthMode::Default && !api_key.is_empty() {
313        let (name, value) = match api {
314            Api::ChatCompletions(_) | Api::Responses(_) => (AUTHORIZATION, format!("Bearer {api_key}")),
315            Api::Messages => (HeaderName::from_static("x-api-key"), api_key.to_string()),
316        };
317        let mut value = HeaderValue::from_str(&value)?;
318        value.set_sensitive(true);
319        headers.insert(name, value);
320    }
321    Ok(headers)
322}
323
324fn openrouter_request(request: &mut CompatibleChatRequest, context: &Context) {
325    request.extra.insert("usage".into(), serde_json::json!({ "include": true }));
326    request.extra.insert("cache_control".into(), serde_json::json!({ "type": "ephemeral" }));
327    if let Some(session_id) = context.session_affinity_key() {
328        request.extra.insert("session_id".into(), session_id.into());
329    }
330    if request.reasoning_effort == Some("none") {
331        request.reasoning_effort = None;
332        request.extra.insert("reasoning".into(), serde_json::json!({ "effort": "none" }));
333    }
334}
335
336#[cfg(test)]
337mod tests {
338    use futures::StreamExt;
339    use serde_json::json;
340
341    use super::*;
342    use crate::providers::test_capture_server::{CaptureServer, ProviderFixture, ResponseSpec, hello_context};
343    use crate::types::IsoString;
344    use crate::{
345        AssistantReasoning, ChatMessage, ContentBlock, EncryptedReasoningContent, LlmResponse, MessageId,
346        ModelSettings, ProviderErrorKind, ReasoningEffort, ToolCallError, ToolCallRequest, ToolCallResult,
347        ToolDefinition,
348    };
349
350    #[tokio::test]
351    async fn gemini_disables_reasoning_with_none() {
352        let model = reasoning_off_model(Provider::Gemini);
353        let mut fixture = ProviderFixture::builder(&GEMINI).model(&model.model_id()).build().await;
354        let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
355        context.set_reasoning_effort(ReasoningEffort::Disabled);
356
357        let (captured, responses) = fixture.run(&context).await;
358
359        assert_successful_stream(&responses);
360        assert_eq!(captured.body["reasoning_effort"], "none");
361    }
362
363    #[tokio::test]
364    async fn moonshot_normalizes_tool_schemas() {
365        let parameters = json!({
366            "$schema": "https://json-schema.org/draft/2020-12/schema",
367            "title": "MySchema",
368            "type": "object",
369            "properties": { "q": { "type": "string", "format": "uri" } }
370        });
371        let mut fixture = ProviderFixture::builder(&MOONSHOT).build().await;
372        let context =
373            Context::new(vec![ChatMessage::user("hi")], vec![ToolDefinition::new("search", "Search", parameters)]);
374
375        let (captured, responses) = fixture.run(&context).await;
376        let body = captured.body;
377
378        assert_successful_stream(&responses);
379        let params = &body["tools"][0]["function"]["parameters"];
380        assert!(params.get("$schema").is_none());
381        assert!(params.get("title").is_none());
382        assert!(params["properties"]["q"].get("format").is_none());
383        assert_eq!(params["type"], "object");
384    }
385
386    #[tokio::test]
387    async fn chat_completions_send_errors_and_summaries_as_user_notes() {
388        let mut fixture = ProviderFixture::builder(&DEEPSEEK).build().await;
389        let context = Context::new(
390            vec![
391                ChatMessage::Error {
392                    message_id: MessageId::new(),
393                    message: "boom".to_string(),
394                    timestamp: IsoString::now(),
395                },
396                ChatMessage::Summary {
397                    message_id: MessageId::new(),
398                    content: "earlier work".to_string(),
399                    timestamp: IsoString::now(),
400                    messages_compacted: 2,
401                },
402            ],
403            vec![],
404        );
405
406        let (captured, responses) = fixture.run(&context).await;
407        let body = captured.body;
408
409        assert_successful_stream(&responses);
410        assert_eq!(body["messages"][0], json!({ "role": "user", "content": "Error: boom" }));
411        assert_eq!(
412            body["messages"][1],
413            json!({ "role": "user", "content": "[Previous conversation handoff]\n\nearlier work" })
414        );
415    }
416
417    #[tokio::test]
418    async fn openrouter_requests_usage_caching_and_session_routing() {
419        let mut fixture = ProviderFixture::builder(&OPENROUTER).model("anthropic/claude-haiku-4.5").build().await;
420        let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
421        context.set_prompt_cache_key(Some("prefix-abc".to_string()));
422        context.set_session_affinity_key(Some("conversation-abc".to_string()));
423        context.set_reasoning_effort(ReasoningEffort::High);
424
425        let (captured, responses) = fixture.run(&context).await;
426        let body = captured.body;
427
428        assert_successful_stream(&responses);
429        assert_eq!(body["model"], "anthropic/claude-haiku-4.5");
430        assert_eq!(body["prompt_cache_key"], "prefix-abc");
431        assert_eq!(body["session_id"], "conversation-abc");
432        assert_eq!(body["reasoning_effort"], "high");
433        assert_eq!(body["cache_control"]["type"], "ephemeral");
434        assert_eq!(body["usage"]["include"], true);
435        assert_eq!(fixture.provider.display_name(), "OpenRouter (anthropic/claude-haiku-4.5)");
436    }
437
438    #[tokio::test]
439    async fn openrouter_disables_reasoning_through_its_unified_reasoning_object() {
440        let model = reasoning_off_model(Provider::OpenRouter);
441        let mut fixture = ProviderFixture::builder(&OPENROUTER).model(&model.model_id()).build().await;
442        let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
443        context.set_reasoning_effort(ReasoningEffort::Disabled);
444
445        let (captured, responses) = fixture.run(&context).await;
446        let body = captured.body;
447
448        assert_successful_stream(&responses);
449        assert_eq!(body["reasoning"], json!({ "effort": "none" }));
450        assert!(body.get("reasoning_effort").is_none());
451    }
452
453    #[tokio::test]
454    async fn chat_rate_limit_rejection_is_a_retryable_provider_error() {
455        let body = r#"{"error":{"message":"Rate limit exceeded: free-models-per-day","code":429}}"#;
456        let error = rejected_chat_request(ResponseSpec::sse(body).with_status(429)).await;
457        let provider_error = error.provider().expect("a 429 must classify as a provider error");
458
459        assert_eq!(provider_error.kind, ProviderErrorKind::RateLimit);
460        assert_eq!(provider_error.http_status, Some(429));
461        assert_eq!(provider_error.code.as_deref(), Some("429"));
462        assert_eq!(provider_error.request_id.as_deref(), Some("request-123"));
463        assert!(provider_error.message.contains("free-models-per-day"));
464        assert!(error.is_retryable(), "429 must be retryable, got {error}");
465    }
466
467    #[tokio::test]
468    async fn chat_rejection_preserves_string_error_codes() {
469        let body = r#"{"error":{"code":"rate_limit_exceeded","message":"slow down"}}"#;
470        let error = rejected_chat_request(ResponseSpec::sse(body).with_status(429)).await;
471        let provider_error = error.provider().unwrap();
472
473        assert_eq!(provider_error.code.as_deref(), Some("rate_limit_exceeded"));
474        assert!(provider_error.message.contains("slow down"));
475        assert!(error.is_retryable());
476    }
477
478    #[tokio::test]
479    async fn chat_bad_request_is_not_retryable() {
480        let body = r#"{"error":{"message":"invalid request"}}"#;
481        let error = rejected_chat_request(ResponseSpec::sse(body).with_status(400)).await;
482
483        assert_eq!(error.provider().map(|provider| provider.kind), Some(ProviderErrorKind::Api));
484        assert!(!error.is_retryable());
485    }
486
487    #[tokio::test]
488    async fn chat_rejection_keeps_http_status_when_the_error_body_fails_to_read() {
489        let error = rejected_chat_request(ResponseSpec::sse("").with_status(429).interrupted()).await;
490        let provider_error = error.provider().unwrap();
491
492        assert!(provider_error.message.ends_with("Unknown error"), "{provider_error:?}");
493        assert_eq!(provider_error.http_status, Some(429));
494        assert_eq!(provider_error.request_id.as_deref(), Some("request-123"));
495        assert_eq!(provider_error.kind, ProviderErrorKind::RateLimit);
496        assert!(error.is_retryable());
497    }
498
499    #[tokio::test]
500    async fn chat_chunk_that_is_not_json_interrupts_the_stream() {
501        let responses = streamed_chat("data: invalid json\n\n").await;
502
503        assert!(matches!(responses[..], [Ok(LlmResponse::Start), Err(_)]), "{responses:?}");
504        assert_interrupted(&responses);
505    }
506
507    #[tokio::test]
508    async fn chat_stream_ends_at_done() {
509        let responses = streamed_chat(&format!("{CHAT_FIXTURE}data: invalid json\n\n")).await;
510
511        assert_successful_stream(&responses);
512    }
513
514    #[tokio::test]
515    async fn chat_stream_skips_keepalive_events() {
516        let responses = streamed_chat(&format!("event: keepalive\ndata: ping\n\n{CHAT_FIXTURE}")).await;
517
518        assert_successful_stream(&responses);
519    }
520
521    #[tokio::test]
522    async fn chat_stream_cut_off_mid_tool_call_is_interrupted() {
523        let tool_call_start = json!({ "choices": [{ "index": 0, "delta": { "tool_calls": [{
524            "index": 0, "id": "call_1", "type": "function",
525            "function": { "name": "bash", "arguments": r#"{"command":"# }
526        }] } }] });
527
528        let responses = streamed_chat(&format!("data: {tool_call_start}\n\n")).await;
529
530        assert!(
531            !responses.iter().any(|response| matches!(
532                response,
533                Ok(LlmResponse::ToolRequestComplete { .. } | LlmResponse::Done { .. })
534            )),
535            "{responses:?}"
536        );
537        assert_interrupted(&responses);
538    }
539
540    #[tokio::test]
541    async fn disabled_toggle_and_unknown_models_never_send_requests() {
542        let mut fixture = ProviderFixture::builder(&DEEPSEEK).build().await;
543        for model in ["deepseek-v4-flash", "unknown"] {
544            fixture.provider = fixture.provider.with_model(model);
545            let mut context = Context::new(vec![], vec![]);
546            context.set_reasoning_effort(ReasoningEffort::Disabled);
547            let responses = fixture.provider.stream_response(&context).collect::<Vec<_>>().await;
548            assert_eq!(responses.len(), 1, "{responses:?}");
549            let error = responses[0].as_ref().unwrap_err();
550            if model == "unknown" {
551                assert!(matches!(error, LlmError::ReasoningValidation(_)));
552            } else {
553                assert!(matches!(error, LlmError::UnsupportedDisableTransport { .. }));
554            }
555            assert!(!error.is_retryable());
556        }
557    }
558
559    #[tokio::test]
560    async fn sampling_settings_are_sent_as_configured() {
561        for config in [&DEEPSEEK, &OPENAI, &ANTHROPIC] {
562            let mut fixture = ProviderFixture::builder(config).build().await;
563            let mut context = hello_context();
564            context.set_model_settings(ModelSettings { temperature: Some(0.2), top_p: Some(0.9), max_tokens: None });
565
566            let (captured, responses) = fixture.run(&context).await;
567            let body = captured.body;
568
569            assert!(responses.iter().all(Result::is_ok), "{responses:?}");
570            assert_eq!(body["temperature"], 0.2, "{}", config.provider);
571            assert_eq!(body["top_p"], 0.9, "{}", config.provider);
572        }
573    }
574
575    #[tokio::test]
576    async fn every_api_validates_reasoning_before_sending() {
577        for config in [&DEEPSEEK, &OPENAI, &ANTHROPIC] {
578            let fixture = ProviderFixture::builder(config).model("unknown").build().await;
579            let mut context = hello_context();
580            context.set_reasoning_effort(ReasoningEffort::Disabled);
581
582            let responses = fixture.provider.stream_response(&context).collect::<Vec<_>>().await;
583
584            assert!(matches!(responses[..], [Err(LlmError::ReasoningValidation(_))]), "{responses:?}");
585        }
586    }
587
588    #[test]
589    fn display_name_names_models_outside_the_catalog() {
590        let provider = GenericProvider::new("key", &DEEPSEEK).unwrap().with_model("my-custom-model");
591        assert_eq!(provider.display_name(), "DeepSeek (my-custom-model)");
592
593        let provider = GenericProvider::new("key", &DEEPSEEK).unwrap().with_model("");
594        assert_eq!(provider.display_name(), "DeepSeek");
595    }
596
597    #[test]
598    fn azure_foundry_requires_a_configured_url() {
599        let Err(error) = GenericProvider::new("key", &AZURE_FOUNDRY) else {
600            panic!("Azure Foundry must require a URL");
601        };
602        assert!(matches!(error, LlmError::MissingProviderUrl { provider } if provider == "azure-foundry"));
603    }
604
605    #[tokio::test]
606    async fn request_model_routes_each_api_without_changing_catalog_identity() {
607        for (config, model, deployment, identity, display_name, auth_mode) in [
608            (
609                &AZURE_FOUNDRY,
610                "gpt-5.5",
611                "production-coding",
612                "azure-foundry:gpt-5.5",
613                "Microsoft Foundry (gpt-5.5)",
614                ProviderAuthMode::None,
615            ),
616            (
617                &XIAOMI,
618                "mimo-v2.6-pro",
619                "mimo-deployment",
620                "xiaomi:mimo-v2.6-pro",
621                "Xiaomi (mimo-v2.6-pro)",
622                ProviderAuthMode::None,
623            ),
624            (
625                &ANTHROPIC,
626                "claude-sonnet-4-5-20250929",
627                "claude-deployment",
628                "anthropic:claude-sonnet-4-5-20250929",
629                "Anthropic (claude-sonnet-4-5-20250929)",
630                ProviderAuthMode::Default,
631            ),
632        ] {
633            let mut fixture = ProviderFixture::builder(config)
634                .model(model)
635                .connection(ProviderConnectionConfig {
636                    request_model: Some(deployment.into()),
637                    auth_mode,
638                    ..Default::default()
639                })
640                .build()
641                .await;
642
643            let (captured, responses) = fixture.run(&hello_context()).await;
644
645            assert_successful_stream(&responses);
646            assert_eq!(captured.body["model"], deployment);
647            assert_eq!(captured.body["stream"], true);
648            assert!(captured.headers.get("authorization").is_none());
649            assert_eq!(fixture.provider.model().unwrap().to_string(), identity);
650            assert_eq!(fixture.provider.display_name(), display_name);
651            match config.api {
652                Api::ChatCompletions(_) => {
653                    assert_eq!(captured.path, "/chat/completions");
654                    assert_eq!(captured.body["stream_options"]["include_usage"], true);
655                }
656                Api::Responses(_) => assert_eq!(captured.path, "/responses"),
657                Api::Messages => {
658                    assert_eq!(captured.path, "/v1/messages");
659                    assert_eq!(captured.headers["anthropic-version"], "2023-06-01");
660                    assert_eq!(captured.headers["x-api-key"], "key");
661                }
662            }
663        }
664    }
665
666    #[tokio::test]
667    async fn default_auth_sends_the_api_key_as_a_bearer_token() {
668        let mut fixture = ProviderFixture::builder(&DEEPSEEK)
669            .api_key("secret")
670            .connection(ProviderConnectionConfig::default())
671            .build()
672            .await;
673        let (captured, responses) = fixture.run(&hello_context()).await;
674
675        assert_successful_stream(&responses);
676        assert_eq!(captured.headers["authorization"], "Bearer secret");
677    }
678
679    #[tokio::test]
680    async fn chat_providers_apply_their_declared_prompt_cache_policy() {
681        for (config, expected_key) in [
682            (&AZURE_FOUNDRY, Some("prefix-abc")),
683            (&FIREWORKS, Some("conversation-abc")),
684            (&DEEPSEEK, None),
685            (&MOONSHOT, None),
686            (&ZAI, None),
687        ] {
688            for keys_set in [true, false] {
689                let mut fixture = ProviderFixture::builder(config).build().await;
690                let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
691                if keys_set {
692                    context.set_prompt_cache_key(Some("prefix-abc".to_string()));
693                    context.set_session_affinity_key(Some("conversation-abc".to_string()));
694                }
695
696                let (captured, responses) = fixture.run(&context).await;
697
698                assert_successful_stream(&responses);
699                let sent_key = captured.body.get("prompt_cache_key").and_then(serde_json::Value::as_str);
700                assert_eq!(sent_key, expected_key.filter(|_| keys_set), "{} keys_set={keys_set}", config.provider);
701                assert!(captured.body.get("user").is_none());
702                assert!(captured.body.get("session_id").is_none());
703            }
704        }
705    }
706
707    #[tokio::test]
708    async fn local_providers_send_bare_chat_requests_under_v1() {
709        for (config, display_name) in [(&OLLAMA, "Ollama (test-model)"), (&LLAMACPP, "LlamaCpp (test-model)")] {
710            let mut server = CaptureServer::start_chat_completions().await;
711            let connection = ProviderConnectionConfig { base_url: Some(server.base_url.clone()), ..Default::default() };
712            let provider =
713                GenericProvider::from_env_with_connection(config, connection).unwrap().with_model("test-model");
714            let tool_call = ChatMessage::Assistant {
715                message_id: MessageId::new(),
716                content: String::new(),
717                reasoning: AssistantReasoning::default(),
718                timestamp: IsoString::now(),
719                tool_calls: vec![ToolCallRequest { id: "call_1".into(), name: "tool".into(), arguments: "{}".into() }],
720            };
721            let mut context = Context::new(vec![ChatMessage::user("Hello"), tool_call], vec![]);
722            context.set_prompt_cache_key(Some("prefix-abc".to_string()));
723            context.set_session_affinity_key(Some("conversation-abc".to_string()));
724
725            let responses = provider.stream_response(&context).collect::<Vec<_>>().await;
726            let captured = server.captured().await;
727
728            assert_successful_stream(&responses);
729            assert_eq!(captured.path, "/v1/chat/completions");
730            assert_eq!(captured.body["model"], "test-model");
731            assert!(captured.headers.get("authorization").is_none());
732            assert!(captured.body.get("prompt_cache_key").is_none());
733            assert!(captured.body["messages"][1].get("reasoning_content").is_none());
734            assert_eq!(provider.display_name(), display_name);
735        }
736    }
737
738    #[tokio::test]
739    async fn local_providers_send_requests_to_their_host_variable() {
740        for (config, host_var) in [(&OLLAMA, "OLLAMA_HOST"), (&LLAMACPP, "LLAMA_CPP_HOST")] {
741            let mut server = CaptureServer::start_chat_completions().await;
742            let vars = Vars::new().with_env_lookup(|_| None).with(host_var, server.base_url.clone());
743            let provider = GenericProvider::from_vars(config, ProviderConnectionConfig::default(), &vars)
744                .unwrap()
745                .with_model("test-model");
746
747            let responses = provider.stream_response(&hello_context()).collect::<Vec<_>>().await;
748
749            assert_successful_stream(&responses);
750            assert_eq!(server.captured().await.path, "/v1/chat/completions", "{host_var}");
751        }
752    }
753
754    #[tokio::test]
755    async fn connection_url_takes_precedence_over_the_host_variable() {
756        let mut server = CaptureServer::start_chat_completions().await;
757        let vars = Vars::new().with_env_lookup(|_| None).with("OLLAMA_HOST", "http://127.0.0.1:1");
758        let connection = ProviderConnectionConfig { base_url: Some(server.base_url.clone()), ..Default::default() };
759        let provider = GenericProvider::from_vars(&OLLAMA, connection, &vars).unwrap().with_model("test-model");
760
761        let responses = provider.stream_response(&hello_context()).collect::<Vec<_>>().await;
762
763        assert_successful_stream(&responses);
764        assert_eq!(server.captured().await.path, "/v1/chat/completions");
765    }
766
767    #[tokio::test]
768    async fn providers_honor_configured_idle_timeout() {
769        for (config, fixture) in [
770            (&OLLAMA, include_str!("../../tests/fixtures/openai/01_minimal.sse")),
771            (&ANTHROPIC, include_str!("../../tests/fixtures/anthropic/01_minimal.sse")),
772        ] {
773            let spec = ResponseSpec::sse(fixture).paced(Duration::from_mins(2));
774            let mut fixture = ProviderFixture::builder(config)
775                .response(spec)
776                .connection(ProviderConnectionConfig {
777                    auth_mode: ProviderAuthMode::None,
778                    idle_timeout: Duration::from_mins(1),
779                    ..Default::default()
780                })
781                .build()
782                .await;
783
784            let responses =
785                fixture.server.collect_on_paused_clock(fixture.provider.stream_response(&hello_context())).await;
786
787            let error = responses.last().and_then(|response| response.as_ref().err()).and_then(LlmError::provider);
788            assert_eq!(error.map(|error| error.kind), Some(ProviderErrorKind::Timeout), "{responses:?}");
789        }
790    }
791
792    #[tokio::test]
793    async fn responses_send_user_input_without_unset_options() {
794        let body = openai_request_body("gpt-4.1", &hello_context()).await;
795
796        assert_eq!(body["model"], "gpt-4.1");
797        assert_eq!(
798            body["input"],
799            json!([{
800                "type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]
801            }])
802        );
803        for field in ["instructions", "tools", "reasoning", "temperature", "top_p", "max_output_tokens"] {
804            assert!(body.get(field).is_none(), "{field}: {body}");
805        }
806    }
807
808    #[tokio::test]
809    async fn responses_lift_system_instructions_and_apply_model_settings() {
810        let mut context = Context::new(vec![ChatMessage::system("You are helpful."), ChatMessage::user("Hi")], vec![]);
811        context.set_model_settings(ModelSettings { temperature: Some(0.0), top_p: Some(0.5), max_tokens: Some(128) });
812
813        let body = openai_request_body("gpt-4.1", &context).await;
814
815        assert_eq!(body["instructions"], "You are helpful.");
816        assert_eq!(body["input"].as_array().unwrap().len(), 1);
817        assert_eq!(body["input"][0]["role"], "user");
818        assert_eq!(body["temperature"], 0.0);
819        assert_eq!(body["top_p"], 0.5);
820        assert_eq!(body["max_output_tokens"], 128);
821    }
822
823    #[tokio::test]
824    async fn responses_preserve_tool_conversations_and_definitions() {
825        let arguments = r#"{"path":"foo.rs"}"#;
826        let context = Context::new(
827            vec![
828                ChatMessage::user("Read foo.rs"),
829                ChatMessage::Assistant {
830                    message_id: MessageId::new(),
831                    content: "I'll read that file.".into(),
832                    reasoning: AssistantReasoning::default(),
833                    timestamp: IsoString::now(),
834                    tool_calls: vec![ToolCallRequest {
835                        id: "call_1".into(),
836                        name: "read_file".into(),
837                        arguments: arguments.into(),
838                    }],
839                },
840                ChatMessage::ToolCallResult(Ok(ToolCallResult {
841                    id: "call_1".into(),
842                    name: "read_file".into(),
843                    arguments: arguments.into(),
844                    result: "fn main() {}".into(),
845                })),
846                ChatMessage::Assistant {
847                    message_id: MessageId::new(),
848                    content: "Here's the file content.".into(),
849                    reasoning: AssistantReasoning::default(),
850                    timestamp: IsoString::now(),
851                    tool_calls: vec![],
852                },
853            ],
854            vec![ToolDefinition::new(
855                "read_file",
856                "Read a file from disk",
857                json!({
858                    "type": "object", "properties": { "path": { "type": "string" } }
859                }),
860            )],
861        );
862
863        let body = openai_request_body("gpt-4.1", &context).await;
864
865        assert_eq!(
866            body["input"],
867            json!([
868                { "type": "message", "role": "user", "content": [{"type": "input_text", "text": "Read foo.rs"}] },
869                { "type": "message", "role": "assistant", "content": "I'll read that file." },
870                { "type": "function_call", "arguments": arguments, "call_id": "call_1", "name": "read_file" },
871                { "type": "function_call_output", "call_id": "call_1", "output": "fn main() {}" },
872                { "type": "message", "role": "assistant", "content": "Here's the file content." },
873            ])
874        );
875        let tools = body["tools"].as_array().unwrap();
876        assert_eq!(tools.len(), 1);
877        assert_eq!(tools[0]["type"], "function");
878        assert_eq!(tools[0]["name"], "read_file");
879        assert_eq!(tools[0]["description"], "Read a file from disk");
880        assert_eq!(tools[0]["parameters"]["properties"]["path"]["type"], "string");
881        assert_eq!(tools[0]["strict"], false);
882    }
883
884    #[tokio::test]
885    async fn responses_preserve_tool_calls_without_an_assistant_text_message() {
886        let context = Context::new(
887            vec![ChatMessage::Assistant {
888                message_id: MessageId::new(),
889                content: String::new(),
890                timestamp: IsoString::now(),
891                reasoning: AssistantReasoning::default(),
892                tool_calls: vec![ToolCallRequest {
893                    id: "call_1".into(),
894                    name: "search".into(),
895                    arguments: "{}".into(),
896                }],
897            }],
898            vec![],
899        );
900
901        let body = openai_request_body("gpt-4.1", &context).await;
902
903        assert_eq!(
904            body["input"],
905            json!([{
906                "type": "function_call", "arguments": "{}", "call_id": "call_1", "name": "search"
907            }])
908        );
909    }
910
911    #[tokio::test]
912    async fn responses_preserve_tool_errors_and_summary_notes() {
913        let context = Context::new(
914            vec![
915                ChatMessage::ToolCallResult(Err(ToolCallError {
916                    id: "call_2".into(),
917                    name: "bash".into(),
918                    arguments: Some("{}".into()),
919                    error: "command failed".into(),
920                })),
921                ChatMessage::Summary {
922                    message_id: MessageId::new(),
923                    content: "User asked about Rust.".into(),
924                    timestamp: IsoString::now(),
925                    messages_compacted: 5,
926                },
927            ],
928            vec![],
929        );
930
931        let body = openai_request_body("gpt-4.1", &context).await;
932        let input = body["input"].as_array().unwrap();
933
934        assert_eq!(input.len(), 2);
935        assert_eq!(
936            input[0],
937            json!({ "type": "function_call_output", "call_id": "call_2", "output": "Error: command failed" })
938        );
939        assert_eq!(input[1]["type"], "message");
940        assert_eq!(input[1]["role"], "user");
941        let note = input[1]["content"].as_str().unwrap();
942        assert!(note.starts_with("[Previous conversation handoff]"));
943        assert!(note.contains("User asked about Rust."));
944    }
945
946    #[tokio::test]
947    async fn responses_replay_encrypted_reasoning_only_for_the_original_model() {
948        for source_model in [Some("openai:gpt-5.6"), Some("openai:gpt-5.5"), None] {
949            let context = Context::new(
950                vec![ChatMessage::Assistant {
951                    message_id: MessageId::new(),
952                    content: "thinking done".into(),
953                    timestamp: IsoString::now(),
954                    tool_calls: vec![],
955                    reasoning: AssistantReasoning::from_parts(
956                        "summary".into(),
957                        source_model.map(|model| EncryptedReasoningContent {
958                            id: "r_1".into(),
959                            model: model.parse().unwrap(),
960                            content: "encrypted-blob".into(),
961                        }),
962                    ),
963                }],
964                vec![],
965            );
966
967            let body = openai_request_body("gpt-5.6", &context).await;
968            let input = body["input"].as_array().unwrap();
969
970            assert_eq!(input[0], json!({ "type": "message", "role": "assistant", "content": "thinking done" }));
971            if source_model == Some("openai:gpt-5.6") {
972                assert_eq!(input.len(), 2);
973                assert_eq!(
974                    input[1],
975                    json!({ "type": "reasoning", "id": "r_1", "summary": [], "encrypted_content": "encrypted-blob" })
976                );
977            } else {
978                assert_eq!(input.len(), 1, "{source_model:?}: {input:?}");
979            }
980        }
981    }
982
983    #[tokio::test]
984    async fn responses_send_each_enabled_effort_with_an_automatic_summary() {
985        for effort in ReasoningEffort::all().iter().filter(|effort| effort.is_enabled()) {
986            let model = LlmModel::all()
987                .iter()
988                .find(|model| {
989                    model.provider_enum() == Provider::Openai && model.effective_reasoning_levels().contains(effort)
990                })
991                .expect("OpenAI has a model supporting this effort");
992            let mut context = hello_context();
993            context.set_reasoning_effort(*effort);
994
995            let body = openai_request_body(&model.model_id(), &context).await;
996
997            assert_eq!(body["reasoning"], json!({ "effort": effort.as_str(), "summary": "auto" }));
998        }
999    }
1000
1001    #[tokio::test]
1002    async fn openai_distinguishes_default_disabled_and_low_effort() {
1003        for (effort, expected) in [
1004            (ReasoningEffort::Default, None),
1005            (ReasoningEffort::Disabled, Some("none")),
1006            (ReasoningEffort::Low, Some("low")),
1007        ] {
1008            let mut fixture = ProviderFixture::builder(&OPENAI).model("gpt-5.4").build().await;
1009            let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
1010            context.set_reasoning_effort(effort);
1011
1012            let (captured, responses) = fixture.run(&context).await;
1013            let body = captured.body;
1014
1015            assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1016            assert_eq!(body["reasoning"]["effort"].as_str(), expected);
1017            if effort == ReasoningEffort::Disabled {
1018                assert!(body["reasoning"]["summary"].is_null());
1019            }
1020        }
1021    }
1022
1023    #[tokio::test]
1024    async fn openai_sends_max_effort_and_prompt_cache_key() {
1025        let mut fixture = ProviderFixture::builder(&OPENAI).model("gpt-5.6").build().await;
1026        let mut context = Context::new(vec![ChatMessage::user("Think harder")], vec![]);
1027        context.set_reasoning_effort(ReasoningEffort::Max);
1028        context.set_prompt_cache_key(Some("cache-key".to_string()));
1029
1030        let (captured, responses) = fixture.run(&context).await;
1031
1032        assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1033        assert_eq!(captured.body["reasoning"]["effort"], "max");
1034        assert_eq!(captured.body["model"], "gpt-5.6");
1035        assert_eq!(captured.body["prompt_cache_key"], "cache-key");
1036        assert_eq!(captured.body["include"], json!(["reasoning.encrypted_content"]));
1037        assert_eq!(captured.body["stream"], true);
1038        assert_eq!(fixture.provider.display_name(), "OpenAI (gpt-5.6)");
1039    }
1040
1041    #[tokio::test]
1042    async fn responses_http_200_failed_server_error_is_retryable_with_request_id() {
1043        let spec = ResponseSpec::sse(include_str!("../../tests/fixtures/openai_responses/04_failed_server.sse"))
1044            .with_header("x-request-id", "req-openai-1");
1045        let mut fixture = ProviderFixture::builder(&OPENAI).response(spec).build().await;
1046        let context = Context::new(vec![ChatMessage::user("hi")], vec![]);
1047
1048        let (_, responses) = fixture.run(&context).await;
1049
1050        assert!(!responses.iter().any(|r| matches!(r, Ok(LlmResponse::Done { .. }))));
1051        let err = responses.iter().find_map(|r| r.as_ref().err()).expect("expected a failure");
1052        assert!(err.is_retryable(), "server_error must be retryable: {err:?}");
1053        let provider_error = err.provider().expect("expected provider error");
1054        assert_eq!(provider_error.kind, ProviderErrorKind::Server);
1055        assert_eq!(provider_error.http_status, Some(200));
1056        assert_eq!(provider_error.request_id.as_deref(), Some("req-openai-1"));
1057        assert_eq!(provider_error.code.as_deref(), Some("server_error"));
1058    }
1059
1060    #[tokio::test]
1061    async fn responses_surface_a_mapping_failure_as_the_only_item() {
1062        let connection = ProviderConnectionConfig { auth_mode: ProviderAuthMode::None, ..Default::default() };
1063        let provider = GenericProvider::from_env_with_connection(&OPENAI, connection).unwrap();
1064        let context = Context::new(
1065            vec![ChatMessage::User {
1066                message_id: MessageId::new(),
1067                content: vec![ContentBlock::Audio { data: "YXVkaW8=".to_string(), mime_type: "audio/wav".to_string() }],
1068                timestamp: IsoString::now(),
1069            }],
1070            vec![],
1071        );
1072
1073        let responses = provider.stream_response(&context).collect::<Vec<_>>().await;
1074
1075        assert_eq!(responses.len(), 1);
1076        assert!(matches!(responses[0], Err(LlmError::UnsupportedContent(_))), "{responses:?}");
1077    }
1078
1079    #[tokio::test]
1080    async fn xiaomi_omits_encrypted_reasoning_summaries_and_prompt_cache_key() {
1081        let mut fixture = ProviderFixture::builder(&XIAOMI).build().await;
1082        let mut context = Context::new(vec![ChatMessage::user("Hello")], vec![]);
1083        context.set_reasoning_effort(ReasoningEffort::High);
1084        context.set_prompt_cache_key(Some("cache-key".to_string()));
1085
1086        let (captured, responses) = fixture.run(&context).await;
1087
1088        assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1089        assert_eq!(captured.path, "/responses");
1090        assert_eq!(captured.body["model"], "mimo-v2.6-pro");
1091        assert_eq!(captured.body["reasoning"], json!({ "effort": "high" }));
1092        assert!(captured.body.get("include").is_none());
1093        assert!(captured.body.get("prompt_cache_key").is_none());
1094        assert_eq!(fixture.provider.display_name(), "Xiaomi (mimo-v2.6-pro)");
1095    }
1096
1097    #[tokio::test]
1098    async fn xiaomi_replays_prior_reasoning_as_plain_text() {
1099        let mut fixture = ProviderFixture::builder(&XIAOMI).build().await;
1100        let context = Context::new(
1101            vec![
1102                ChatMessage::user("Hello"),
1103                ChatMessage::Assistant {
1104                    message_id: MessageId::new(),
1105                    content: "Hi".to_string(),
1106                    reasoning: AssistantReasoning::from_parts("greeting the user".to_string(), None),
1107                    timestamp: IsoString::now(),
1108                    tool_calls: vec![],
1109                },
1110                ChatMessage::user("Again"),
1111            ],
1112            vec![],
1113        );
1114
1115        let (captured, responses) = fixture.run(&context).await;
1116
1117        assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1118        let reasoning = captured.body["input"]
1119            .as_array()
1120            .unwrap()
1121            .iter()
1122            .find(|item| item["type"] == "reasoning")
1123            .expect("reasoning item should be replayed");
1124        assert_eq!(reasoning["content"], json!([{ "type": "reasoning_text", "text": "greeting the user" }]));
1125        assert!(reasoning.get("encrypted_content").is_none_or(serde_json::Value::is_null));
1126    }
1127
1128    #[tokio::test]
1129    async fn xiaomi_drops_null_from_optional_tool_parameters() {
1130        let mut fixture = ProviderFixture::builder(&XIAOMI).build().await;
1131        let tool = ToolDefinition::new(
1132            "bash",
1133            "Run a command",
1134            json!({
1135                "type": "object",
1136                "properties": {
1137                    "command": { "type": "string" },
1138                    "description": { "type": ["string", "null"] }
1139                },
1140                "required": ["command"]
1141            }),
1142        );
1143        let context = Context::new(vec![ChatMessage::user("Hello")], vec![tool]);
1144
1145        let (captured, responses) = fixture.run(&context).await;
1146
1147        assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1148        assert_eq!(captured.body["tools"][0]["parameters"]["properties"]["description"], json!({ "type": "string" }));
1149    }
1150
1151    #[tokio::test]
1152    async fn anthropic_sends_its_version_and_api_key_unless_auth_is_none() {
1153        for (auth_mode, api_key) in [(ProviderAuthMode::Default, Some("secret")), (ProviderAuthMode::None, None)] {
1154            let mut fixture = ProviderFixture::builder(&ANTHROPIC)
1155                .api_key("secret")
1156                .connection(ProviderConnectionConfig { auth_mode, ..Default::default() })
1157                .build()
1158                .await;
1159            let (captured, responses) = fixture.run(&hello_context()).await;
1160
1161            assert_successful_stream(&responses);
1162            assert_eq!(captured.path, "/v1/messages");
1163            assert_eq!(captured.headers["anthropic-version"], "2023-06-01");
1164            assert_eq!(captured.headers.get("x-api-key").map(|key| key.to_str().unwrap()), api_key);
1165            assert!(captured.headers.get("authorization").is_none());
1166        }
1167    }
1168
1169    #[tokio::test]
1170    async fn anthropic_caches_the_system_prompt_and_sends_tools() {
1171        let mut fixture = ProviderFixture::builder(&ANTHROPIC).build().await;
1172        let context = Context::new(
1173            vec![ChatMessage::system("You are helpful"), ChatMessage::user("Hello")],
1174            vec![ToolDefinition::new("search", "Search", json!({ "type": "object" }))],
1175        );
1176
1177        let (captured, responses) = fixture.run(&context).await;
1178        let body = captured.body;
1179
1180        assert_successful_stream(&responses);
1181        assert_eq!(body["model"], "claude-sonnet-4-5-20250929");
1182        assert_eq!(body["max_tokens"], 16_384);
1183        assert_eq!(body["stream"], true);
1184        assert_eq!(body["cache_control"], json!({ "type": "ephemeral" }));
1185        assert_eq!(
1186            body["system"],
1187            json!([{ "type": "text", "text": "You are helpful", "cache_control": { "type": "ephemeral" } }])
1188        );
1189        assert_eq!(body["messages"].as_array().map(Vec::len), Some(1));
1190        assert_eq!(
1191            body["tools"],
1192            json!([{ "name": "search", "description": "Search", "input_schema": { "type": "object" } }])
1193        );
1194        assert_eq!(fixture.provider.display_name(), "Anthropic (claude-sonnet-4-5-20250929)");
1195    }
1196
1197    #[tokio::test]
1198    async fn anthropic_default_and_disabled_thinking_keep_sampling_and_token_limit() {
1199        for (effort, thinking) in
1200            [(ReasoningEffort::Default, None), (ReasoningEffort::Disabled, Some(json!({ "type": "disabled" })))]
1201        {
1202            let body = anthropic_request_body(effort).await;
1203
1204            assert_eq!(body.get("thinking"), thinking.as_ref());
1205            assert_eq!(body["max_tokens"], 500);
1206            assert_eq!(body["temperature"], 0.5);
1207            assert_eq!(body["top_p"], 0.5);
1208        }
1209    }
1210
1211    #[tokio::test]
1212    async fn anthropic_enabled_thinking_clears_sampling_and_raises_token_limit() {
1213        for (effort, budget_tokens) in
1214            [(ReasoningEffort::Low, 1024), (ReasoningEffort::High, 10240), (ReasoningEffort::Max, 32768)]
1215        {
1216            let body = anthropic_request_body(effort).await;
1217
1218            assert_eq!(body["thinking"], json!({ "type": "enabled", "budget_tokens": budget_tokens }));
1219            assert_eq!(body["max_tokens"], budget_tokens + 1024);
1220            assert!(body.get("temperature").is_none());
1221            assert!(body.get("top_p").is_none());
1222        }
1223    }
1224
1225    const CHAT_FIXTURE: &str = include_str!("../../tests/fixtures/openai/01_minimal.sse");
1226
1227    async fn streamed_chat(body: &str) -> Vec<Result<LlmResponse>> {
1228        let mut fixture = ProviderFixture::builder(&OPENROUTER).response(ResponseSpec::sse(body)).build().await;
1229
1230        let (_, responses) = fixture.run(&hello_context()).await;
1231        responses
1232    }
1233
1234    fn assert_successful_stream(responses: &[Result<LlmResponse>]) {
1235        assert!(responses.iter().all(Result::is_ok), "{responses:?}");
1236        assert!(responses.iter().any(|response| matches!(response, Ok(LlmResponse::Done { .. }))));
1237    }
1238
1239    fn assert_interrupted(responses: &[Result<LlmResponse>]) {
1240        let error = responses.last().and_then(|response| response.as_ref().err()).expect("the stream must fail");
1241        assert_eq!(error.provider().map(|provider| provider.kind), Some(ProviderErrorKind::StreamInterrupted));
1242        assert!(error.is_retryable());
1243    }
1244
1245    async fn openai_request_body(model: &str, context: &Context) -> serde_json::Value {
1246        let mut fixture = ProviderFixture::builder(&OPENAI).model(model).build().await;
1247
1248        let (captured, responses) = fixture.run(context).await;
1249
1250        assert_successful_stream(&responses);
1251        captured.body
1252    }
1253
1254    async fn anthropic_request_body(effort: ReasoningEffort) -> serde_json::Value {
1255        let mut fixture = ProviderFixture::builder(&ANTHROPIC)
1256            .model(&reasoning_off_model(Provider::Anthropic).model_id())
1257            .build()
1258            .await;
1259        let mut context = hello_context();
1260        context.set_reasoning_effort(effort);
1261        context.set_model_settings(ModelSettings { temperature: Some(0.5), top_p: Some(0.5), max_tokens: Some(500) });
1262
1263        let (captured, responses) = fixture.run(&context).await;
1264
1265        assert_successful_stream(&responses);
1266        captured.body
1267    }
1268
1269    fn reasoning_off_model(provider: Provider) -> &'static LlmModel {
1270        LlmModel::all()
1271            .iter()
1272            .find(|model| model.provider_enum() == provider && model.supports_reasoning_off())
1273            .expect("the catalog has a model that can turn reasoning off")
1274    }
1275
1276    async fn rejected_chat_request(spec: ResponseSpec) -> LlmError {
1277        let mut fixture = ProviderFixture::builder(&OPENROUTER)
1278            .response(spec.with_header("x-request-id", "request-123"))
1279            .build()
1280            .await;
1281
1282        let (_, mut events) = fixture.run(&hello_context()).await;
1283
1284        assert_eq!(events.len(), 1, "a rejected request must yield exactly one error: {events:?}");
1285        events.pop().unwrap().expect_err("request must fail")
1286    }
1287}
llm/providers/generic.rs

llm/providers/
generic.rs
