Skip to main content

reflex/semantic/providers/
mod.rs

1//! LLM provider implementations
2
3pub mod anthropic;
4#[cfg(test)]
5pub mod mock;
6pub mod openai;
7pub mod openai_compatible;
8pub mod openrouter;
9pub mod wire;
10
11use anyhow::{Context, Result};
12use async_trait::async_trait;
13use std::collections::HashMap;
14use std::time::Duration;
15
16/// What shape the model must answer in.
17#[derive(Debug, Clone, Copy)]
18pub enum OutputMode<'a> {
19    /// Free text.
20    Text,
21    /// Any JSON object (OpenAI `json_object`, Anthropic prompt-only).
22    JsonObject,
23    /// JSON that matches `schema` (OpenAI strict `json_schema`, Anthropic forced tool use).
24    JsonSchema {
25        name: &'a str,
26        schema: &'a serde_json::Value,
27    },
28}
29
30impl OutputMode<'_> {
31    /// Stable label for cache keys and logs.
32    pub fn label(&self) -> &'static str {
33        match self {
34            OutputMode::Text => "text",
35            OutputMode::JsonObject => "json_object",
36            OutputMode::JsonSchema { .. } => "json_schema",
37        }
38    }
39
40    pub fn is_json(&self) -> bool {
41        !matches!(self, OutputMode::Text)
42    }
43}
44
45/// A completion request with a separate system prompt and an explicit output contract.
46///
47/// Used by Pulse. `rfx ask` still uses [`LlmProvider::complete`].
48#[derive(Debug, Clone, Copy)]
49pub struct CompletionRequest<'a> {
50    pub system: &'a str,
51    pub user: &'a str,
52    pub output: OutputMode<'a>,
53    pub max_tokens: u32,
54    pub temperature: f32,
55    /// Task id for logs and test routing. Never sent to the provider.
56    pub tag: &'a str,
57}
58
59/// Token usage as the provider reports it.
60#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
61pub struct Usage {
62    pub input_tokens: u32,
63    pub output_tokens: u32,
64    /// Input tokens served from the provider's prompt cache, when reported.
65    #[serde(default)]
66    pub cached_input_tokens: u32,
67}
68
69/// Why the model stopped.
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub enum StopReason {
72    End,
73    MaxTokens,
74    Other(String),
75}
76
77#[derive(Debug, Clone)]
78pub struct CompletionResponse {
79    pub text: String,
80    pub usage: Option<Usage>,
81    pub stop: StopReason,
82    pub model: String,
83}
84
85/// Classification of a provider failure. Fatal kinds will not succeed on retry.
86#[derive(Debug, Clone, Copy, PartialEq, Eq)]
87pub enum ProviderErrorKind {
88    Auth,
89    NotFound,
90    BadRequest,
91    RateLimited,
92    Server,
93    Timeout,
94    Network,
95    Malformed,
96}
97
98impl ProviderErrorKind {
99    /// Auth, NotFound and BadRequest fail the same way every time.
100    pub fn is_fatal(&self) -> bool {
101        matches!(self, Self::Auth | Self::NotFound | Self::BadRequest)
102    }
103
104    /// Transient failures worth a retry with backoff.
105    pub fn is_retryable(&self) -> bool {
106        matches!(
107            self,
108            Self::RateLimited | Self::Server | Self::Timeout | Self::Network
109        )
110    }
111}
112
113/// A classified provider error. Returned inside `anyhow::Error`; recover it with
114/// `err.downcast_ref::<ProviderError>()`.
115#[derive(Debug, Clone, thiserror::Error)]
116#[error("{provider} API error ({kind:?}{}): {body}", status.map(|s| format!(", HTTP {s}")).unwrap_or_default())]
117pub struct ProviderError {
118    pub provider: String,
119    pub kind: ProviderErrorKind,
120    pub status: Option<u16>,
121    pub retry_after: Option<Duration>,
122    pub body: String,
123}
124
125/// What a provider supports natively.
126#[derive(Debug, Clone, Copy, Default)]
127pub struct ProviderCaps {
128    pub system_role: bool,
129    pub json_object: bool,
130    pub json_schema: bool,
131    pub reports_usage: bool,
132}
133
134/// Trait for LLM providers that generate structured query responses
135// `#[async_trait]` puts `#[must_use]` on the boxed future of a default method, and
136// clippy 1.99 reports that as `double_must_use`. The code is fine.
137#[allow(clippy::double_must_use)]
138#[async_trait]
139pub trait LlmProvider: Send + Sync {
140    /// Send a prompt and get response
141    ///
142    /// # Arguments
143    ///
144    /// * `prompt` - The prompt to send to the LLM
145    /// * `json_mode` - Whether to request JSON structured output (true) or plain text (false)
146    ///
147    /// When `json_mode` is true, the response should be valid JSON matching the QueryResponse schema.
148    /// When `json_mode` is false, the response can be plain text (used for answer generation).
149    async fn complete(&self, prompt: &str, json_mode: bool) -> Result<String>;
150
151    /// Get provider name (for logging and error messages)
152    fn name(&self) -> &str;
153
154    /// Get default model identifier
155    fn default_model(&self) -> &str;
156
157    /// The model this instance actually calls (configured, else default).
158    fn model(&self) -> &str {
159        self.default_model()
160    }
161
162    /// Native capabilities. The default claims none.
163    fn caps(&self) -> ProviderCaps {
164        ProviderCaps::default()
165    }
166
167    /// Send a structured request.
168    ///
169    /// The default joins system and user into one prompt and calls [`Self::complete`],
170    /// so providers without native support still work (no usage, no schema enforcement).
171    async fn complete_request(&self, req: &CompletionRequest<'_>) -> Result<CompletionResponse> {
172        let prompt = if req.system.is_empty() {
173            req.user.to_string()
174        } else {
175            format!("{}\n\n{}", req.system, req.user)
176        };
177        let text = self.complete(&prompt, req.output.is_json()).await?;
178        Ok(CompletionResponse {
179            text,
180            usage: None,
181            stop: StopReason::End,
182            model: self.model().to_string(),
183        })
184    }
185}
186
187/// Default model name for a provider, for display when no model is configured.
188pub fn default_model_for(provider_name: &str) -> &'static str {
189    match provider_name.to_lowercase().as_str() {
190        "openai" => "gpt-4o-mini",
191        "anthropic" => "claude-3-5-haiku-20241022",
192        "openrouter" => "anthropic/claude-sonnet-4",
193        "openai-compatible" | "openai_compatible" => "",
194        _ => "",
195    }
196}
197
198/// Create a provider instance from name, API key, and request timeout.
199pub fn create_provider(
200    provider_name: &str,
201    api_key: String,
202    model: Option<String>,
203    options: Option<HashMap<String, String>>,
204    timeout_secs: u64,
205) -> Result<Box<dyn LlmProvider>> {
206    match provider_name.to_lowercase().as_str() {
207        "openai" => Ok(Box::new(openai::OpenAiProvider::new(
208            api_key,
209            model,
210            timeout_secs,
211        )?)),
212        "anthropic" => Ok(Box::new(anthropic::AnthropicProvider::new(
213            api_key,
214            model,
215            timeout_secs,
216        )?)),
217        "openrouter" => {
218            let sort = options.as_ref().and_then(|o| o.get("sort").cloned());
219            Ok(Box::new(openrouter::OpenRouterProvider::new(
220                api_key,
221                model,
222                sort,
223                timeout_secs,
224            )?))
225        }
226        "openai-compatible" | "openai_compatible" => {
227            let base_url = options
228                .as_ref()
229                .and_then(|o| o.get("base_url").cloned())
230                .context(
231                    "openai-compatible provider requires 'base_url' in options \
232                     (set credentials.openai_compatible_base_url in ~/.reflex/config.toml \
233                     or the OPENAI_COMPATIBLE_BASE_URL env var)",
234                )?;
235            let model = model.unwrap_or_default();
236            let key = if api_key.is_empty() {
237                None
238            } else {
239                Some(api_key)
240            };
241            Ok(Box::new(openai_compatible::OpenAiCompatibleProvider::new(
242                key,
243                model,
244                base_url,
245                timeout_secs,
246            )?))
247        }
248        _ => anyhow::bail!(
249            "Unknown provider: {}. Supported: openai, anthropic, openrouter, openai-compatible",
250            provider_name
251        ),
252    }
253}
254
255#[cfg(test)]
256mod tests {
257    use super::*;
258
259    #[test]
260    fn test_create_provider_openai() {
261        let provider = create_provider("openai", "test-key".to_string(), None, None, 300);
262        assert!(provider.is_ok());
263        assert_eq!(provider.unwrap().name(), "openai");
264    }
265
266    #[test]
267    fn test_default_model_for_known_providers() {
268        assert_eq!(default_model_for("openai"), "gpt-4o-mini");
269        assert_eq!(default_model_for("OpenAI"), "gpt-4o-mini");
270        assert_eq!(default_model_for("anthropic"), "claude-3-5-haiku-20241022");
271        assert_eq!(default_model_for("openrouter"), "anthropic/claude-sonnet-4");
272        assert_eq!(default_model_for("openai-compatible"), "");
273        assert_eq!(default_model_for("openai_compatible"), "");
274        assert_eq!(default_model_for("unknown"), "");
275    }
276
277    #[test]
278    fn test_create_provider_case_insensitive() {
279        let provider = create_provider("OpenAI", "test-key".to_string(), None, None, 300);
280        assert!(provider.is_ok());
281    }
282
283    #[test]
284    fn test_create_provider_unknown() {
285        let provider = create_provider("unknown", "test-key".to_string(), None, None, 300);
286        assert!(provider.is_err());
287        if let Err(e) = provider {
288            assert!(e.to_string().contains("Unknown provider"));
289        }
290    }
291
292    #[test]
293    fn test_create_provider_openrouter() {
294        let provider = create_provider("openrouter", "test-key".to_string(), None, None, 300);
295        assert!(provider.is_ok());
296        assert_eq!(provider.unwrap().name(), "openrouter");
297    }
298
299    #[test]
300    fn test_create_provider_openrouter_with_sort() {
301        let mut opts = HashMap::new();
302        opts.insert("sort".to_string(), "speed".to_string());
303        let provider = create_provider(
304            "openrouter",
305            "test-key".to_string(),
306            Some("openai/gpt-4o-mini".to_string()),
307            Some(opts),
308            300,
309        );
310        assert!(provider.is_ok());
311    }
312
313    #[test]
314    fn test_create_provider_openai_compatible_with_base_url() {
315        let mut opts = HashMap::new();
316        opts.insert(
317            "base_url".to_string(),
318            "http://localhost:1234/v1".to_string(),
319        );
320        let provider = create_provider(
321            "openai-compatible",
322            "test-key".to_string(),
323            Some("qwen2.5-coder".to_string()),
324            Some(opts),
325            300,
326        );
327        assert!(provider.is_ok());
328        assert_eq!(provider.unwrap().name(), "openai-compatible");
329    }
330
331    #[test]
332    fn test_create_provider_openai_compatible_accepts_underscore_alias() {
333        let mut opts = HashMap::new();
334        opts.insert(
335            "base_url".to_string(),
336            "http://localhost:1234/v1".to_string(),
337        );
338        let provider = create_provider(
339            "openai_compatible",
340            "test-key".to_string(),
341            Some("qwen2.5-coder".to_string()),
342            Some(opts),
343            300,
344        );
345        assert!(provider.is_ok());
346    }
347
348    #[test]
349    fn test_create_provider_openai_compatible_allows_empty_api_key() {
350        let mut opts = HashMap::new();
351        opts.insert(
352            "base_url".to_string(),
353            "http://localhost:1234/v1".to_string(),
354        );
355        let provider = create_provider(
356            "openai-compatible",
357            String::new(),
358            Some("qwen2.5-coder".to_string()),
359            Some(opts),
360            300,
361        );
362        assert!(provider.is_ok());
363    }
364
365    #[test]
366    fn test_create_provider_openai_compatible_requires_base_url() {
367        let provider = create_provider(
368            "openai-compatible",
369            String::new(),
370            Some("qwen2.5-coder".to_string()),
371            None,
372            300,
373        );
374        assert!(provider.is_err());
375        if let Err(e) = provider {
376            assert!(e.to_string().contains("base_url"));
377        }
378    }
379
380    #[test]
381    fn test_create_provider_openai_compatible_requires_model() {
382        let mut opts = HashMap::new();
383        opts.insert(
384            "base_url".to_string(),
385            "http://localhost:1234/v1".to_string(),
386        );
387        let provider = create_provider("openai-compatible", String::new(), None, Some(opts), 300);
388        assert!(provider.is_err());
389    }
390}