llm_universal/
lib.rs

1//! One async client for many LLM APIs: Anthropic, OpenAI, Google Gemini, Cohere, Azure OpenAI,
2//! Amazon Bedrock, Vertex AI and every OpenAI-compatible server.
3//!
4//! The same code (`ChatRequest` -> `ChatResponse`) runs against any provider; switching
5//! providers means switching the constructor:
6//!
7//! ```no_run
8//! # use llm_universal::*;
9//! # async fn demo() -> Result<()> {
10//! let provider: Box<dyn LlmProvider> = Box::new(AnthropicProvider::new("sk-ant-..."));
11//! // or: Box::new(OpenAiProvider::new("sk-...")),
12//! // or: Box::new(GeminiProvider::new("AIza...")),
13//! // or: Box::new(OpenAiProvider::ollama()),
14//! let req = ChatRequest::new("claude-opus-5-5")
15//!     .system("Be concise.")
16//!     .max_tokens(512)
17//!     .message(Message::user("Hello!"));
18//! let resp = provider.chat(&req).await?;
19//! println!("{}", resp.text());
20//!
21//! // Streaming:
22//! let mut stream = provider.chat_stream(&req).await?;
23//! while let Some(event) = stream.next().await {
24//!     if let StreamEvent::TextDelta(text) = event? {
25//!         print!("{text}");
26//!     }
27//! }
28//! # Ok(()) }
29//! ```
30//!
31//! Errors follow the official SDKs: see [`LlmError`] and [`ApiError`]. Failed requests are
32//! retried like the SDKs do (2 retries by default; see `with_max_retries`).
33
34#![forbid(unsafe_code)]
35#![warn(missing_debug_implementations)]
36
37mod anthropic;
38mod cohere;
39mod error;
40mod gemini;
41mod http;
42mod openai;
43mod runner;
44mod stream;
45mod types;
46
47pub use anthropic::AnthropicProvider;
48pub use cohere::CohereProvider;
49pub use error::{ApiError, ApiErrorKind, LlmError};
50pub use futures_util::StreamExt;
51pub use gemini::GeminiProvider;
52pub use http::DEFAULT_MAX_RETRIES;
53pub use openai::OpenAiProvider;
54pub use runner::run_tools;
55pub use stream::{ChatStream, StreamEvent};
56pub use types::*;
57
58use async_trait::async_trait;
59
60pub type Result<T> = std::result::Result<T, LlmError>;
61
62/// Common contract implemented by every provider.
63#[async_trait]
64pub trait LlmProvider: Send + Sync {
65    /// Sends the request and waits for the whole answer.
66    async fn chat(&self, request: &ChatRequest) -> Result<ChatResponse>;
67
68    /// Streams the answer as it is generated. The last event is [`StreamEvent::Done`]
69    /// with the full response.
70    ///
71    /// The default implementation (for custom providers) calls [`chat`](Self::chat) and
72    /// yields the whole text at once.
73    async fn chat_stream(&self, request: &ChatRequest) -> Result<ChatStream> {
74        let response = self.chat(request).await?;
75        let events = vec![
76            Ok(StreamEvent::TextDelta(response.text())),
77            Ok(StreamEvent::Done(response)),
78        ];
79        Ok(Box::pin(futures_util::stream::iter(events)))
80    }
81
82    /// Models available to the key.
83    async fn list_models(&self) -> Result<Vec<ModelInfo>> {
84        Err(LlmError::InvalidArgument(
85            "list_models is not supported by this provider".into(),
86        ))
87    }
88}
89
90/// Providers that can turn text into vectors (OpenAI and compatible servers, Gemini, Cohere).
91#[async_trait]
92pub trait EmbeddingProvider: Send + Sync {
93    async fn embed(&self, model: &str, inputs: &[&str]) -> Result<Embeddings>;
94}
95
96fn invalid(message: String) -> LlmError {
97    LlmError::InvalidArgument(message)
98}
99
100/// Host part of `base_url`, or an error if it carries credentials or a query string.
101fn checked_host<'a>(provider: &str, base_url: &'a str) -> Result<&'a str> {
102    let rest = base_url
103        .strip_prefix("https://")
104        .or_else(|| base_url.strip_prefix("http://"))
105        .ok_or_else(|| {
106            invalid(format!(
107                "{provider}: base_url must start with http:// or https://"
108            ))
109        })?;
110    let authority = rest.split('/').next().unwrap_or_default();
111    if authority.contains('@') || base_url.contains('?') || base_url.contains('#') {
112        // Credentials in the URL would leak into logs and error chains.
113        return Err(invalid(format!(
114            "{provider}: base_url must not contain credentials, a query string or a fragment"
115        )));
116    }
117    if authority.is_empty() {
118        return Err(invalid(format!("{provider}: base_url has no host")));
119    }
120    Ok(authority)
121}
122
123fn is_loopback(authority: &str) -> bool {
124    let host = if let Some(rest) = authority.strip_prefix('[') {
125        rest.split(']').next().unwrap_or_default()
126    } else {
127        authority.split(':').next().unwrap_or_default()
128    };
129    host == "localhost" || host.ends_with(".localhost") || host == "::1" || host.starts_with("127.")
130}
131
132/// Checks `base_url` and the key's transport before any request.
133pub(crate) fn validate_endpoint(
134    provider: &'static str,
135    api_key: &str,
136    require_key: bool,
137    base_url: &str,
138) -> Result<()> {
139    let authority = checked_host(provider, base_url)?;
140    if base_url.starts_with("http://") && !api_key.is_empty() && !is_loopback(authority) {
141        return Err(invalid(format!(
142            "{provider}: refusing to send an API key over plain http:// to a non-local host; use https://"
143        )));
144    }
145    if require_key && api_key.trim().is_empty() {
146        return Err(invalid(format!("{provider}: api_key must not be empty")));
147    }
148    Ok(())
149}
150
151/// A value that goes into a URL path (model, project, region) may only use these characters.
152pub(crate) fn checked_path_segment<'a>(
153    provider: &str,
154    what: &str,
155    value: &'a str,
156    extra: &[char],
157) -> Result<&'a str> {
158    // A value made only of dots (`.`, `..`) would change the URL path.
159    let ok = !value.is_empty()
160        && !value.chars().all(|c| c == '.')
161        && value.chars().all(|c| {
162            c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.') || extra.contains(&c)
163        });
164    if ok {
165        Ok(value)
166    } else {
167        Err(invalid(format!(
168            "{provider}: {what} `{value}` may only contain a-z, A-Z, 0-9, '-', '_' and '.'"
169        )))
170    }
171}
172
173const IMAGE_TYPES: [&str; 4] = ["image/png", "image/jpeg", "image/gif", "image/webp"];
174
175/// Validates the request and provider settings before any network call.
176pub(crate) fn validate(
177    provider: &'static str,
178    api_key: &str,
179    require_key: bool,
180    base_url: &str,
181    max_tool_name: usize,
182    req: &ChatRequest,
183) -> Result<()> {
184    validate_endpoint(provider, api_key, require_key, base_url)?;
185    if req.model.trim().is_empty() {
186        return Err(invalid("model must not be empty".to_string()));
187    }
188    if req.model.chars().any(char::is_control) {
189        return Err(invalid(
190            "model must not contain control characters".to_string(),
191        ));
192    }
193    if req.messages.is_empty() {
194        return Err(invalid("messages must not be empty".to_string()));
195    }
196    for (i, message) in req.messages.iter().enumerate() {
197        let has_content = message.parts.iter().any(|p| match p {
198            Part::Text { text } => !text.is_empty(),
199            Part::Reasoning { .. } => false,
200            _ => true,
201        });
202        if !has_content {
203            return Err(invalid(format!("messages[{i}] has no content")));
204        }
205        if message.has_images() && message.role != Role::User {
206            return Err(invalid(format!(
207                "messages[{i}]: images are only allowed in user messages"
208            )));
209        }
210        for part in &message.parts {
211            if let Part::Image { source } = part {
212                validate_image(i, source)?;
213            }
214        }
215    }
216    if let Some(t) = req.temperature {
217        if !(0.0..=2.0).contains(&t) {
218            return Err(invalid(format!(
219                "temperature must be between 0.0 and 2.0, got {t}"
220            )));
221        }
222    }
223    if let Some(format) = &req.response_format {
224        let valid_name = !format.name.is_empty()
225            && format.name.len() <= 64
226            && format
227                .name
228                .chars()
229                .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-');
230        if !valid_name {
231            return Err(invalid(format!(
232                "response_format name `{}` must match ^[a-zA-Z0-9_-]{{1,64}}$",
233                format.name
234            )));
235        }
236        if !format.schema.is_object() {
237            return Err(invalid(
238                "response_format schema must be a JSON object".to_string(),
239            ));
240        }
241    }
242    let mut names = std::collections::HashSet::new();
243    for tool in &req.tools {
244        let valid_name = !tool.name.is_empty()
245            && tool.name.len() <= max_tool_name
246            && tool
247                .name
248                .chars()
249                .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-');
250        if !valid_name {
251            return Err(invalid(format!(
252                "tool name `{}` must match ^[a-zA-Z0-9_-]{{1,{max_tool_name}}}$",
253                tool.name
254            )));
255        }
256        if !names.insert(tool.name.as_str()) {
257            return Err(invalid(format!("duplicate tool name `{}`", tool.name)));
258        }
259        if tool.parameters["type"] != "object" {
260            return Err(invalid(format!(
261                "tool `{}`: parameters must be a JSON Schema with \"type\": \"object\"",
262                tool.name
263            )));
264        }
265    }
266    Ok(())
267}
268
269fn validate_image(i: usize, source: &ImageSource) -> Result<()> {
270    match source {
271        ImageSource::Url { url } => {
272            if !url.starts_with("https://") && !url.starts_with("http://") {
273                return Err(invalid(format!(
274                    "messages[{i}]: image url must start with http:// or https://"
275                )));
276            }
277        }
278        ImageSource::Base64 { media_type, data } => {
279            if !IMAGE_TYPES.contains(&media_type.as_str()) {
280                return Err(invalid(format!(
281                    "messages[{i}]: image media_type `{media_type}` must be one of {}",
282                    IMAGE_TYPES.join(", ")
283                )));
284            }
285            let base64 = |c: char| c.is_ascii_alphanumeric() || matches!(c, '+' | '/' | '=');
286            if data.is_empty() || !data.chars().all(base64) {
287                return Err(invalid(format!(
288                    "messages[{i}]: image data must be non-empty base64 without a `data:` prefix"
289                )));
290            }
291        }
292    }
293    Ok(())
294}
295
296/// Compiles the README examples as doctests.
297#[cfg(doctest)]
298#[doc = include_str!("../README.md")]
299pub struct ReadmeDoctests;
llm_universal/lib.rs

llm_universal/
lib.rs
