zeph_llm/
provider_dyn.rs

1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! Object-safe adapter for [`LlmProvider`].
5//!
6//! [`LlmProviderDyn`] mirrors every method of [`LlmProvider`] but returns
7//! [`BoxFuture`] instead of `impl Future + Send`. A blanket implementation
8//! over any `T: LlmProvider + Send + Sync + 'static` wires the two traits
9//! together automatically.
10//!
11//! ## Usage
12//!
13//! Use [`LlmProvider`] as the *implementation* surface (concrete types, monomorphic
14//! call sites). Use `Arc<dyn LlmProviderDyn>` as the *storage* type wherever runtime
15//! polymorphism is required (router, cascade, dependency injection).
16//!
17//! Implementors never need to implement [`LlmProviderDyn`] directly — the blanket impl
18//! handles it. Implement [`LlmProvider`] instead.
19//!
20//! ## Generic methods
21//!
22//! `LlmProvider::chat_typed<T: DeserializeOwned>` cannot be part of a dyn-safe trait
23//! because it carries a generic type parameter. Use the free function
24//! [`chat_typed_dyn`] instead when working with `dyn LlmProviderDyn`.
25//!
26//! ## Examples
27//!
28//! ```rust,no_run
29//! use std::sync::Arc;
30//! use zeph_llm::provider::{LlmProvider, Message, Role};
31//! use zeph_llm::provider_dyn::LlmProviderDyn;
32//! use zeph_llm::ollama::OllamaProvider;
33//!
34//! # async fn example() -> Result<(), zeph_llm::LlmError> {
35//! let provider = OllamaProvider::new(
36//!     "http://localhost:11434",
37//!     "llama3.2".into(),
38//!     "nomic-embed-text".into(),
39//! );
40//!
41//! // Erase the concrete type for storage in a router or DI container.
42//! let dyn_provider: Arc<dyn LlmProviderDyn> = Arc::new(provider);
43//!
44//! let messages = vec![Message::from_legacy(Role::User, "Hello!")];
45//! let response = dyn_provider.chat(&messages).await?;
46//! println!("{response}");
47//! # Ok(())
48//! # }
49//! ```
50
51use futures::future::BoxFuture;
52use serde::de::DeserializeOwned;
53
54use crate::error::LlmError;
55use crate::provider::{
56    ChatExtras, ChatResponse, ChatStream, LlmProvider, Message, Role, ToolDefinition,
57    cached_schema, short_type_name,
58};
59
60mod private {
61    pub trait Sealed {}
62    impl<T: super::LlmProvider> Sealed for T {}
63}
64
65/// Object-safe shadow of [`LlmProvider`].
66///
67/// Sealed — only the blanket `impl<T: LlmProvider + Send + Sync + 'static>` exists.
68/// External crates cannot implement this trait directly; implement [`LlmProvider`] instead
69/// and the blanket impl wires everything up automatically.
70///
71/// All async methods return [`BoxFuture`] rather than `impl Future + Send`, making this
72/// trait dyn-compatible and usable behind `Arc<dyn LlmProviderDyn>`.
73pub trait LlmProviderDyn: private::Sealed + std::fmt::Debug + Send + Sync {
74    /// Report the model's context window size in tokens. `None` if unknown.
75    fn context_window(&self) -> Option<usize>;
76
77    /// Send messages to the LLM and return the assistant response.
78    ///
79    /// # Errors
80    ///
81    /// Returns an error if the provider fails to communicate or the response is invalid.
82    fn chat<'a>(&'a self, messages: &'a [Message]) -> BoxFuture<'a, Result<String, LlmError>>;
83
84    /// Send messages and return a stream of response chunks.
85    ///
86    /// # Errors
87    ///
88    /// Returns an error if the provider fails to communicate or the response is invalid.
89    fn chat_stream<'a>(
90        &'a self,
91        messages: &'a [Message],
92    ) -> BoxFuture<'a, Result<ChatStream, LlmError>>;
93
94    /// Whether this provider supports native streaming.
95    fn supports_streaming(&self) -> bool;
96
97    /// Generate an embedding vector from text.
98    ///
99    /// # Errors
100    ///
101    /// Returns an error if the provider does not support embeddings or the request fails.
102    fn embed<'a>(&'a self, text: &'a str) -> BoxFuture<'a, Result<Vec<f32>, LlmError>>;
103
104    /// Embed multiple texts in a single API call.
105    ///
106    /// # Errors
107    ///
108    /// Returns an error if any embedding fails.
109    fn embed_batch<'a>(
110        &'a self,
111        texts: &'a [&'a str],
112    ) -> BoxFuture<'a, Result<Vec<Vec<f32>>, LlmError>>;
113
114    /// Whether this provider supports embedding generation.
115    fn supports_embeddings(&self) -> bool;
116
117    /// Provider name for logging and identification.
118    fn name(&self) -> &str;
119
120    /// Model identifier string (e.g. `gpt-4o-mini`, `claude-sonnet-5`).
121    fn model_identifier(&self) -> &str;
122
123    /// Model identifier that actually served the most recent dispatch. See
124    /// [`LlmProvider::effective_model_identifier`] for the full contract.
125    fn effective_model_identifier(&self) -> &str;
126
127    /// Whether this provider supports image input (vision).
128    fn supports_vision(&self) -> bool;
129
130    /// Whether this provider supports native `tool_use` / function calling.
131    fn supports_tool_use(&self) -> bool;
132
133    /// Send messages with tool definitions, returning a structured response.
134    ///
135    /// # Errors
136    ///
137    /// Returns an error if the provider fails to communicate or the response is invalid.
138    fn chat_with_tools<'a>(
139        &'a self,
140        messages: &'a [Message],
141        tools: &'a [ToolDefinition],
142    ) -> BoxFuture<'a, Result<ChatResponse, LlmError>>;
143
144    /// Return the cache usage from the last API call, if available.
145    /// Returns `(cache_creation_tokens, cache_read_tokens)`.
146    fn last_cache_usage(&self) -> Option<(u64, u64)>;
147
148    /// Return token counts from the last API call, if available.
149    /// Returns `(input_tokens, output_tokens)`.
150    fn last_usage(&self) -> Option<(u64, u64)>;
151
152    /// Return reasoning tokens from the last API call, if the provider reports them.
153    ///
154    /// Reasoning tokens are a **subset** of completion tokens (`OpenAI` o-series only).
155    /// Returns `None` for providers that do not expose reasoning token counts.
156    fn last_reasoning_tokens(&self) -> Option<u64> {
157        None
158    }
159
160    /// Return the compaction summary from the most recent API call, if available.
161    fn take_compaction_summary(&self) -> Option<String>;
162
163    /// Send messages and return the assistant response together with per-call extras.
164    ///
165    /// # Errors
166    ///
167    /// Same as [`chat`](Self::chat).
168    fn chat_with_extras<'a>(
169        &'a self,
170        messages: &'a [Message],
171    ) -> BoxFuture<'a, Result<(String, ChatExtras), LlmError>>;
172
173    /// Return the request payload that will be sent to the provider, for debug dumps.
174    #[must_use]
175    fn debug_request_json(
176        &self,
177        messages: &[Message],
178        tools: &[ToolDefinition],
179        stream: bool,
180    ) -> serde_json::Value;
181
182    /// Return the list of model identifiers this provider can serve.
183    fn list_models(&self) -> Vec<String>;
184
185    /// Whether this provider supports native structured output.
186    fn supports_structured_output(&self) -> bool;
187}
188
189impl<T: LlmProvider + std::fmt::Debug + Send + Sync + 'static> LlmProviderDyn for T {
190    fn context_window(&self) -> Option<usize> {
191        LlmProvider::context_window(self)
192    }
193
194    fn chat<'a>(&'a self, messages: &'a [Message]) -> BoxFuture<'a, Result<String, LlmError>> {
195        Box::pin(LlmProvider::chat(self, messages))
196    }
197
198    fn chat_stream<'a>(
199        &'a self,
200        messages: &'a [Message],
201    ) -> BoxFuture<'a, Result<ChatStream, LlmError>> {
202        Box::pin(LlmProvider::chat_stream(self, messages))
203    }
204
205    fn supports_streaming(&self) -> bool {
206        LlmProvider::supports_streaming(self)
207    }
208
209    fn embed<'a>(&'a self, text: &'a str) -> BoxFuture<'a, Result<Vec<f32>, LlmError>> {
210        Box::pin(LlmProvider::embed(self, text))
211    }
212
213    fn embed_batch<'a>(
214        &'a self,
215        texts: &'a [&'a str],
216    ) -> BoxFuture<'a, Result<Vec<Vec<f32>>, LlmError>> {
217        Box::pin(LlmProvider::embed_batch(self, texts))
218    }
219
220    fn supports_embeddings(&self) -> bool {
221        LlmProvider::supports_embeddings(self)
222    }
223
224    fn name(&self) -> &str {
225        LlmProvider::name(self)
226    }
227
228    fn model_identifier(&self) -> &str {
229        LlmProvider::model_identifier(self)
230    }
231
232    fn effective_model_identifier(&self) -> &str {
233        LlmProvider::effective_model_identifier(self)
234    }
235
236    fn supports_vision(&self) -> bool {
237        LlmProvider::supports_vision(self)
238    }
239
240    fn supports_tool_use(&self) -> bool {
241        LlmProvider::supports_tool_use(self)
242    }
243
244    fn chat_with_tools<'a>(
245        &'a self,
246        messages: &'a [Message],
247        tools: &'a [ToolDefinition],
248    ) -> BoxFuture<'a, Result<ChatResponse, LlmError>> {
249        Box::pin(LlmProvider::chat_with_tools(self, messages, tools))
250    }
251
252    fn last_cache_usage(&self) -> Option<(u64, u64)> {
253        LlmProvider::last_cache_usage(self)
254    }
255
256    fn last_usage(&self) -> Option<(u64, u64)> {
257        LlmProvider::last_usage(self)
258    }
259
260    fn take_compaction_summary(&self) -> Option<String> {
261        LlmProvider::take_compaction_summary(self)
262    }
263
264    fn chat_with_extras<'a>(
265        &'a self,
266        messages: &'a [Message],
267    ) -> BoxFuture<'a, Result<(String, ChatExtras), LlmError>> {
268        Box::pin(LlmProvider::chat_with_extras(self, messages))
269    }
270
271    fn debug_request_json(
272        &self,
273        messages: &[Message],
274        tools: &[ToolDefinition],
275        stream: bool,
276    ) -> serde_json::Value {
277        LlmProvider::debug_request_json(self, messages, tools, stream)
278    }
279
280    fn list_models(&self) -> Vec<String> {
281        LlmProvider::list_models(self)
282    }
283
284    fn supports_structured_output(&self) -> bool {
285        LlmProvider::supports_structured_output(self)
286    }
287}
288
289/// Send messages and parse the response into a typed value `T`.
290///
291/// This is the dyn-compatible equivalent of [`LlmProvider::chat_typed`]. Because
292/// `chat_typed` carries a generic type parameter, it cannot be part of a dyn-safe
293/// trait. Use this free function when working with `&dyn LlmProviderDyn` or
294/// `Arc<dyn LlmProviderDyn>`.
295///
296/// The default implementation injects the JSON schema into the system prompt and
297/// retries once on parse failure, matching the behaviour of the trait method.
298///
299/// # Errors
300///
301/// Returns [`LlmError::StructuredParse`] when the response cannot be parsed as `T`
302/// after one retry. Propagates any underlying [`LlmError`] from the provider.
303///
304/// # Examples
305///
306/// ```rust,no_run
307/// use std::sync::Arc;
308/// use schemars::JsonSchema;
309/// use serde::Deserialize;
310/// use zeph_llm::provider::{Message, Role};
311/// use zeph_llm::provider_dyn::{LlmProviderDyn, chat_typed_dyn};
312/// use zeph_llm::ollama::OllamaProvider;
313///
314/// #[derive(Debug, Deserialize, JsonSchema)]
315/// struct Answer {
316///     value: String,
317/// }
318///
319/// # async fn example() -> Result<(), zeph_llm::LlmError> {
320/// let provider = OllamaProvider::new(
321///     "http://localhost:11434",
322///     "llama3.2".into(),
323///     "nomic-embed-text".into(),
324/// );
325/// let dyn_provider: Arc<dyn LlmProviderDyn> = Arc::new(provider);
326/// let messages = vec![Message::from_legacy(Role::User, "What is 2+2?")];
327/// let answer: Answer = chat_typed_dyn(&*dyn_provider, &messages).await?;
328/// println!("{}", answer.value);
329/// # Ok(())
330/// # }
331/// ```
332#[tracing::instrument(name = "llm.provider_dyn.chat_typed_dyn", skip_all)]
333pub async fn chat_typed_dyn<T, P>(provider: &P, messages: &[Message]) -> Result<T, LlmError>
334where
335    T: DeserializeOwned + schemars::JsonSchema + 'static,
336    P: ?Sized + LlmProviderDyn,
337{
338    let (_, schema_json) = cached_schema::<T>()?;
339    let type_name = short_type_name::<T>();
340
341    let instruction = format!(
342        "Respond with a valid JSON object matching this schema. \
343         Output ONLY the JSON, no markdown fences or extra text.\n\n\
344         Type: {type_name}\nSchema:\n```json\n{schema_json}\n```"
345    );
346
347    let mut augmented = messages.to_vec();
348    augmented.insert(0, Message::from_legacy(Role::System, instruction));
349
350    let raw = provider.chat(&augmented).await?;
351    let cleaned = strip_json_fences(&raw);
352    match serde_json::from_str::<T>(cleaned) {
353        Ok(val) => Ok(val),
354        Err(first_err) => {
355            augmented.push(Message::from_legacy(Role::Assistant, &raw));
356            augmented.push(Message::from_legacy(
357                Role::User,
358                format!(
359                    "Your response was not valid JSON. Error: {first_err}. \
360                     Please output ONLY valid JSON matching the schema."
361                ),
362            ));
363            let retry_raw = provider.chat(&augmented).await?;
364            let retry_cleaned = strip_json_fences(&retry_raw);
365            serde_json::from_str::<T>(retry_cleaned)
366                .map_err(|e| LlmError::StructuredParse(format!("parse failed after retry: {e}")))
367        }
368    }
369}
370
371/// Strip markdown code fences from LLM output.
372fn strip_json_fences(s: &str) -> &str {
373    s.trim()
374        .trim_start_matches("```json")
375        .trim_start_matches("```")
376        .trim_end_matches("```")
377        .trim()
378}
379
380#[cfg(test)]
381mod tests {
382    use std::sync::Arc;
383
384    use super::*;
385    use crate::provider::{ChatStream, StreamChunk};
386
387    #[derive(Debug)]
388    struct StubProvider {
389        response: String,
390    }
391
392    impl LlmProvider for StubProvider {
393        async fn chat(&self, _messages: &[Message]) -> Result<String, LlmError> {
394            Ok(self.response.clone())
395        }
396
397        async fn chat_stream(&self, messages: &[Message]) -> Result<ChatStream, LlmError> {
398            let response = LlmProvider::chat(self, messages).await?;
399            Ok(Box::pin(tokio_stream::once(Ok(StreamChunk::Content(
400                response,
401            )))))
402        }
403
404        fn supports_streaming(&self) -> bool {
405            false
406        }
407
408        async fn embed(&self, _text: &str) -> Result<Vec<f32>, LlmError> {
409            Ok(vec![0.1, 0.2, 0.3])
410        }
411
412        fn supports_embeddings(&self) -> bool {
413            false
414        }
415
416        fn name(&self) -> &'static str {
417            "stub"
418        }
419    }
420
421    #[tokio::test]
422    async fn dyn_chat_works() {
423        let provider: Arc<dyn LlmProviderDyn> = Arc::new(StubProvider {
424            response: "hello".into(),
425        });
426        let msgs = vec![Message::from_legacy(Role::User, "test")];
427        let result = provider.chat(&msgs).await.unwrap();
428        assert_eq!(result, "hello");
429    }
430
431    #[tokio::test]
432    async fn dyn_embed_works() {
433        let provider: Arc<dyn LlmProviderDyn> = Arc::new(StubProvider {
434            response: String::new(),
435        });
436        let result = provider.embed("hello").await.unwrap();
437        assert_eq!(result, vec![0.1_f32, 0.2, 0.3]);
438    }
439
440    #[test]
441    fn dyn_sync_methods_forward_correctly() {
442        let provider: Arc<dyn LlmProviderDyn> = Arc::new(StubProvider {
443            response: String::new(),
444        });
445        assert_eq!(provider.name(), "stub");
446        assert!(!provider.supports_streaming());
447        assert!(!provider.supports_embeddings());
448        assert!(provider.context_window().is_none());
449        assert!(provider.last_cache_usage().is_none());
450        assert!(provider.last_usage().is_none());
451    }
452
453    #[derive(Debug, serde::Deserialize, schemars::JsonSchema, PartialEq)]
454    struct TestOutput {
455        value: String,
456    }
457
458    #[tokio::test]
459    async fn chat_typed_dyn_happy_path() {
460        let provider: Arc<dyn LlmProviderDyn> = Arc::new(StubProvider {
461            response: r#"{"value": "hello"}"#.into(),
462        });
463        let msgs = vec![Message::from_legacy(Role::User, "test")];
464        let result: TestOutput = chat_typed_dyn(&*provider, &msgs).await.unwrap();
465        assert_eq!(
466            result,
467            TestOutput {
468                value: "hello".into()
469            }
470        );
471    }
472
473    #[tokio::test]
474    async fn chat_typed_dyn_strips_fences() {
475        let provider: Arc<dyn LlmProviderDyn> = Arc::new(StubProvider {
476            response: "```json\n{\"value\": \"fenced\"}\n```".into(),
477        });
478        let msgs = vec![Message::from_legacy(Role::User, "test")];
479        let result: TestOutput = chat_typed_dyn(&*provider, &msgs).await.unwrap();
480        assert_eq!(
481            result,
482            TestOutput {
483                value: "fenced".into()
484            }
485        );
486    }
487}
zeph_llm/provider_dyn.rs

zeph_llm/
provider_dyn.rs
