llm/
provider.rs

1use crate::LlmError;
2use crate::LlmModel;
3use crate::Result as LlmResult;
4use crate::catalog::{Provider, ReasoningEffortError};
5use std::pin::Pin;
6use tokio_stream::Stream;
7use utils::ReasoningEffort;
8
9use super::{Context, LlmResponse};
10
11/// A stream of [`LlmResponse`] events from an LLM provider.
12///
13/// This is a pinned, boxed, `Send` stream used as the return type of
14/// [`StreamingModelProvider::stream_response`]. Boxing is required to support
15/// trait objects (`Vec<Box<dyn StreamingModelProvider>>`) in types like
16/// [`AlloyedModelProvider`](crate::alloyed::AlloyedModelProvider).
17pub type LlmResponseStream = Pin<Box<dyn Stream<Item = LlmResult<LlmResponse>> + Send>>;
18
19#[doc = include_str!("docs/streaming_model_provider.md")]
20pub trait StreamingModelProvider: Send + Sync {
21    fn stream_response(&self, context: &Context) -> LlmResponseStream;
22
23    /// Provider and model, e.g. `Anthropic (claude-sonnet-4-5)`.
24    fn display_name(&self) -> String {
25        self.model()
26            .map_or_else(|| "Unknown model".to_string(), |model| display_name(model.provider_enum(), &model.model_id()))
27    }
28
29    /// Context window size in tokens for the current model.
30    /// Returns `None` for unknown models (e.g. Ollama, `LlamaCpp`).
31    fn context_window(&self) -> Option<u32> {
32        self.model().and_then(|model| model.context_window())
33    }
34
35    /// The `LlmModel` this provider is currently configured to use.
36    /// Returns `None` for providers where the model is unknown at compile time
37    /// (e.g. test fakes).
38    fn model(&self) -> Option<LlmModel> {
39        None
40    }
41}
42
43pub(crate) fn display_name(provider: Provider, model_id: &str) -> String {
44    if model_id.is_empty() {
45        provider.display_name().to_string()
46    } else {
47        format!("{} ({model_id})", provider.display_name())
48    }
49}
50
51pub(crate) fn validate_reasoning(context: &Context, model: Option<&LlmModel>) -> LlmResult<()> {
52    if context.reasoning_effort() != ReasoningEffort::Disabled {
53        return Ok(());
54    }
55
56    let model = model.ok_or_else(|| ReasoningEffortError::Unsupported {
57        model: "unknown".to_string(),
58        effort: ReasoningEffort::Disabled,
59        supported: Vec::new(),
60    })?;
61
62    if !model.supports_reasoning_off() {
63        model.validate_reasoning_effort(ReasoningEffort::Disabled)?;
64    }
65
66    if !model.supports_reasoning_off_transport() {
67        return Err(LlmError::UnsupportedDisableTransport { model: model.to_string() });
68    }
69
70    Ok(())
71}
72
73impl StreamingModelProvider for Box<dyn StreamingModelProvider> {
74    fn stream_response(&self, context: &Context) -> LlmResponseStream {
75        (**self).stream_response(context)
76    }
77
78    fn display_name(&self) -> String {
79        (**self).display_name()
80    }
81
82    fn context_window(&self) -> Option<u32> {
83        (**self).context_window()
84    }
85
86    fn model(&self) -> Option<LlmModel> {
87        (**self).model()
88    }
89}
90
91impl<T: StreamingModelProvider + ?Sized> StreamingModelProvider for std::sync::Arc<T> {
92    fn stream_response(&self, context: &Context) -> LlmResponseStream {
93        (**self).stream_response(context)
94    }
95
96    fn display_name(&self) -> String {
97        (**self).display_name()
98    }
99
100    fn context_window(&self) -> Option<u32> {
101        (**self).context_window()
102    }
103
104    fn model(&self) -> Option<LlmModel> {
105        (**self).model()
106    }
107}
llm/provider.rs

llm/
provider.rs
