defect_core/llm/
provider.rs

1//! [`LlmProvider`] trait — the primary LLM vendor integration interface.
2
3use std::pin::Pin;
4
5use futures::{Stream, future::BoxFuture};
6use tokio_util::sync::CancellationToken;
7
8use super::capability::{Capabilities, HostedCapabilities};
9use super::chunk::ProviderChunk;
10use super::error::ProviderError;
11use super::model::{ModelInfo, ProviderInfo};
12use super::request::CompletionRequest;
13
14/// A type-erased stream of events produced by a provider during streaming generation,
15/// enabling direct use with `dyn LlmProvider`.
16pub type ProviderStream = Pin<Box<dyn Stream<Item = Result<ProviderChunk, ProviderError>> + Send>>;
17
18/// LLM provider abstraction.
19///
20/// Cancellation semantics: [`LlmProvider::complete`] receives a [`CancellationToken`];
21/// the caller may call `cancel()` at any point to abort the call and the downstream
22/// stream. Dropping the returned stream also counts as cancellation.
23pub trait LlmProvider: Send + Sync {
24    /// Provider metadata (vendor name, API style, tracing labels, etc.).
25    fn info(&self) -> ProviderInfo;
26
27    /// Provider-level capability matrix. Model-level differences are expressed via
28    /// [`super::ModelCapabilityOverrides`] and merged on demand by the main loop.
29    fn capabilities(&self) -> Capabilities;
30
31    /// The set of hosted capabilities that this provider adapter advertises.
32    ///
33    /// Unlike [`Self::capabilities`] (which describes model-level properties), this
34    /// reflects the current adapter implementation state: whether it can expose hosted
35    /// `web_search`, `fetch`, etc. to the model over the wire. During session startup,
36    /// this value is read together with `capabilities.web_search.mode` to determine the
37    /// source of hosted web search capabilities.
38    ///
39    /// The default implementation returns all `false`; new providers do not need to
40    /// override it. Adapters that truly support hosted capabilities (Anthropic / OpenAI
41    /// Responses) should explicitly override this method.
42    fn hosted_capabilities(&self) -> HostedCapabilities {
43        HostedCapabilities::default()
44    }
45
46    /// Lists the models currently available from this provider.
47    ///
48    /// The implementation may make network calls (e.g., OpenAI `/v1/models`); results
49    /// should be cached inside the provider for synchronous lookup by
50    /// [`Self::model_info`].
51    ///
52    /// # Errors
53    ///
54    /// Network errors, authentication errors, server errors, etc., are all mapped to
55    /// [`ProviderError`].
56    fn list_models(&self) -> BoxFuture<'_, Result<Vec<ModelInfo>, ProviderError>>;
57
58    /// Synchronously query metadata for a given model.
59    ///
60    /// This is a fast path for context trimming in the main loop; **must not trigger a
61    /// network call**.
62    /// Returns `None` if the provider's cache does not contain the model. The caller may
63    /// then decide to call [`Self::list_models`] and retry, or treat it as an unknown
64    /// model.
65    fn model_info(&self, model_id: &str) -> Option<ModelInfo>;
66
67    /// Start a streaming generation.
68    ///
69    /// # Errors
70    ///
71    /// Authentication failures, invalid parameters, transport errors, server errors, etc.
72    /// are all mapped to [`ProviderError`]. Errors produced within the stream are
73    /// delivered via the stream's `Err` variant, not through this return value.
74    fn complete(
75        &self,
76        req: CompletionRequest,
77        cancel: CancellationToken,
78    ) -> BoxFuture<'_, Result<ProviderStream, ProviderError>>;
79}
defect_core/llm/provider.rs

defect_core/llm/
provider.rs
