pub trait GenerationProvider: Send + Sync {
// Required methods
fn name(&self) -> &str;
fn capabilities(&self) -> ProviderCapabilities;
fn generation_capabilities(&self) -> GenerationCapabilities;
fn list_models<'life0, 'async_trait>(
&'life0 self,
cancellation: Option<CancellationToken>,
) -> Pin<Box<dyn Future<Output = Result<Vec<GenerationModelInfo>, RuntimeError>> + Send + 'async_trait>>
where 'life0: 'async_trait,
Self: 'async_trait;
fn chat<'life0, 'async_trait>(
&'life0 self,
request: ChatRequest,
cancellation: Option<CancellationToken>,
) -> Pin<Box<dyn Future<Output = Result<ChatResponse, RuntimeError>> + Send + 'async_trait>>
where 'life0: 'async_trait,
Self: 'async_trait;
fn chat_stream<'life0, 'async_trait>(
&'life0 self,
request: ChatRequest,
cancellation: Option<CancellationToken>,
) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<ChatStreamEvent, RuntimeError>> + Send>>, RuntimeError>> + Send + 'async_trait>>
where 'life0: 'async_trait,
Self: 'async_trait;
}Expand description
A provider of generative inference. Separate from crate::Provider,
which serves typed inference.
Required Methods§
fn name(&self) -> &str
Sourcefn capabilities(&self) -> ProviderCapabilities
fn capabilities(&self) -> ProviderCapabilities
Shared provider descriptor (local/remote, endpoint, streaming, …).
fn generation_capabilities(&self) -> GenerationCapabilities
Sourcefn list_models<'life0, 'async_trait>(
&'life0 self,
cancellation: Option<CancellationToken>,
) -> Pin<Box<dyn Future<Output = Result<Vec<GenerationModelInfo>, RuntimeError>> + Send + 'async_trait>>where
'life0: 'async_trait,
Self: 'async_trait,
fn list_models<'life0, 'async_trait>(
&'life0 self,
cancellation: Option<CancellationToken>,
) -> Pin<Box<dyn Future<Output = Result<Vec<GenerationModelInfo>, RuntimeError>> + Send + 'async_trait>>where
'life0: 'async_trait,
Self: 'async_trait,
Lists models when the upstream supports discovery; otherwise returns
a CapabilityMismatch error.
fn chat<'life0, 'async_trait>(
&'life0 self,
request: ChatRequest,
cancellation: Option<CancellationToken>,
) -> Pin<Box<dyn Future<Output = Result<ChatResponse, RuntimeError>> + Send + 'async_trait>>where
'life0: 'async_trait,
Self: 'async_trait,
Sourcefn chat_stream<'life0, 'async_trait>(
&'life0 self,
request: ChatRequest,
cancellation: Option<CancellationToken>,
) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<ChatStreamEvent, RuntimeError>> + Send>>, RuntimeError>> + Send + 'async_trait>>where
'life0: 'async_trait,
Self: 'async_trait,
fn chat_stream<'life0, 'async_trait>(
&'life0 self,
request: ChatRequest,
cancellation: Option<CancellationToken>,
) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<ChatStreamEvent, RuntimeError>> + Send>>, RuntimeError>> + Send + 'async_trait>>where
'life0: 'async_trait,
Self: 'async_trait,
Dropping the returned stream must release the upstream connection.
Dyn Compatibility§
This trait is dyn compatible.
In older versions of Rust, dyn compatibility was called "object safety".