Skip to main content

GenerationProvider

Trait GenerationProvider 

Source
pub trait GenerationProvider: Send + Sync {
    // Required methods
    fn name(&self) -> &str;
    fn capabilities(&self) -> ProviderCapabilities;
    fn generation_capabilities(&self) -> GenerationCapabilities;
    fn list_models<'life0, 'async_trait>(
        &'life0 self,
        cancellation: Option<CancellationToken>,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<GenerationModelInfo>, RuntimeError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait;
    fn chat<'life0, 'async_trait>(
        &'life0 self,
        request: ChatRequest,
        cancellation: Option<CancellationToken>,
    ) -> Pin<Box<dyn Future<Output = Result<ChatResponse, RuntimeError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait;
    fn chat_stream<'life0, 'async_trait>(
        &'life0 self,
        request: ChatRequest,
        cancellation: Option<CancellationToken>,
    ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<ChatStreamEvent, RuntimeError>> + Send>>, RuntimeError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait;
}
Expand description

A provider of generative inference. Separate from crate::Provider, which serves typed inference.

Required Methods§

Source

fn name(&self) -> &str

Source

fn capabilities(&self) -> ProviderCapabilities

Shared provider descriptor (local/remote, endpoint, streaming, …).

Source

fn generation_capabilities(&self) -> GenerationCapabilities

Source

fn list_models<'life0, 'async_trait>( &'life0 self, cancellation: Option<CancellationToken>, ) -> Pin<Box<dyn Future<Output = Result<Vec<GenerationModelInfo>, RuntimeError>> + Send + 'async_trait>>
where 'life0: 'async_trait, Self: 'async_trait,

Lists models when the upstream supports discovery; otherwise returns a CapabilityMismatch error.

Source

fn chat<'life0, 'async_trait>( &'life0 self, request: ChatRequest, cancellation: Option<CancellationToken>, ) -> Pin<Box<dyn Future<Output = Result<ChatResponse, RuntimeError>> + Send + 'async_trait>>
where 'life0: 'async_trait, Self: 'async_trait,

Source

fn chat_stream<'life0, 'async_trait>( &'life0 self, request: ChatRequest, cancellation: Option<CancellationToken>, ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<ChatStreamEvent, RuntimeError>> + Send>>, RuntimeError>> + Send + 'async_trait>>
where 'life0: 'async_trait, Self: 'async_trait,

Dropping the returned stream must release the upstream connection.

Dyn Compatibility§

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors§