Trait BatchProvider 

pub trait BatchProvider: Send + Sync {
    // Required methods
    fn batch<'life0, 'async_trait>(
        &'life0 self,
        requests: Vec<CompletionRequest>,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<CompletionResponse>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn max_batch_size(&self) -> usize;
}

For providers that support batch processing (e.g. Anthropic Batch API — 50% cheaper).

Required Methods

fn batch<'life0, 'async_trait>( &'life0 self, requests: Vec<CompletionRequest>, ) -> Pin<Box<dyn Future<Output = Result<Vec<CompletionResponse>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Submit a batch of completion requests for async processing.

fn max_batch_size(&self) -> usize

Maximum number of requests in a single batch.

Implementors