Trait StreamingEngine 

pub trait StreamingEngine: Send + Sync {
    // Required methods
    fn handle_completion<'life0, 'async_trait>(
        &'life0 self,
        req: SingleIn<NvCreateCompletionRequest>,
    ) -> Pin<Box<dyn Future<Output = Result<ManyOut<Annotated<NvCreateCompletionResponse>>, Error>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn handle_chat<'life0, 'async_trait>(
        &'life0 self,
        req: SingleIn<NvCreateChatCompletionRequest>,
    ) -> Pin<Box<dyn Future<Output = Result<ManyOut<Annotated<NvCreateChatCompletionStreamResponse>>, Error>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
}

Trait that allows handling both completion and chat completions requests

Required Methods

fn handle_completion<'life0, 'async_trait>( &'life0 self, req: SingleIn<NvCreateCompletionRequest>, ) -> Pin<Box<dyn Future<Output = Result<ManyOut<Annotated<NvCreateCompletionResponse>>, Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

fn handle_chat<'life0, 'async_trait>( &'life0 self, req: SingleIn<NvCreateChatCompletionRequest>, ) -> Pin<Box<dyn Future<Output = Result<ManyOut<Annotated<NvCreateChatCompletionStreamResponse>>, Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Implementors

impl<E> StreamingEngine for EngineDispatcher<E>

where E: AsyncEngine<SingleIn<NvCreateCompletionRequest>, ManyOut<Annotated<NvCreateCompletionResponse>>, Error> + AsyncEngine<SingleIn<NvCreateChatCompletionRequest>, ManyOut<Annotated<NvCreateChatCompletionStreamResponse>>, Error> + AsyncEngine<SingleIn<NvCreateEmbeddingRequest>, ManyOut<Annotated<NvCreateEmbeddingResponse>>, Error> + Send + Sync,