Skip to main content

InferenceEngine

ferrum_interfaces::engine

Trait InferenceEngine 

Source 
pub trait InferenceEngine: Send + Sync {
    // Required methods
    fn infer<'life0, 'async_trait>(
        &'life0 self,
        request: InferenceRequest,
    ) -> Pin<Box<dyn Future<Output = Result<InferenceResponse>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn infer_stream<'life0, 'async_trait>(
        &'life0 self,
        request: InferenceRequest,
    ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send>>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn status<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = EngineStatus> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn shutdown<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn config(&self) -> &EngineConfig;
    fn metrics(&self) -> EngineMetrics;
    fn health_check<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = HealthStatus> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
}
Expand description

Core inference engine trait

Required Methods§

Source

fn infer<'life0, 'async_trait>( &'life0 self, request: InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<InferenceResponse>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Execute single inference request

Source

fn infer_stream<'life0, 'async_trait>( &'life0 self, request: InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send>>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Execute streaming inference request

Source

fn status<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = EngineStatus> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Get current engine status

Source

fn shutdown<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Shutdown engine gracefully

Source

fn config(&self) -> &EngineConfig

Get engine configuration

Source

fn metrics(&self) -> EngineMetrics

Get engine metrics

Source

fn health_check<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = HealthStatus> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Health check

Implementors§