Trait Engine 

pub trait Engine: Send + Sync {
    // Required methods
    fn load<'life0, 'life1, 'async_trait>(
        &'life0 mut self,
        model_path: &'life1 Path,
    ) -> Pin<Box<dyn Future<Output = Result<(), EngineError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn infer(
        &self,
        request: &InferenceRequest,
    ) -> Result<BinaryTensorPacket, EngineError>;
    fn infer_stream(&self, request: &InferenceRequest) -> EngineStream;
    fn unload(&mut self);
    fn metrics(&self) -> EngineMetrics;
    fn health_check(&self) -> Result<(), EngineError>;

    // Provided methods
    fn infer_async<'life0, 'life1, 'async_trait>(
        &'life0 self,
        request: &'life1 InferenceRequest,
    ) -> Pin<Box<dyn Future<Output = Result<BinaryTensorPacket, EngineError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn infer_batch(
        &self,
        requests: &[InferenceRequest],
    ) -> Result<Vec<BinaryTensorPacket>, EngineError> { ... }
    fn infer_batch_async<'life0, 'life1, 'async_trait>(
        &'life0 self,
        requests: &'life1 [InferenceRequest],
    ) -> Pin<Box<dyn Future<Output = Result<Vec<BinaryTensorPacket>, EngineError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn infer_with_cancellation(
        &self,
        request: &InferenceRequest,
        cancellation: &CancellationToken,
    ) -> Result<BinaryTensorPacket, EngineError> { ... }
    fn warmup<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<(), EngineError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait { ... }
    fn model_info(&self) -> Option<EngineModelInfo> { ... }
}

Required Methods

fn load<'life0, 'life1, 'async_trait>( &'life0 mut self, model_path: &'life1 Path, ) -> Pin<Box<dyn Future<Output = Result<(), EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Load model weights and prepare runtime state.

fn infer( &self, request: &InferenceRequest, ) -> Result<BinaryTensorPacket, EngineError>

Run a single inference request and return the output tensor.

fn infer_stream(&self, request: &InferenceRequest) -> EngineStream

Run a streaming inference request.

fn unload(&mut self)

Release any held resources.

fn metrics(&self) -> EngineMetrics

Report the latest metrics snapshot.

fn health_check(&self) -> Result<(), EngineError>

Check if the model is healthy.

Provided Methods

fn infer_async<'life0, 'life1, 'async_trait>( &'life0 self, request: &'life1 InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<BinaryTensorPacket, EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Run a single inference request asynchronously.

fn infer_batch( &self, requests: &[InferenceRequest], ) -> Result<Vec<BinaryTensorPacket>, EngineError>

Run a batch of inference requests.

fn infer_batch_async<'life0, 'life1, 'async_trait>( &'life0 self, requests: &'life1 [InferenceRequest], ) -> Pin<Box<dyn Future<Output = Result<Vec<BinaryTensorPacket>, EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Run a batch of inference requests asynchronously.

fn infer_with_cancellation( &self, request: &InferenceRequest, cancellation: &CancellationToken, ) -> Result<BinaryTensorPacket, EngineError>

Run inference with cancellation support.

fn warmup<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Warm up the model runtime before serving requests.

fn model_info(&self) -> Option<EngineModelInfo>

Report model metadata when available.

Trait Implementations

impl Engine for Box<dyn Engine>

fn load<'life0, 'life1, 'async_trait>( &'life0 mut self, model_path: &'life1 Path, ) -> Pin<Box<dyn Future<Output = Result<(), EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Load model weights and prepare runtime state.

fn infer( &self, request: &InferenceRequest, ) -> Result<BinaryTensorPacket, EngineError>

Run a single inference request and return the output tensor.

fn infer_async<'life0, 'life1, 'async_trait>( &'life0 self, request: &'life1 InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<BinaryTensorPacket, EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Run a single inference request asynchronously.

fn infer_batch( &self, requests: &[InferenceRequest], ) -> Result<Vec<BinaryTensorPacket>, EngineError>

Run a batch of inference requests.

fn infer_batch_async<'life0, 'life1, 'async_trait>( &'life0 self, requests: &'life1 [InferenceRequest], ) -> Pin<Box<dyn Future<Output = Result<Vec<BinaryTensorPacket>, EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Run a batch of inference requests asynchronously.

fn infer_stream(&self, request: &InferenceRequest) -> EngineStream

Run a streaming inference request.

fn infer_with_cancellation( &self, request: &InferenceRequest, cancellation: &CancellationToken, ) -> Result<BinaryTensorPacket, EngineError>

Run inference with cancellation support.

fn warmup<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Warm up the model runtime before serving requests.

fn unload(&mut self)

Release any held resources.

fn metrics(&self) -> EngineMetrics

Report the latest metrics snapshot.

fn model_info(&self) -> Option<EngineModelInfo>

Report model metadata when available.

fn health_check(&self) -> Result<(), EngineError>

Check if the model is healthy.

Implementations on Foreign Types

impl Engine for Box<dyn Engine>

fn load<'life0, 'life1, 'async_trait>( &'life0 mut self, model_path: &'life1 Path, ) -> Pin<Box<dyn Future<Output = Result<(), EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

fn infer( &self, request: &InferenceRequest, ) -> Result<BinaryTensorPacket, EngineError>

fn infer_async<'life0, 'life1, 'async_trait>( &'life0 self, request: &'life1 InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<BinaryTensorPacket, EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

fn infer_batch( &self, requests: &[InferenceRequest], ) -> Result<Vec<BinaryTensorPacket>, EngineError>

fn infer_batch_async<'life0, 'life1, 'async_trait>( &'life0 self, requests: &'life1 [InferenceRequest], ) -> Pin<Box<dyn Future<Output = Result<Vec<BinaryTensorPacket>, EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

fn infer_stream(&self, request: &InferenceRequest) -> EngineStream

fn infer_with_cancellation( &self, request: &InferenceRequest, cancellation: &CancellationToken, ) -> Result<BinaryTensorPacket, EngineError>

fn warmup<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), EngineError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

fn unload(&mut self)

fn metrics(&self) -> EngineMetrics

fn model_info(&self) -> Option<EngineModelInfo>

fn health_check(&self) -> Result<(), EngineError>

Implementors