Skip to main content

InferenceEngine

ferrum_interfaces::engine

Trait InferenceEngine 

Source 
pub trait InferenceEngine: Send + Sync {
    // Required methods
    fn infer<'life0, 'async_trait>(
        &'life0 self,
        request: InferenceRequest,
    ) -> Pin<Box<dyn Future<Output = Result<InferenceResponse>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn infer_stream<'life0, 'async_trait>(
        &'life0 self,
        request: InferenceRequest,
    ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send>>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn status<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = EngineStatus> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn shutdown<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn config(&self) -> &EngineConfig;
    fn metrics(&self) -> EngineMetrics;
    fn health_check<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = HealthStatus> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;

    // Provided methods
    fn embed_text<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _text: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn embed_image<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _image: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn embedding_dim(&self) -> usize { ... }
}
Expand description

Core inference engine trait

Required Methods§

Source

fn infer<'life0, 'async_trait>( &'life0 self, request: InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<InferenceResponse>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Execute single inference request

Source

fn infer_stream<'life0, 'async_trait>( &'life0 self, request: InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send>>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Execute streaming inference request

Source

fn status<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = EngineStatus> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Get current engine status

Source

fn shutdown<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Shutdown engine gracefully

Source

fn config(&self) -> &EngineConfig

Get engine configuration

Source

fn metrics(&self) -> EngineMetrics

Get engine metrics

Source

fn health_check<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = HealthStatus> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Health check

Provided Methods§

Source

fn embed_text<'life0, 'life1, 'async_trait>( &'life0 self, _text: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Embed raw text string → float vector (engine handles tokenization).

Source

fn embed_image<'life0, 'life1, 'async_trait>( &'life0 self, _image: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Embed image (file path or base64) → float vector. Default: not supported.

Source

fn embedding_dim(&self) -> usize

Get embedding dimension. Default: 0 (not an embedding model).

Implementors§