ferrum_interfaces/
engine.rs

1//! Inference engine interface with streaming and batch support
2//!
3//! This module provides the top-level inference engine interface that
4//! orchestrates all other components: tokenizer, model executor, scheduler,
5//! and sampler.
6
7use async_trait::async_trait;
8use ferrum_types::{EngineConfig, InferenceRequest, InferenceResponse, Result, StreamChunk};
9use futures::Stream;
10use std::pin::Pin;
11
12/// Core inference engine trait
13#[async_trait]
14pub trait InferenceEngine: Send + Sync {
15    /// Execute single inference request
16    async fn infer(&self, request: InferenceRequest) -> Result<InferenceResponse>;
17
18    /// Execute streaming inference request
19    async fn infer_stream(
20        &self,
21        request: InferenceRequest,
22    ) -> Result<Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send>>>;
23
24    /// Get current engine status
25    async fn status(&self) -> ferrum_types::EngineStatus;
26
27    /// Shutdown engine gracefully
28    async fn shutdown(&self) -> Result<()>;
29
30    /// Get engine configuration
31    fn config(&self) -> &EngineConfig;
32
33    /// Get engine metrics
34    fn metrics(&self) -> ferrum_types::EngineMetrics;
35
36    /// Health check
37    async fn health_check(&self) -> ferrum_types::HealthStatus;
38
39    /// Embed raw text string → float vector (engine handles tokenization).
40    async fn embed_text(&self, _text: &str) -> Result<Vec<f32>> {
41        Err(ferrum_types::FerrumError::model(
42            "This engine does not support text embedding",
43        ))
44    }
45
46    /// Embed image (file path or base64) → float vector. Default: not supported.
47    async fn embed_image(&self, _image: &str) -> Result<Vec<f32>> {
48        Err(ferrum_types::FerrumError::model(
49            "This engine does not support image embedding",
50        ))
51    }
52
53    /// Get embedding dimension. Default: 0 (not an embedding model).
54    fn embedding_dim(&self) -> usize {
55        0
56    }
57
58    /// Transcribe audio file → text. Default: not supported.
59    async fn transcribe_file(&self, _path: &str, _language: Option<&str>) -> Result<String> {
60        Err(ferrum_types::FerrumError::model(
61            "This engine does not support audio transcription",
62        ))
63    }
64
65    /// Transcribe audio bytes (WAV) → text. Default: not supported.
66    async fn transcribe_bytes(&self, _data: &[u8], _language: Option<&str>) -> Result<String> {
67        Err(ferrum_types::FerrumError::model(
68            "This engine does not support audio transcription",
69        ))
70    }
71}
72
73/// Advanced engine capabilities
74#[async_trait]
75pub trait AdvancedInferenceEngine: InferenceEngine {
76    /// Execute batch inference
77    async fn infer_batch(
78        &self,
79        requests: Vec<InferenceRequest>,
80    ) -> Result<Vec<Result<InferenceResponse>>>;
81
82    /// Execute speculative inference
83    async fn infer_speculative(
84        &self,
85        request: InferenceRequest,
86        speculation_config: ferrum_types::SpeculationConfig,
87    ) -> Result<InferenceResponse>;
88
89    /// Warm up engine with sample requests
90    async fn warmup(
91        &mut self,
92        warmup_requests: Vec<InferenceRequest>,
93    ) -> Result<ferrum_types::WarmupResult>;
94
95    /// Configure engine at runtime
96    async fn reconfigure(&mut self, config: EngineConfig) -> Result<()>;
97
98    /// Get detailed diagnostics
99    async fn diagnostics(&self) -> ferrum_types::DiagnosticsReport;
100
101    /// Export engine state for debugging
102    async fn export_state(&self) -> Result<ferrum_types::EngineState>;
103
104    /// Import engine state for debugging/testing
105    async fn import_state(&mut self, state: ferrum_types::EngineState) -> Result<()>;
106}
107
108/// Speculation configuration for speculative decoding
109pub type SpeculationConfig = ferrum_types::SpeculationConfig;
110
111/// Hardware constraints alias
112pub type HardwareConstraints = ferrum_types::HardwareConstraints;
113
114/// Request characteristics alias
115pub type RequestCharacteristics = ferrum_types::RequestCharacteristics;
116
117/// Latency requirements alias
118pub type LatencyRequirements = ferrum_types::LatencyRequirements;
ferrum_interfaces/engine.rs

ferrum_interfaces/
engine.rs
