ferrum_interfaces/
engine.rs

1//! Inference engine interfaces — split per modality.
2//!
3//! Phase 5a step 2 splits the historical mega-trait (which mixed LLM
4//! generation, embedding, transcription, and TTS in one) into a base
5//! lifecycle trait and four modality-specific supertraits. Each
6//! engine impl now implements exactly the trait its modality needs;
7//! no more inert "unsupported" stubs.
8
9use async_trait::async_trait;
10use ferrum_types::{
11    EngineConfig, ExecutorAdmissionSnapshot, InferenceRequest, InferenceResponse, Result,
12    StreamChunk,
13};
14use futures::Stream;
15use std::pin::Pin;
16
17/// Lifecycle / status methods shared by every engine kind.
18///
19/// LLM engines, embedders, transcribers, and TTS services all expose
20/// the same minimal status/metrics surface to the server / CLI. The
21/// modality-specific traits below extend this base.
22#[async_trait]
23pub trait InferenceEngine: Send + Sync {
24    /// Get current engine status.
25    async fn status(&self) -> ferrum_types::EngineStatus;
26
27    /// Shutdown engine gracefully.
28    async fn shutdown(&self) -> Result<()>;
29
30    /// Get engine configuration.
31    fn config(&self) -> &EngineConfig;
32
33    /// Get engine metrics.
34    fn metrics(&self) -> ferrum_types::EngineMetrics;
35
36    /// Health check.
37    async fn health_check(&self) -> ferrum_types::HealthStatus;
38
39    /// Optional cache metrics emitted by concrete LLM engines.
40    ///
41    /// The default keeps non-LLM and stub engines source-compatible. Real
42    /// engines can expose prefix/session cache counters without forcing those
43    /// fields into every modality's core metrics type.
44    fn cache_metrics_snapshot(&self) -> Option<serde_json::Value> {
45        None
46    }
47
48    /// Runtime-authoritative admission state. Startup sizing estimates are
49    /// intentionally not accepted through this method.
50    fn admission_snapshot(&self) -> Result<Option<ExecutorAdmissionSnapshot>> {
51        Ok(None)
52    }
53
54    /// Optional LoRA runtime metrics emitted by concrete LLM engines.
55    fn lora_metrics_snapshot(&self) -> Option<serde_json::Value> {
56        None
57    }
58}
59
60/// LLM text-generation engine.
61///
62/// Implemented by `ContinuousBatchEngine` (the production path) and
63/// `DefaultInferenceEngine` (legacy reference path). Backs
64/// `/v1/chat/completions` and `/v1/completions`.
65#[async_trait]
66pub trait LlmInferenceEngine: InferenceEngine {
67    /// Execute single inference request.
68    async fn infer(&self, request: InferenceRequest) -> Result<InferenceResponse>;
69
70    /// Execute streaming inference request.
71    async fn infer_stream(
72        &self,
73        request: InferenceRequest,
74    ) -> Result<Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send>>>;
75}
76
77/// Embedding engine (CLIP, BERT, etc.).
78///
79/// Backs `/v1/embeddings`. Distinct from LLM engines — no token
80/// generation, no scheduling, no KV cache.
81#[async_trait]
82pub trait EmbedEngine: InferenceEngine {
83    /// Embed raw text string → float vector (engine handles tokenization).
84    async fn embed_text(&self, text: &str) -> Result<Vec<f32>>;
85
86    /// Embed image (file path or base64) → float vector.
87    async fn embed_image(&self, image: &str) -> Result<Vec<f32>>;
88
89    /// Get embedding dimension.
90    fn embedding_dim(&self) -> usize;
91}
92
93/// Speech-to-text (Whisper) engine.
94///
95/// Backs `/v1/audio/transcriptions`.
96#[async_trait]
97pub trait TranscribeEngine: InferenceEngine {
98    /// Transcribe audio file → text.
99    async fn transcribe_file(&self, path: &str, language: Option<&str>) -> Result<String>;
100
101    /// Transcribe audio bytes (WAV / etc.) → text.
102    async fn transcribe_bytes(&self, data: &[u8], language: Option<&str>) -> Result<String>;
103}
104
105/// Text-to-speech (Qwen3-TTS, etc.) engine.
106///
107/// Backs `/v1/audio/speech`.
108#[async_trait]
109pub trait TtsEngine: InferenceEngine {
110    /// Synthesize speech → PCM audio chunks (streaming).
111    /// Returns Vec of PCM f32 samples per chunk.
112    async fn synthesize_speech(
113        &self,
114        text: &str,
115        language: Option<&str>,
116        chunk_frames: usize,
117    ) -> Result<Vec<Vec<f32>>>;
118
119    /// Get TTS sample rate.
120    fn tts_sample_rate(&self) -> u32;
121}
122
123/// Advanced engine capabilities — opt-in addition to LLM engines that
124/// support batching / speculation / runtime reconfig / diagnostics.
125#[async_trait]
126pub trait AdvancedInferenceEngine: LlmInferenceEngine {
127    /// Execute batch inference.
128    async fn infer_batch(
129        &self,
130        requests: Vec<InferenceRequest>,
131    ) -> Result<Vec<Result<InferenceResponse>>>;
132
133    /// Execute speculative inference.
134    async fn infer_speculative(
135        &self,
136        request: InferenceRequest,
137        speculation_config: ferrum_types::SpeculationConfig,
138    ) -> Result<InferenceResponse>;
139
140    /// Warm up engine with sample requests.
141    async fn warmup(
142        &mut self,
143        warmup_requests: Vec<InferenceRequest>,
144    ) -> Result<ferrum_types::WarmupResult>;
145
146    /// Configure engine at runtime.
147    async fn reconfigure(&mut self, config: EngineConfig) -> Result<()>;
148
149    /// Get detailed diagnostics.
150    async fn diagnostics(&self) -> ferrum_types::DiagnosticsReport;
151
152    /// Export engine state for debugging.
153    async fn export_state(&self) -> Result<ferrum_types::EngineState>;
154
155    /// Import engine state for debugging/testing.
156    async fn import_state(&mut self, state: ferrum_types::EngineState) -> Result<()>;
157}
158
159/// Speculation configuration for speculative decoding.
160pub type SpeculationConfig = ferrum_types::SpeculationConfig;
161
162/// Hardware constraints alias.
163pub type HardwareConstraints = ferrum_types::HardwareConstraints;
164
165/// Request characteristics alias.
166pub type RequestCharacteristics = ferrum_types::RequestCharacteristics;
167
168/// Latency requirements alias.
169pub type LatencyRequirements = ferrum_types::LatencyRequirements;
ferrum_interfaces/engine.rs

ferrum_interfaces/
engine.rs
