ferrum_interfaces/
engine.rs

1//! Inference engine interface with streaming and batch support
2//!
3//! This module provides the top-level inference engine interface that
4//! orchestrates all other components: tokenizer, model executor, scheduler,
5//! and sampler.
6
7use async_trait::async_trait;
8use ferrum_types::{EngineConfig, InferenceRequest, InferenceResponse, Result, StreamChunk};
9use futures::Stream;
10use std::pin::Pin;
11
12/// Core inference engine trait
13#[async_trait]
14pub trait InferenceEngine: Send + Sync {
15    /// Execute single inference request
16    async fn infer(&self, request: InferenceRequest) -> Result<InferenceResponse>;
17
18    /// Execute streaming inference request
19    async fn infer_stream(
20        &self,
21        request: InferenceRequest,
22    ) -> Result<Pin<Box<dyn Stream<Item = Result<StreamChunk>> + Send>>>;
23
24    /// Get current engine status
25    async fn status(&self) -> ferrum_types::EngineStatus;
26
27    /// Shutdown engine gracefully
28    async fn shutdown(&self) -> Result<()>;
29
30    /// Get engine configuration
31    fn config(&self) -> &EngineConfig;
32
33    /// Get engine metrics
34    fn metrics(&self) -> ferrum_types::EngineMetrics;
35
36    /// Health check
37    async fn health_check(&self) -> ferrum_types::HealthStatus;
38}
39
40/// Advanced engine capabilities
41#[async_trait]
42pub trait AdvancedInferenceEngine: InferenceEngine {
43    /// Execute batch inference
44    async fn infer_batch(
45        &self,
46        requests: Vec<InferenceRequest>,
47    ) -> Result<Vec<Result<InferenceResponse>>>;
48
49    /// Execute speculative inference
50    async fn infer_speculative(
51        &self,
52        request: InferenceRequest,
53        speculation_config: ferrum_types::SpeculationConfig,
54    ) -> Result<InferenceResponse>;
55
56    /// Warm up engine with sample requests
57    async fn warmup(
58        &mut self,
59        warmup_requests: Vec<InferenceRequest>,
60    ) -> Result<ferrum_types::WarmupResult>;
61
62    /// Configure engine at runtime
63    async fn reconfigure(&mut self, config: EngineConfig) -> Result<()>;
64
65    /// Get detailed diagnostics
66    async fn diagnostics(&self) -> ferrum_types::DiagnosticsReport;
67
68    /// Export engine state for debugging
69    async fn export_state(&self) -> Result<ferrum_types::EngineState>;
70
71    /// Import engine state for debugging/testing
72    async fn import_state(&mut self, state: ferrum_types::EngineState) -> Result<()>;
73}
74
75/// Speculation configuration for speculative decoding
76pub type SpeculationConfig = ferrum_types::SpeculationConfig;
77
78/// Hardware constraints alias
79pub type HardwareConstraints = ferrum_types::HardwareConstraints;
80
81/// Request characteristics alias
82pub type RequestCharacteristics = ferrum_types::RequestCharacteristics;
83
84/// Latency requirements alias
85pub type LatencyRequirements = ferrum_types::LatencyRequirements;
ferrum_interfaces/engine.rs

ferrum_interfaces/
engine.rs
