ferrum_engine/
lib.rs

1//! # Ferrum Engine
2//!
3//! LLM inference engine orchestration layer with strong streaming support.
4//!
5//! ## Overview
6//!
7//! This crate provides the main inference engine implementation that orchestrates
8//! all the components from other ferrum crates:
9//!
10//! - Request admission and scheduling (ferrum-scheduler)
11//! - KV-cache allocation and management (ferrum-kv)
12//! - Tokenization and incremental decoding (ferrum-tokenizer)
13//! - Logits processing and sampling (ferrum-sampler)
14//! - Model execution and weight loading (ferrum-models)
15//! - Runtime and compute backends (ferrum-runtime)
16//!
17//! ## Design Principles
18//!
19//! - **Strong Streaming**: TTFT optimization and consistent inter-token latency
20//! - **Orchestration Layer**: Compose components rather than implement functionality
21//! - **Batch Processing**: Dynamic continuous batching for throughput
22//! - **Pipeline Optimization**: Prefill→decode loops with minimal overhead
23//! - **Registry Pattern**: Dynamic component registration and lookup
24//!
25//! ## Usage
26//!
27//! ### Using the Engine Builder (Recommended)
28//!
29//! ```rust,ignore
30//! use ferrum_engine::{EngineBuilder, EngineConfig};
31//!
32//! let config = EngineConfig::default();
33//! let engine = EngineBuilder::new(config)
34//!     .with_scheduler("fifo")
35//!     .with_sampler("greedy")
36//!     .build()
37//!     .await?;
38//! ```
39//!
40//! ### Registering Custom Components
41//!
42//! ```rust,ignore
43//! use ferrum_engine::{ComponentRegistry, global_registry};
44//!
45//! let registry = global_registry();
46//! registry.register_backend_factory("my_backend", Arc::new(MyBackendFactory));
47//! ```
48
49pub mod builder;
50pub mod continuous_engine;
51pub mod embedding_engine;
52pub(crate) mod layer_split;
53pub mod modality_stubs;
54pub mod parallel;
55pub mod pipeline;
56mod product_composition;
57pub mod recurrent_state;
58pub mod registry;
59pub(crate) mod resource_lifecycle;
60pub mod speculative;
61pub mod tensor_factory;
62pub mod transcription_engine;
63pub mod tts_engine;
64#[cfg(feature = "cuda")]
65pub mod vnext_determinism;
66
67// Re-exports of interfaces
68pub use ferrum_interfaces::engine::{EmbedEngine, LlmInferenceEngine, TranscribeEngine, TtsEngine};
69pub use ferrum_interfaces::{
70    IncrementalTokenizer, InferenceEngine as InferenceEngineInterface, KvCacheManager,
71    ModelExecutor, Sampler, SchedulerInterface as Scheduler, Tokenizer,
72};
73
74pub use ferrum_types::{
75    BatchId, EngineConfig, EngineStatus, FerrumError, InferenceRequest, InferenceResponse,
76    RequestId, Result, StreamChunk,
77};
78
79// Re-exports from implementation crates
80pub use ferrum_scheduler::BatchPlan;
81
82// Re-exports of engine implementation
83pub use continuous_engine::{ContinuousBatchEngine, SequenceState};
84
85// Re-exports of pipeline
86pub use pipeline::{
87    ChunkedPrefillConfig, ChunkedPrefillExecutor, ExecutionPhase, PipelineConfig, PipelineExecutor,
88};
89
90pub use recurrent_state::{
91    InMemoryRecurrentStateConfig, InMemoryRecurrentStateHandle, InMemoryRecurrentStateManager,
92};
93
94// Re-exports of builder
95pub use builder::{
96    create_engine, create_prepared_product_engine, create_product_engine, EngineBuilder,
97};
98
99// Re-exports of registry
100pub use registry::{
101    global_registry, set_global_registry, ComponentConfig, ComponentFactory, ComponentMetadata,
102    ComponentRegistry, ContinuousBatchSchedulerFactory, DefaultKvCacheFactory,
103    FifoSchedulerFactory, GreedySampler, GreedySamplerFactory, HuggingFaceTokenizerFactory,
104    LlmExecutorFactory, MultinomialSamplerFactory, PagedKvCacheFactory, PrioritySchedulerFactory,
105    StubExecutorFactory, StubTokenizer, StubTokenizerFactory,
106};
107
108// Back-compat alias for the renamed factory (PR A).
109#[allow(deprecated)]
110pub use registry::CandleExecutorFactory;
111
112// Re-exports of parallel module
113pub use parallel::{
114    global_device_manager, DeviceCapability, DeviceInfo, DeviceManager, LayerDistribution,
115    ParallelConfig, ParallelExecutor, ParallelExecutorFactory, ParallelismType,
116    TensorParallelConfig, TensorParallelGroup,
117};
118
119/// Create default inference engine with MVP configuration
120///
121/// This is a convenience function that uses the default registry.
122pub async fn create_default_engine(
123    config: EngineConfig,
124) -> Result<Box<dyn LlmInferenceEngine + Send + Sync>> {
125    create_engine(config).await
126}
127
128// ============================================================================
129// Integration Tests
130// ============================================================================
131
132#[cfg(test)]
133mod integration_tests {
134    use super::*;
135
136    fn test_config() -> EngineConfig {
137        let mut config = EngineConfig::default();
138        config.model.model_id = ferrum_types::ModelId::new("test-model");
139        config.backend.device = ferrum_types::Device::CPU;
140        config
141    }
142
143    #[tokio::test]
144    async fn test_create_engine_via_builder() {
145        let engine = EngineBuilder::new(test_config())
146            .with_tokenizer("stub")
147            .with_executor("stub")
148            .build()
149            .await;
150
151        assert!(engine.is_ok());
152    }
153
154    #[tokio::test]
155    async fn test_create_engine_convenience() {
156        let engine = create_default_engine(test_config()).await;
157
158        assert!(engine.is_ok());
159    }
160
161    #[test]
162    fn test_global_registry() {
163        let registry = global_registry();
164
165        // Should have default factories
166        assert!(registry.list_tokenizers().contains(&"stub".to_string()));
167        assert!(registry
168            .list_samplers()
169            .contains(&"multinomial".to_string()));
170    }
171
172    #[test]
173    fn test_custom_registry() {
174        let registry = ComponentRegistry::new();
175        assert!(registry.list_tokenizers().is_empty());
176
177        registry.register_defaults();
178        assert!(!registry.list_tokenizers().is_empty());
179    }
180}
ferrum_engine/lib.rs

ferrum_engine/
lib.rs
