runtime/
lib.rs

1//! UniLLM Runtime
2//!
3//! High-performance inference runtime for large language models.
4//!
5//! This crate provides a clean, solid abstraction system for LLM inference
6//! with support for multiple model architectures and deployment targets.
7
8// === CORE ABSTRACTION LAYERS ===
9
10/// Unified tensor operations and device management
11pub mod tensor_core;
12
13/// Model trait and configuration system
14pub mod model_core;
15
16/// Weight loading from various formats
17pub mod weight_loader_core;
18
19// === MODEL IMPLEMENTATIONS ===
20
21/// Clean model implementations using solid abstractions
22pub mod models_v2;
23
24// === INFERENCE PIPELINE ===
25
26/// Tokenization utilities
27pub mod tokenizer;
28
29/// Basic inference implementation
30pub mod inference;
31
32/// Sampling and decoding
33pub mod sampler;
34
35/// KV cache for efficient autoregressive generation
36pub mod kv_cache;
37
38/// Precomputed static tensors for performance (RoPE, causal masks)
39pub mod precompute;
40
41/// Native SIMD kernels for quantized inference
42#[cfg(feature = "simd")]
43pub mod simd;
44
45// === UTILITIES ===
46
47/// Type definitions
48pub mod types;
49
50/// Simple observability
51pub mod simple_observability;
52
53/// Ollama registry client
54pub mod ollama;
55
56/// Benchmark comparison module
57pub mod benchmark;
58
59// === RE-EXPORTS ===
60
61pub use tensor_core::{Tensor, Device, DataType};
62pub use model_core::{Model, ModelInputs, ModelOutputs, GenerationConfig, MemoryRequirements, ModelWeights};
63pub use weight_loader_core::{WeightLoader};
64pub use kv_cache::{KVCache, LayerKVCache};
65pub use precompute::{RoPECache, CausalMaskCache, SlidingWindowMaskCache};
66
67#[cfg(feature = "simd")]
68pub use simd::{get_simd_backend, init_simd_backend, cpu_features, SimdBackend};
69
70/// Main runtime instance
71pub struct Runtime {
72    _placeholder: (),
73}
74
75impl Runtime {
76    /// Create a new runtime instance
77    pub fn new() -> Self {
78        Self {
79            _placeholder: (),
80        }
81    }
82}
83
84impl Default for Runtime {
85    fn default() -> Self {
86        Self::new()
87    }
88}
runtime/lib.rs

runtime/
lib.rs
