ferrum-engine 0.8.1

Model orchestration engine for Ferrum LLM inference
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

[package]
name = "ferrum-engine"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
description = "Model orchestration engine for Ferrum LLM inference"
readme = "../../README.md"

[dependencies]
# Core dependencies - New architecture
ferrum-types = { workspace = true }
ferrum-interfaces = { workspace = true }

# Implementation crates
ferrum-scheduler = { workspace = true }
ferrum-tokenizer = { workspace = true }
ferrum-sampler = { workspace = true }
ferrum-bench-core = { workspace = true }
ferrum-kv = { workspace = true }
ferrum-kernels = { workspace = true }
ferrum-quantization = { workspace = true }
ferrum-models = { workspace = true }

# Async
async-trait = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] }
tokio-stream = { workspace = true }
futures = { workspace = true }
parking_lot = { workspace = true }

# Serialization
serde = { workspace = true }
serde_json = { workspace = true }
sha2 = "0.10"

# Error handling
anyhow = { workspace = true }
thiserror = { workspace = true }

# Logging
tracing = { workspace = true }

# Utils
uuid = { workspace = true }
chrono = { workspace = true }

# Metrics
metrics = { workspace = true }

# ML Framework - Candle
candle-core = { workspace = true }
candle-nn = { workspace = true }
candle-transformers = { workspace = true }
candle-flash-attn = { workspace = true, optional = true }
candle-metal-kernels = { workspace = true, optional = true }
hf-hub = { workspace = true }
tokenizers = { workspace = true }
safetensors = { workspace = true }

# Random support
rand = { workspace = true }
rand_distr = { workspace = true }
half = { workspace = true }
ndarray = { version = "0.16.1", optional = true }
once_cell = { workspace = true }
smallvec = "1.11"
crossbeam-channel = "0.5"
crossbeam-utils = "0.8"
pin-project = "1.1"
bytesize = "1.3"

# HTTP client for manual downloads
reqwest = { workspace = true }

[dev-dependencies]
tokio-test = "0.4"
criterion = "0.5"
ferrum-testkit = { workspace = true }
# Tiny-model construction for the full-stack tiny_stack suite. test-support is
# a dev-only feature of ferrum-models; this never reaches release builds.
ferrum-models = { workspace = true, features = ["test-support"] }
futures = { workspace = true }

[[bench]]
name = "engine_bench"
harness = false

[features]
default = []
accelerate = ["candle-core/accelerate", "candle-nn/accelerate", "ferrum-models/accelerate"]
cuda = [
    "ferrum-models/cuda",
    "ferrum-kernels/cuda",
    "ferrum-quantization/cuda",
]
candle-cuda-compat = [
    "cuda",
    "candle-core/cuda",
    "candle-nn/cuda",
    "ferrum-models/candle-cuda-compat",
    "ferrum-kernels/candle-cuda-compat",
]
marlin = ["cuda"]  # kept for backward compat, cuda already enables marlin
tensor-parallel = ["cuda"]  # backward compat alias
# Forward the triton-kernels feature so cuda_decode launches use triton-rs
# PTX in place of hand-written .cu PTX. Requires `cuda` (always co-enabled).
triton-kernels = ["cuda", "ferrum-kernels/triton-kernels"]
metal = ["candle-core/metal", "dep:candle-metal-kernels", "dep:metal", "dep:mpsgraph", "dep:objc2", "dep:objc2-foundation", "dep:bytemuck", "dep:ndarray", "ferrum-models/metal", "ferrum-kernels/metal"]

# Metal dependencies for Apple GPU support (only on macOS/iOS)
[target.'cfg(target_os = "macos")'.dependencies]
metal = { version = "0.27.0", optional = true }
mpsgraph = { version = "0.2.0", optional = true }
objc2 = { version = "0.6.1", optional = true }
objc2-foundation = { version = "0.3.1", optional = true }
bytemuck = { version = "1.23.0", optional = true }

[target.'cfg(target_os = "ios")'.dependencies]
metal = { version = "0.27.0", optional = true }
mpsgraph = { version = "0.2.0", optional = true }
objc2 = { version = "0.6.1", optional = true }
objc2-foundation = { version = "0.3.1", optional = true }
bytemuck = { version = "1.23.0", optional = true }