ferrox-server 0.21.0

OpenAI-compatible HTTP server for the Ferrox inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

[package]
name = "ferrox-server"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
description = "OpenAI-compatible HTTP server for the Ferrox inference engine"
readme = "README.md"
keywords = ["llm", "openai", "server", "inference", "ferrox"]
categories = ["web-programming::http-server", "science"]

# The server is a library with a two-line binary in front of it, not a
# binary with everything inside `fn main`. ferrox-cli's optional `serve`
# feature links this lib, so `ferrox serve` and `ferrox-server` run the
# same code from the same `ServerArgs`.
[lib]
name = "ferrox_server"
path = "src/lib.rs"

[[bin]]
name = "ferrox-server"
path = "src/main.rs"

[dependencies]
ferrox-api = { workspace = true }
ferrox-core = { workspace = true }
# For the deterministic receipt id in `policy::outbox`: a retried stop
# must derive the same id, or it writes a second billing event for work
# that happened once.
sha2 = { workspace = true }
# The ported FreeToken serving policy: the reasoning / tool-call
# parsers this server's chat path uses, and the withhold rule that
# `stop::StopMatcher` shares with them.
ferrox-models = { workspace = true, features = ["hub"] }
ferrox-gguf = { workspace = true }
ferrox-safetensors = { workspace = true }
axum = { workspace = true }
tokio = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
clap = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
anyhow = { workspace = true }
thiserror = { workspace = true }
ferrox-cuda = { workspace = true }
ferrox-metal = { workspace = true }
ureq = { workspace = true }
futures-util = "0.3"
# `tls-rustls-no-provider`, not `tls-rustls`: the latter is defined as
# `["tls-rustls-no-provider", "rustls/aws-lc-rs"]`, and that one feature
# was the ONLY thing pulling aws-lc-rs (and its C/assembly aws-lc-sys)
# into the tree. ureq already asks rustls for `ring` with
# default-features off, so cargo's feature unification was forcing
# aws-lc-rs onto ureq's rustls too. Selecting the provider here instead
# means one crypto backend, chosen deliberately, shared by both.
#
# The cost of `no-provider` is that rustls has no default provider at
# runtime, so `main` installs one before serving TLS. That is a real
# obligation -- forget it and TLS fails at accept time, not at compile
# time -- so `install_ring_crypto_provider` is called unconditionally at
# startup and has a test.
axum-server = { version = "0.8.0", features = ["tls-rustls-no-provider"] }
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12", "logging"] }
tower-http = { version = "0.7.0", features = ["cors"] }

[dev-dependencies]
tower = { version = "0.5", features = ["util"] }
http-body-util = "0.1"
ferrox-moe = { workspace = true }
base64 = "0.23.0"

[features]
default = []
# Real NVRTC-compiled CUDA kernel dispatch for GPU-placed routed
# experts, propagated down through
# ferrox-models/ferrox-moe/ferrox-core. Builds cleanly without a CUDA
# toolkit/GPU present (see ferrox-cuda's Cargo.toml comment); without
# this feature, FERROX_GPU_VRAM_BUDGET_BYTES still works but every
# expert always runs on CPU regardless of placement.
cuda = ["ferrox-models/cuda"]
# Apple Silicon Metal kernels through the same WeightMatrix::apply_gpu /
# ExpertPlacement::GpuDevice path. macOS only.
metal = ["ferrox-models/metal"]