furnace-server 0.14.0

Fast, unified, Rust-native, Axum-compatible inference endpoints and nervous system for tensor models and LLMs: KServe / Triton and OpenAI APIs from one pure-Rust server
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

# Example configuration: `furnace --config furnace.example.toml`
# (or set FURNACE_CONFIG). Every key is optional; options given on the command line override
# the file. Unknown keys are an error, so typos don't go unnoticed.

[server]
host = "127.0.0.1"
port = 3000
max_concurrent_requests = 100   # beyond this, requests get 503
max_body_mb = 100
# request_timeout_ms = 5000     # beyond this, requests get 504 (default: no timeout)
log_level = "info"              # error, warn, info, debug or trace
# threads = 8                   # inference threads (default: one per physical core)
# grpc_port = 8001              # also serve gRPC (KServe / Triton protocol); needs the `grpc` feature
# admin_token = "..."           # enable POST /admin/models/{model}/reload; prefer $FURNACE_ADMIN_TOKEN
# api_keys = ["..."]            # require one of these keys; prefer $FURNACE_API_KEYS
# rate_limit = 50               # requests/s per client (API key, or IP address), beyond which 429
# rate_limit_burst = 100        # requests a client may send at once (default: rate_limit)
# tls_cert = "/etc/furnace/tls.crt"      # HTTPS and gRPC over TLS (`tls` feature); reloaded on change
# tls_key = "/etc/furnace/tls.key"       # ECDSA P-256 / P-384 or Ed25519
# tls_client_ca = "/etc/furnace/ca.crt"  # mutual TLS: only clients with a certificate from this CA

[model]
name = "resnet18"               # built-in model compiled from models/<name>.onnx
# name = ["resnet18", "mnist"]  # or several; the first is the one /predict serves
# path = "examples/basic_mnist/model"   # or a file-based Burn model instead of `name`
# weights = "/srv/models/resnet18.bpk"  # default: <name>.bpk next to the binary
backend = "cpu"                 # cpu, or wgpu / metal / cuda when built with that feature

[batching]
max_batch_size = 8              # 1 turns dynamic batching off
timeout_us = 2000
workers = 4