hf2q 0.1.9

Pure Rust CLI for converting HuggingFace models to hardware-optimized formats and serving them over an OpenAI-compatible API on Apple Silicon
1
2
3
4
5
6
7
8
9
10
11
12

kind = "hf2q.config"
schema_version = 2
package = "hf2q"

[convert]
quant = "q4_k_m"

[serve]
host = "127.0.0.1"
port = 8081
scheduler = "inflight_batched"
max_slots = 1