vllm-cpp 0.0.1

Safe model inference, streaming, chat, and concurrent requests for vllm.cpp
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

[package]
name = "vllm-cpp"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
description = "Safe model inference, streaming, chat, and concurrent requests for vllm.cpp"
readme = "README.md"
documentation = "https://docs.rs/vllm-cpp"
keywords = ["llm", "inference", "ffi", "vllm", "machine-learning"]
categories = ["api-bindings", "science"]

[features]
default = ["bundled"]
bundled = ["vllm-cpp-sys/bundled"]
system = ["vllm-cpp-sys/system"]
dynamic-link = ["vllm-cpp-sys/dynamic-link"]
cuda = ["vllm-cpp-sys/cuda"]
cuda-cutlass = ["cuda", "vllm-cpp-sys/cuda-cutlass"]
triton-aot = ["cuda", "vllm-cpp-sys/triton-aot"]
vulkan = ["vllm-cpp-sys/vulkan"]
metal = ["vllm-cpp-sys/metal"]
mlx = ["metal", "vllm-cpp-sys/mlx"]
serde = []

[package.metadata.docs.rs]
features = ["serde"]
targets = ["x86_64-unknown-linux-gnu"]

[dependencies]
hf-hub = { version = "0.5.0", default-features = false, features = ["ureq"] }
serde_json = "1.0.149"
vllm-cpp-sys = { workspace = true, default-features = false }

[dev-dependencies]
clap = { version = "=4.6.1", features = ["derive"] }
static_assertions = "1.1.0"