vllm-cpp 0.0.1

Safe model inference, streaming, chat, and concurrent requests for vllm.cpp
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

use std::process::ExitCode;

use vllm_cpp::HuggingFaceModel;

const REPOSITORY: &str = "Qwen/Qwen3-0.6B";
const REVISION: &str = "c1899de289a04d12100db370d81485cdf75e47ca";

fn main() -> ExitCode {
    match HuggingFaceModel::safetensors(REPOSITORY)
        .revision(REVISION)
        .progress(true)
        .resolve()
    {
        Ok(path) => {
            println!("{}", path.display());
            ExitCode::SUCCESS
        }
        Err(error) => {
            eprintln!("could not set up test model: {error}");
            ExitCode::FAILURE
        }
    }
}