vllm-cpp 0.0.1

Safe model inference, streaming, chat, and concurrent requests for vllm.cpp
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14

mod common;

use vllm_cpp::{Engine, SamplingParams};

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let model = common::resolve_model("complete")?;
    let engine = Engine::load(model)?;
    let completion = engine.complete(
        "The capital of France is",
        &SamplingParams::greedy().max_tokens(16),
    )?;
    println!("{}", completion.text);
    Ok(())
}