vllm-cpp 0.0.1

Safe model inference, streaming, chat, and concurrent requests for vllm.cpp
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

mod common;

use std::io::{self, Write};

use vllm_cpp::{Engine, SamplingParams, StreamControl};

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let model = common::resolve_model("stream")?;
    let engine = Engine::load(model)?;
    engine.complete_stream(
        "Write one short sentence about Rust:",
        &SamplingParams::greedy().max_tokens(32),
        |event| {
            print!("{}", event.delta);
            io::stdout().flush().expect("flush stdout");
            StreamControl::Continue
        },
    )?;
    println!();
    Ok(())
}