vllm-cpp 0.0.1

Safe model inference, streaming, chat, and concurrent requests for vllm.cpp
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

mod common;

use vllm_cpp::{Engine, SamplingParams, StructuredOutput};

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let model = common::resolve_model("structured")?;
    let engine = Engine::load(model)?;
    let schema = r#"{
        "type": "object",
        "properties": {
            "location": { "type": "string" },
            "temperature_celsius": { "type": "number" },
            "condition": { "type": "string" }
        },
        "required": ["location", "temperature_celsius", "condition"],
        "additionalProperties": false
    }"#;
    let completion = engine.complete(
        "Extract the weather report as JSON: Paris is sunny and 22 degrees Celsius.",
        &SamplingParams::greedy()
            .max_tokens(64)
            .structured_output(StructuredOutput::JsonSchema(schema.to_owned())),
    )?;
    println!("{}", completion.text);
    Ok(())
}