quaynor 2.2.0

Lightweight local AI inference engine: load GGUF models and chat on-device with streaming, tool calling, embeddings, and reranking
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

//! Minimal streaming chat example.
//!
//! Run with:
//!   cargo run --example chat -- /path/to/model.gguf
//!   cargo run --example chat -- hf://bartowski/Qwen_Qwen3-0.6B-GGUF/Qwen_Qwen3-0.6B-Q4_K_M.gguf

use quaynor::chat::ChatBuilder;
use quaynor::llm::get_model;
use std::io::Write;
use std::sync::Arc;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let model_path = std::env::args()
        .nth(1)
        .ok_or("usage: chat <model path, URL, or hf://owner/repo/file.gguf>")?;

    let model = Arc::new(get_model(&model_path, true, None)?);

    let chat = ChatBuilder::new(model)
        .with_context_size(4096)
        .with_system_prompt(Some("You are a helpful assistant."))
        .build();

    let mut stream = chat.ask("Is a zebra black or white?");
    while let Some(token) = stream.next_token() {
        print!("{token}");
        std::io::stdout().flush()?;
    }
    println!();
    Ok(())
}