oxillama 0.1.3

Pure Rust LLM inference engine — the sovereign alternative to llama.cpp (meta crate)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

//! Example: Streaming token-by-token output.
//!
//! Usage:
//! ```text
//! cargo run --example 03_streaming -- model.gguf "Tell me a story"
//! ```
//!
//! Tokens are printed as they are produced rather than buffering the full
//! response. If no model path is provided the example exits cleanly.

fn main() -> anyhow::Result<()> {
    let mut args = std::env::args().skip(1);

    let model_path = match args.next() {
        Some(p) => p,
        None => {
            eprintln!("Usage: 03_streaming <model.gguf> [prompt]");
            eprintln!("(No model path provided — exiting cleanly)");
            return Ok(());
        }
    };

    let prompt = args
        .next()
        .unwrap_or_else(|| "Tell me a story.".to_string());

    let config = oxillama_runtime::EngineConfig {
        model_path: model_path.clone(),
        ..Default::default()
    };

    let mut engine = oxillama_runtime::InferenceEngine::new(config);
    engine.load_model()?;

    println!("Streaming response for: {prompt}");
    println!("---");

    // The callback is invoked for each decoded token fragment.
    // Use `std::io::Write::flush` to ensure each piece is visible immediately.
    use std::io::Write as _;
    engine.generate(&prompt, 256, |tok| {
        print!("{tok}");
        let _ = std::io::stdout().flush();
    })?;
    println!("\n---");
    println!("Done.");

    Ok(())
}