oxillama 0.1.3

Pure Rust LLM inference engine — the sovereign alternative to llama.cpp (meta crate)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

//! Example: Load and apply a LoRA adapter.
//!
//! Usage:
//! ```text
//! cargo run --example 04_lora -- base.gguf adapter.gguf "Hello"
//! ```
//!
//! If no paths are provided the example exits cleanly.

fn main() -> anyhow::Result<()> {
    let mut args = std::env::args().skip(1);

    let base_path = match args.next() {
        Some(p) => p,
        None => {
            eprintln!("Usage: 04_lora <base.gguf> <adapter.gguf> [prompt]");
            eprintln!("(No model path provided — exiting cleanly)");
            return Ok(());
        }
    };

    let adapter_path = match args.next() {
        Some(p) => p,
        None => {
            eprintln!("Usage: 04_lora <base.gguf> <adapter.gguf> [prompt]");
            eprintln!("(No adapter path provided — exiting cleanly)");
            return Ok(());
        }
    };

    let prompt = args.next().unwrap_or_else(|| "Hello!".to_string());

    // 1. Load the base model.
    let config = oxillama_runtime::EngineConfig {
        model_path: base_path,
        ..Default::default()
    };
    let mut engine = oxillama_runtime::InferenceEngine::new(config);
    engine.load_model()?;
    println!("Base model loaded.");

    // 2. Apply the LoRA adapter on top.
    oxillama_runtime::apply_lora(&mut engine, &adapter_path)?;
    println!("LoRA adapter applied: {adapter_path}");

    // 3. Generate with the patched model.
    println!("Generating…");
    engine.generate(&prompt, 128, |tok| print!("{tok}"))?;
    println!();

    Ok(())
}