mullama 0.3.0

Comprehensive Rust bindings for llama.cpp with memory-safe API and advanced features
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

//! Simple text generation example
//!
//! Shows the minimal code to load a model and generate text.
//!
//! Usage:
//!   cargo run --example simple -- path/to/model.gguf "Your prompt here"

use mullama::{Context, ContextParams, Model, SamplerParams};
use std::sync::Arc;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 3 {
        eprintln!("Usage: {} <model.gguf> <prompt>", args[0]);
        eprintln!(
            "Example: {} ./models/llama-3.2-1b-q4_k_m.gguf \"The meaning of life is\"",
            args[0]
        );
        std::process::exit(1);
    }
    let model_path = &args[1];
    let prompt = &args[2];

    // Initialize backend
    mullama::backend_init();

    // Load model
    println!("Loading model: {}", model_path);
    let model = Arc::new(Model::load(model_path)?);
    println!("  Parameters: {}M", model.n_params() / 1_000_000);
    println!("  Vocab size: {}", model.n_vocab());

    // Create context
    let ctx_params = ContextParams {
        n_ctx: 2048,
        n_batch: 512,
        n_threads: (num_cpus::get() / 2).max(1) as i32,
        ..ContextParams::default()
    };
    let mut context = Context::new(model.clone(), ctx_params)?;

    // Configure sampling
    let sampler_params = SamplerParams {
        temperature: 0.7,
        top_k: 40,
        top_p: 0.9,
        ..SamplerParams::default()
    };
    let mut sampler = sampler_params.build_chain(model.clone())?;

    // Tokenize and decode prompt
    let tokens = model.tokenize(prompt, true, false)?;
    println!("\nPrompt: {} ({} tokens)", prompt, tokens.len());
    context.decode(&tokens)?;

    // Generate
    print!("\nGeneration: ");
    let mut generated_tokens = 0;
    let start = std::time::Instant::now();

    for _ in 0..256 {
        let token = sampler.sample(&mut context, -1);

        if model.vocab_is_eog(token) {
            break;
        }

        if let Ok(text) = model.token_to_str(token, 0, false) {
            print!("{}", text);
        }

        sampler.accept(token);
        context.decode_single(token)?;
        generated_tokens += 1;
    }

    let elapsed = start.elapsed();
    let tok_per_sec = generated_tokens as f64 / elapsed.as_secs_f64();
    println!(
        "\n\n--- {} tokens in {:.1}s ({:.1} tok/s) ---",
        generated_tokens,
        elapsed.as_secs_f64(),
        tok_per_sec
    );

    mullama::backend_free();
    Ok(())
}