realizar 0.3.2

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

//! Test coherence with a meaningful prompt
use realizar::gguf::{MappedGGUFModel, OwnedQuantizedModel};
use std::time::Instant;

fn main() {
    println!("Loading model...");
    let mapped =
        MappedGGUFModel::from_path("/home/noah/src/aprender/tinyllama-1.1b-chat-v1.0.Q4_0.gguf")
            .unwrap();
    let model = OwnedQuantizedModel::from_mapped(&mapped).unwrap();
    let vocab = mapped.model.vocabulary().unwrap();

    // Test prompt
    let prompt = "The capital of France is";
    let tokens = mapped.model.encode(prompt).unwrap();
    println!("Prompt: '{}'", prompt);
    println!("Tokens: {:?}", tokens);

    // Show what tokens decode to
    print!("Decoded: ");
    for &t in &tokens {
        let tok = if (t as usize) < vocab.len() {
            &vocab[t as usize]
        } else {
            "?"
        };
        print!("{}", tok.replace("▁", " "));
    }
    println!();

    // Single forward pass
    println!("\nRunning forward pass...");
    let start = Instant::now();
    let logits = model.forward(&tokens).unwrap();
    println!("Forward pass: {:.3}s", start.elapsed().as_secs_f32());

    // Find top 10 predictions
    let mut indexed: Vec<(usize, f32)> = logits.iter().cloned().enumerate().collect();
    indexed.sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));

    println!("\nTop 10 predictions for next token:");
    for (rank, (idx, logit)) in indexed.iter().take(10).enumerate() {
        let tok = if *idx < vocab.len() {
            &vocab[*idx]
        } else {
            "?"
        };
        println!(
            "  {}: {} '{}' = {:.4}",
            rank + 1,
            idx,
            tok.replace("▁", " "),
            logit
        );
    }

    // Check where "Paris" ranks
    if let Some(paris_idx) = vocab.iter().position(|t| t == "▁Paris" || t == "Paris") {
        let paris_rank = indexed
            .iter()
            .position(|(idx, _)| *idx == paris_idx)
            .unwrap_or(vocab.len());
        println!(
            "\n'Paris' (token {}) rank: {}/{}",
            paris_idx,
            paris_rank + 1,
            vocab.len()
        );
        println!("'Paris' logit: {:.4}", logits[paris_idx]);
    }

    // Generate 3 tokens
    println!("\nGenerating 3 tokens (greedy):");
    let mut all_tokens = tokens;
    for i in 0..3 {
        let start = Instant::now();
        let logits = model.forward(&all_tokens).unwrap();
        let (best_idx, _) = logits
            .iter()
            .enumerate()
            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
            .unwrap();

        let tok = if best_idx < vocab.len() {
            &vocab[best_idx]
        } else {
            "?"
        };
        println!(
            "  Token {}: {} '{}' ({:.2}s)",
            i + 1,
            best_idx,
            tok.replace("▁", " "),
            start.elapsed().as_secs_f32()
        );
        all_tokens.push(best_idx as u32);
    }

    // Show full generated text
    print!("\nFull output: ");
    for &t in &all_tokens {
        let tok = if (t as usize) < vocab.len() {
            &vocab[t as usize]
        } else {
            "?"
        };
        print!("{}", tok.replace("▁", " "));
    }
    println!();
}