realizar 0.8.5

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

//! Trace forward pass - check if single token forward produces sensible output
use realizar::gguf::{MappedGGUFModel, OwnedQuantizedKVCache, OwnedQuantizedModel};

fn stats(name: &str, v: &[f32]) {
    let sum: f32 = v.iter().sum();
    let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
    let min = v.iter().copied().fold(f32::INFINITY, f32::min);
    let max = v.iter().copied().fold(f32::NEG_INFINITY, f32::max);
    eprintln!(
        "{}: sum={:.4}, norm={:.4}, min={:.4}, max={:.4}",
        name, sum, norm, min, max
    );
}

fn main() {
    let path = "../aprender/models/qwen2.5-coder-0.5b-instruct-q4_k_m.gguf";
    let mapped = MappedGGUFModel::from_path(path).expect("load");
    let model = OwnedQuantizedModel::from_mapped(&mapped).expect("model");
    let vocab = mapped.model.vocabulary().expect("vocab");

    // Test: After seeing <|im_start|>, model should predict "system" or "user"
    let token_id = 151644u32; // <|im_start|>
    let position = 0;

    eprintln!(
        "=== Testing single token: {} at position {} ===",
        token_id, position
    );
    eprintln!(
        "Token string: {}",
        vocab.get(token_id as usize).unwrap_or(&"?".to_string())
    );

    let head_dim = model.config().hidden_dim / model.config().num_heads;
    let kv_dim = model.config().num_kv_heads * head_dim;
    let mut cache = OwnedQuantizedKVCache::new(model.config().num_layers, kv_dim, 8);

    let logits = model
        .forward_single_with_cache(token_id, &mut cache, position)
        .expect("forward");

    stats("Logits", &logits);

    // Top 10
    let mut indexed: Vec<(usize, f32)> = logits.iter().copied().enumerate().collect();
    indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));

    eprintln!("\nTop 10 after seeing '<|im_start|>':");
    for (rank, (idx, score)) in indexed.iter().take(10).enumerate() {
        let tok_str = vocab
            .get(*idx)
            .map_or("?".to_string(), |s| s.escape_debug().to_string());
        eprintln!(
            "{:2}. {:6} score={:.4}  '{}'",
            rank + 1,
            idx,
            score,
            tok_str
        );
    }

    // Expected: "system", "user" should be top predictions
    eprintln!("\nExpected tokens (should be ranked high):");
    for check in ["system", "user", "assistant", "Ċ"] {
        if let Some(idx) = vocab.iter().position(|t| t == check) {
            let score = logits[idx];
            let rank = indexed
                .iter()
                .position(|(i, _)| *i == idx)
                .map_or(0, |r| r + 1);
            eprintln!(
                "  '{}' (id={}) score={:.4} rank={}",
                check, idx, score, rank
            );
        }
    }

    // Now test two tokens: <|im_start|> followed by "system"
    eprintln!("\n=== Testing two tokens: <|im_start|> system ===");
    cache = OwnedQuantizedKVCache::new(model.config().num_layers, kv_dim, 8);

    // First token
    let _ = model
        .forward_single_with_cache(151644, &mut cache, 0)
        .expect("forward1");
    // Second token: "system" = 8948
    let logits2 = model
        .forward_single_with_cache(8948, &mut cache, 1)
        .expect("forward2");

    let mut indexed2: Vec<(usize, f32)> = logits2.iter().copied().enumerate().collect();
    indexed2.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));

    eprintln!("\nTop 10 after '<|im_start|>system' (expecting newline Ċ):");
    for (rank, (idx, score)) in indexed2.iter().take(10).enumerate() {
        let tok_str = vocab
            .get(*idx)
            .map_or("?".to_string(), |s| s.escape_debug().to_string());
        eprintln!(
            "{:2}. {:6} score={:.4}  '{}'",
            rank + 1,
            idx,
            score,
            tok_str
        );
    }

    // Check newline rank
    if let Some(idx) = vocab.iter().position(|t| t == "Ċ") {
        let score = logits2[idx];
        let rank = indexed2
            .iter()
            .position(|(i, _)| *i == idx)
            .map_or(0, |r| r + 1);
        eprintln!(
            "\n  'Ċ' (newline, id={}) score={:.4} rank={}",
            idx, score, rank
        );
    }
}