aprender-serve 0.31.2

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

//! Test all digit pair combinations
use realizar::gguf::{MappedGGUFModel, OwnedQuantizedModel};

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let path = "/home/noah/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B-Instruct-GGUF/snapshots/198f08841147e5196a6a69bd0053690fb1fd3857/qwen2-0_5b-instruct-q4_0.gguf";
    let mapped = MappedGGUFModel::from_path(path)?;
    let model = OwnedQuantizedModel::from_mapped(&mapped)?;
    let vocab = mapped.model.vocabulary().expect("vocab");

    println!("=== Digit Pair Test ===\n");

    // Token IDs for digits 0-9
    let digit_tokens: Vec<(char, u32)> = ('0'..='9')
        .map(|c| {
            let tok_id = vocab
                .iter()
                .enumerate()
                .find(|(_, s)| s.as_str() == c.to_string())
                .map_or(0, |(i, _)| i as u32);
            (c, tok_id)
        })
        .collect();

    println!("Digit token IDs:");
    for (c, tok) in &digit_tokens {
        println!("  '{}' = {}", c, tok);
    }

    println!("\n=== Testing All Digit Pairs ===");
    println!("{:>6} -> {:>15} {:>10}", "Input", "Top Token", "Logit");

    for (c1, t1) in &digit_tokens {
        for (c2, t2) in &digit_tokens {
            let tokens = vec![*t1, *t2];
            let logits = model.forward(&tokens)?;

            let mut indexed: Vec<_> = logits.iter().enumerate().collect();
            indexed.sort_by(|a, b| b.1.partial_cmp(a.1).unwrap());

            let top_tok = indexed[0].0;
            let top_s = vocab.get(top_tok).map_or("?", |s| s.as_str());

            // Highlight if "!" is top
            let marker = if top_tok == 0 { " <-- BUG!" } else { "" };

            println!(
                "  '{}{}' -> {:>15} {:>10.4}{}",
                c1,
                c2,
                format!("{} ({:?})", top_tok, top_s),
                indexed[0].1,
                marker
            );
        }
    }

    // Also test with "+"
    println!("\n=== Testing Digit + '+' ===");
    let plus_tok = vocab
        .iter()
        .enumerate()
        .find(|(_, s)| s.as_str() == "+")
        .map_or(0, |(i, _)| i as u32);
    println!("'+' token ID: {}", plus_tok);

    for (c, t) in &digit_tokens {
        let tokens = vec![*t, plus_tok];
        let logits = model.forward(&tokens)?;

        let mut indexed: Vec<_> = logits.iter().enumerate().collect();
        indexed.sort_by(|a, b| b.1.partial_cmp(a.1).unwrap());

        let top_tok = indexed[0].0;
        let top_s = vocab.get(top_tok).map_or("?", |s| s.as_str());

        let marker = if top_tok == 0 { " <-- BUG!" } else { "" };
        println!(
            "  '{}+' -> {:>15} {:>10.4}{}",
            c,
            format!("{} ({:?})", top_tok, top_s),
            indexed[0].1,
            marker
        );
    }

    // Test "+" + digit
    println!("\n=== Testing '+' + Digit ===");
    for (c, t) in &digit_tokens {
        let tokens = vec![plus_tok, *t];
        let logits = model.forward(&tokens)?;

        let mut indexed: Vec<_> = logits.iter().enumerate().collect();
        indexed.sort_by(|a, b| b.1.partial_cmp(a.1).unwrap());

        let top_tok = indexed[0].0;
        let top_s = vocab.get(top_tok).map_or("?", |s| s.as_str());

        let marker = if top_tok == 0 { " <-- BUG!" } else { "" };
        println!(
            "  '+{}' -> {:>15} {:>10.4}{}",
            c,
            format!("{} ({:?})", top_tok, top_s),
            indexed[0].1,
            marker
        );
    }

    Ok(())
}