llama-rs 0.17.0

A high-performance Rust implementation of llama.cpp - LLM inference engine with full GGUF support
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

//! Test if the issue is specifically with position 1.

use llama_rs::backend::Backend;
use llama_rs::backend::cpu::CpuBackend;
use llama_rs::model::{InferenceContext, Model, ModelLoader};
use std::sync::Arc;

fn test_position(
    model: &dyn Model,
    backend: Arc<dyn Backend>,
    config: &llama_rs::model::ModelConfig,
    tokens: &[u32],
    desc: &str,
) {
    let mut ctx = InferenceContext::new(config, backend);
    let logits = model.forward(tokens, &mut ctx).expect("Forward failed");

    let logits_data = logits.as_f32().unwrap();
    let logit_17 = logits_data[17];
    let mut indexed: Vec<(usize, f32)> = logits_data.iter().cloned().enumerate().collect();
    indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
    let rank_17 = indexed.iter().position(|(idx, _)| *idx == 17).unwrap() + 1;

    println!("{:<40} | {:>8.4} | {:>6}", desc, logit_17, rank_17);
}

fn main() {
    let model_path = "/home/joseph/Models/qwen2.5-0.5b-instruct-q4_k_m.gguf";

    eprintln!("Loading model...");
    let loader = ModelLoader::load(model_path).expect("Failed to load model");
    let config = loader.config().clone();
    let model = loader.build_model().expect("Failed to build model");
    let backend: Arc<dyn Backend> = Arc::new(CpuBackend::new());

    eprintln!("Running tests...");
    eprintln!();

    println!("Token sequence                           | Logit 17 | Rank 17");
    println!("-----------------------------------------+----------+--------");

    // Test single "=" at position 0
    test_position(&model, backend.clone(), &config, &[28], "'=' at pos 0");

    // Test "1" at pos 0, "=" at pos 1
    test_position(
        &model,
        backend.clone(),
        &config,
        &[16, 28],
        "'1=' - '=' at pos 1",
    );

    // Test "+" at pos 0, "=" at pos 1 (different first token)
    test_position(
        &model,
        backend.clone(),
        &config,
        &[10, 28],
        "'+=' - '=' at pos 1",
    );

    // Test "a" at pos 0, "=" at pos 1 (different first token)
    test_position(
        &model,
        backend.clone(),
        &config,
        &[64, 28],
        "'a=' - '=' at pos 1",
    );

    // Test " " (space) at pos 0, "=" at pos 1
    test_position(
        &model,
        backend.clone(),
        &config,
        &[220, 28],
        "' =' - '=' at pos 1",
    );

    // What about "=" at pos 0, "=" at pos 1?
    test_position(
        &model,
        backend.clone(),
        &config,
        &[28, 28],
        "'==' - '=' at pos 1",
    );

    // Multiple "=" tokens
    test_position(
        &model,
        backend.clone(),
        &config,
        &[28, 28, 28],
        "'===' - '=' at pos 2",
    );
    test_position(
        &model,
        backend.clone(),
        &config,
        &[28, 28, 28, 28],
        "'====' - '=' at pos 3",
    );

    // Original "1+1="
    test_position(
        &model,
        backend.clone(),
        &config,
        &[16, 10, 16, 28],
        "'1+1=' - '=' at pos 3",
    );

    // What about "11"? (token 16 twice)
    test_position(
        &model,
        backend.clone(),
        &config,
        &[16, 16],
        "'11' - '1' at pos 1",
    );

    println!();
    println!("Analysis:");
    println!(
        "- If rank is similar across different first tokens, issue is in RoPE/position handling"
    );
    println!("- If rank depends on first token, issue is in how attention combines information");
}