aprender-serve 0.33.0

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

//! Verify tensor offsets match GGUF file
use realizar::gguf::MappedGGUFModel;

fn main() {
    let path = "/tmp/parity-bench/tinyllama-1.1b-q4_k_m.gguf";
    let mapped = MappedGGUFModel::from_path(path).expect("Failed");

    println!("tensor_data_start = {}", mapped.model.tensor_data_start);
    println!("\nChecking FFN gate tensor offsets:");

    for tensor in &mapped.model.tensors {
        if tensor.name.contains("ffn_gate")
            && (tensor.name.contains("blk.0.")
                || tensor.name.contains("blk.1.")
                || tensor.name.contains("blk.2.")
                || tensor.name.contains("blk.3.")
                || tensor.name.contains("blk.4."))
        {
            let abs_offset = mapped.model.tensor_data_start as u64 + tensor.offset;
            println!(
                "  {}: relative={}, absolute={}, dims={:?}",
                tensor.name, tensor.offset, abs_offset, tensor.dims
            );
        }
    }
}