realizar 0.8.4

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

//! Benchmark forward pass timing
use realizar::gguf::{MappedGGUFModel, OwnedQuantizedModel};
use std::time::Instant;

fn main() {
    let model_path = std::env::args()
        .nth(1)
        .or_else(|| std::env::var("GGUF_MODEL").ok())
        .unwrap_or_else(|| {
            eprintln!("Usage: bench_forward <model.gguf>");
            eprintln!("   or: GGUF_MODEL=path cargo run --example bench_forward");
            std::process::exit(1);
        });

    println!("Loading model: {}", model_path);
    let start = Instant::now();
    let mapped = MappedGGUFModel::from_path(&model_path).expect("test");
    let model = OwnedQuantizedModel::from_mapped(&mapped).expect("test");
    println!("Model loaded in {:.2}s", start.elapsed().as_secs_f32());

    let config = model.config();
    println!(
        "Config: hidden_dim={}, vocab_size={}, num_layers={}",
        config.hidden_dim, config.vocab_size, config.num_layers
    );

    // Warmup
    println!("\nWarming up (3 iterations)...");
    for _ in 0..3 {
        let _ = model.forward(&[1]).expect("test");
    }

    // Benchmark single-token forward passes
    println!("\nBenchmarking single-token forward (10 iterations)...");
    let iterations = 10;
    let start = Instant::now();
    for _ in 0..iterations {
        let _ = model.forward(&[1]).expect("test");
    }
    let total_time = start.elapsed();
    let avg_time = total_time / iterations;
    let tok_per_sec = 1.0 / avg_time.as_secs_f64();

    println!(
        "Total time for {} iterations: {:.3}s",
        iterations,
        total_time.as_secs_f32()
    );
    println!(
        "Average forward pass: {:.1}ms",
        avg_time.as_secs_f64() * 1000.0
    );
    println!("Throughput: {:.1} tok/s", tok_per_sec);

    // Test with single token (for validation)
    let logits = model.forward(&[1]).expect("test");
    println!(
        "\nLogits len: {}, first 5: {:?}",
        logits.len(),
        &logits[..5]
    );

    // Find top prediction
    let (best_idx, best_logit) = logits
        .iter()
        .enumerate()
        .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
        .expect("test");

    let vocab = mapped.model.vocabulary().expect("test");
    let tok = if best_idx < vocab.len() {
        &vocab[best_idx]
    } else {
        "?"
    };
    println!("Top prediction: {} '{}' = {:.4}", best_idx, tok, best_logit);
}