aprender-serve 0.33.0

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

//! Measure actual forward pass time for Qwen2.5-Coder-1.5B
use realizar::gguf::{MappedGGUFModel, OwnedQuantizedModel, QuantizedGenerateConfig};
use realizar::RealizarError;
use std::time::Instant;

fn main() -> Result<(), RealizarError> {
    let model_path = std::env::args().nth(1).unwrap_or_else(|| {
        "/home/noah/src/single-shot-eval/models/raw/qwen2.5-coder-1.5b-instruct-q4_k_m.gguf"
            .to_string()
    });

    eprintln!("Loading model: {}", model_path);
    let mapped = MappedGGUFModel::from_path(&model_path)?;
    let model = OwnedQuantizedModel::from_mapped(&mapped)?;

    eprintln!(
        "Model: hidden_dim={}, intermediate_dim={}, layers={}",
        model.config().hidden_dim,
        model.config().intermediate_dim,
        model.config().num_layers
    );

    // Warmup
    let prompt = vec![1u32, 2, 3, 4];
    let config = QuantizedGenerateConfig {
        max_tokens: 5,
        temperature: 0.0,
        top_k: 1,
        stop_tokens: vec![],
        trace: false,
        ..Default::default()
    };
    let _ = model.generate_with_cache(&prompt, &config)?;
    let _ = model.generate_with_scratch(&prompt, &config)?;

    // Measure forward pass only (no sampling overhead)
    let gen_config = QuantizedGenerateConfig {
        max_tokens: 50, // Generate 50 tokens
        temperature: 0.0,
        top_k: 1,
        stop_tokens: vec![],
        trace: false,
        ..Default::default()
    };

    let iterations = 3;

    // Cache path
    let start = Instant::now();
    for _ in 0..iterations {
        let _ = model.generate_with_cache(&prompt, &gen_config)?;
    }
    let cache_total_ms = start.elapsed().as_millis() as f64 / iterations as f64;
    let cache_per_token_ms = cache_total_ms / 50.0;

    // Scratch path
    let start = Instant::now();
    for _ in 0..iterations {
        let _ = model.generate_with_scratch(&prompt, &gen_config)?;
    }
    let scratch_total_ms = start.elapsed().as_millis() as f64 / iterations as f64;
    let scratch_per_token_ms = scratch_total_ms / 50.0;

    println!(
        "\n=== Forward Pass Timing (50 tokens, {} iterations) ===",
        iterations
    );
    println!("Cache path:");
    println!("  Total:     {:.1} ms", cache_total_ms);
    println!(
        "  Per token: {:.1} ms ({:.1} tok/s)",
        cache_per_token_ms,
        1000.0 / cache_per_token_ms
    );
    println!("Scratch path:");
    println!("  Total:     {:.1} ms", scratch_total_ms);
    println!(
        "  Per token: {:.1} ms ({:.1} tok/s)",
        scratch_per_token_ms,
        1000.0 / scratch_per_token_ms
    );

    // Compare to estimated
    let estimated_per_token_ms = 46.8; // From profile
    println!("\n=== Gap Analysis ===");
    println!("Estimated per token: {:.1} ms", estimated_per_token_ms);
    println!("Actual (cache):      {:.1} ms", cache_per_token_ms);
    println!("Actual (scratch):    {:.1} ms", scratch_per_token_ms);
    println!(
        "Gap (cache):         {:.1} ms ({:.0}% overhead)",
        cache_per_token_ms - estimated_per_token_ms,
        (cache_per_token_ms / estimated_per_token_ms - 1.0) * 100.0
    );
    println!(
        "Gap (scratch):       {:.1} ms ({:.0}% overhead)",
        scratch_per_token_ms - estimated_per_token_ms,
        (scratch_per_token_ms / estimated_per_token_ms - 1.0) * 100.0
    );

    Ok(())
}