realizar 0.8.4

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

//! PAR-091 DEBUG: Diagnose low speculative acceptance rate
//!
//! Compares draft (0.5B) vs target (1.5B) predictions token-by-token
//! to understand why acceptance is only 25% instead of expected 70%+
//!
//! Run with: cargo run --release --features cuda --example debug_speculative

use realizar::cuda::CudaExecutor;
use realizar::gguf::{
    MappedGGUFModel, OwnedQuantizedModel, OwnedQuantizedModelCuda, QuantizedGenerateConfig,
};
use std::path::Path;

fn main() {
    println!("═══════════════════════════════════════════════════════════════");
    println!("  PAR-091 DEBUG: Speculative Acceptance Analysis");
    println!("═══════════════════════════════════════════════════════════════");
    println!();

    if !CudaExecutor::is_available() {
        println!("❌ CUDA not available");
        return;
    }

    let target_path =
        "/home/noah/src/single-shot-eval/models/raw/qwen2.5-coder-1.5b-instruct-q4_k_m.gguf";
    let draft_path =
        "/home/noah/src/single-shot-eval/models/raw/qwen2.5-coder-0.5b-instruct-q4_0.gguf";

    if !Path::new(target_path).exists() || !Path::new(draft_path).exists() {
        println!("❌ Models not found");
        return;
    }

    // Load models
    println!("Loading models...");
    let target_mapped = MappedGGUFModel::from_path(target_path).expect("target model");
    let target_model = OwnedQuantizedModel::from_mapped(&target_mapped).expect("target model");
    let mut target_cuda = OwnedQuantizedModelCuda::new(target_model, 0).expect("target CUDA");

    let draft_mapped = MappedGGUFModel::from_path(draft_path).expect("draft model");
    let draft_model = OwnedQuantizedModel::from_mapped(&draft_mapped).expect("draft model");
    let mut draft_cuda = OwnedQuantizedModelCuda::new(draft_model, 0).expect("draft CUDA");

    // Check vocab sizes
    println!();
    println!("Model configs:");
    println!(
        "  Target (1.5B): vocab={}, hidden={}",
        target_cuda.model().config().vocab_size,
        target_cuda.model().config().hidden_dim
    );
    println!(
        "  Draft (0.5B):  vocab={}, hidden={}",
        draft_cuda.model().config().vocab_size,
        draft_cuda.model().config().hidden_dim
    );
    println!();

    // Generate independently with each model
    let prompt = vec![9707u32]; // "Hello"
    let config = QuantizedGenerateConfig::deterministic(20);

    println!("Generating 20 tokens with each model...");
    println!();

    // Clear graphs
    target_cuda.clear_decode_graph();
    draft_cuda.clear_decode_graph();

    let target_result = target_cuda.generate_gpu_resident(&prompt, &config);
    draft_cuda.clear_decode_graph();
    let draft_result = draft_cuda.generate_gpu_resident(&prompt, &config);

    match (target_result, draft_result) {
        (Ok(target_tokens), Ok(draft_tokens)) => {
            println!("Comparing token sequences:");
            println!();
            println!("Position | Target | Draft  | Match?");
            println!("---------|--------|--------|-------");

            let mut matches = 0;
            let max_len = target_tokens.len().max(draft_tokens.len());

            for i in 0..max_len {
                let target = target_tokens.get(i).copied();
                let draft = draft_tokens.get(i).copied();
                let matched = target == draft;
                if matched && target.is_some() {
                    matches += 1;
                }

                println!(
                    "   {:2}    | {:6} | {:6} | {}",
                    i,
                    target.map_or("-".to_string(), |t| t.to_string()),
                    draft.map_or("-".to_string(), |t| t.to_string()),
                    if matched { "✓" } else { "✗" }
                );
            }

            println!();
            println!("═══════════════════════════════════════════════════════════════");
            println!(
                "  Match rate: {}/{} ({:.1}%)",
                matches,
                max_len,
                (matches as f64) / (max_len as f64) * 100.0
            );
            println!("  Expected for same-family models: 60-80%");
            println!("═══════════════════════════════════════════════════════════════");

            if (matches as f64) / (max_len as f64) < 0.5 {
                println!();
                println!("⚠️  LOW MATCH RATE - Potential causes:");
                println!("   1. Different model architectures diverge quickly");
                println!("   2. 0.5B and 1.5B may have different training data");
                println!("   3. Quantization differences (Q4_0 vs Q4K_M)");
                println!();
                println!("   Recommendation: Try same-size draft/target (e.g., both 1.5B)");
                println!("   or different quantization (Q8 draft for Q4K target)");
            }
        },
        (Err(e), _) => println!("❌ Target generation failed: {}", e),
        (_, Err(e)) => println!("❌ Draft generation failed: {}", e),
    }
}