aprender-serve 0.64.0

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131


/// IMP-400d: Full E2E comparison with Ollama (requires server)
#[test]
#[ignore = "Requires running Ollama server on port 11434"]
fn test_imp_400d_full_e2e_comparison() {
    use crate::gguf::{GGUFConfig, GGUFTransformer, GGUFTransformerLayer};
    use std::time::Instant;

    // Step 1: Measure Ollama throughput
    let config = HttpBenchmarkConfig {
        cv_criterion: CvStoppingCriterion::new(5, 10, 0.15),
        warmup_iterations: 1,
        prompt: "Hello".to_string(),
        max_tokens: 20,
        temperature: 0.0,
        ..Default::default()
    };

    let mut runner = HttpBenchmarkRunner::new(config);
    let ollama_result = runner
        .benchmark_ollama("http://127.0.0.1:11434", "phi2:2.7b")
        .expect("Ollama benchmark should succeed");

    // Step 2: Measure realizar forward pass
    let hidden_dim = 2560;
    let num_layers = 32;
    let vocab_size = 51200;
    let intermediate_dim = 10240;

    let gguf_config = GGUFConfig {
        architecture: "phi2_comparison".to_string(),
        constraints: crate::gguf::ArchConstraints::from_architecture("phi2_comparison"),
        hidden_dim,
        num_layers,
        num_heads: 32,
        num_kv_heads: 32,
        vocab_size,
        intermediate_dim,
        context_length: 2048,
        rope_theta: 10000.0,
        eps: 1e-5,
        rope_type: 0,
        bos_token_id: None,
            eos_token_id: None,
    };

    let layers: Vec<GGUFTransformerLayer> = (0..num_layers)
        .map(|_| GGUFTransformerLayer {
            attn_norm_weight: vec![1.0; hidden_dim],
            attn_norm_bias: None,
            qkv_weight: vec![0.01; hidden_dim * 3 * hidden_dim],
            qkv_bias: None,
            attn_output_weight: vec![0.01; hidden_dim * hidden_dim],
            attn_output_bias: None,
            ffn_norm_weight: Some(vec![1.0; hidden_dim]),
            ffn_norm_bias: None,
            ffn_gate_weight: Some(vec![0.01; hidden_dim * intermediate_dim]),
            ffn_gate_bias: None,
            ffn_up_weight: vec![0.01; hidden_dim * intermediate_dim],
            ffn_up_bias: None,
            ffn_down_weight: vec![0.01; intermediate_dim * hidden_dim],
            ffn_down_bias: None,
            attn_q_norm_weight: None,
            attn_k_norm_weight: None,
        })
        .collect();

    let transformer = GGUFTransformer {
        config: gguf_config,
        token_embedding: vec![0.01; vocab_size * hidden_dim],
        layers,
        output_norm_weight: vec![1.0; hidden_dim],
        output_norm_bias: None,
        lm_head_weight: vec![0.01; vocab_size * hidden_dim],
        lm_head_bias: None,
    };

    let _token_ids = vec![1u32];
    let iterations = 5;
    let mut latencies_ms = Vec::new();

    // Stub: actual forward pass requires OwnedQuantizedModel
    let _ = &transformer.config;
    for _ in 0..iterations {
        let start = Instant::now();
        // Placeholder: actual forward requires OwnedQuantizedModel conversion
        let _output: Vec<f32> = vec![0.0; transformer.config.vocab_size];
        latencies_ms.push(start.elapsed().as_secs_f64() * 1000.0);
    }

    let realizar_avg_ms = latencies_ms.iter().sum::<f64>() / iterations as f64;
    let realizar_tps = 1000.0 / realizar_avg_ms;

    // Step 3: Create comparison
    let comparison = E2EPerformanceComparison::from_measurements(
        ollama_result.throughput_tps,
        ollama_result.p50_latency_ms,
        realizar_tps,
        realizar_avg_ms,
        "phi-2 Q4_K_M (test weights)",
        20,
    );

    println!("\n╔══════════════════════════════════════════════════════════════╗");
    println!("║        IMP-400d: E2E Performance Comparison (phi-2)         ║");
    println!("╠══════════════════════════════════════════════════════════════╣");
    println!("║ Metric          │ Ollama (GPU)      │ Realizar (CPU)        ║");
    println!("╠─────────────────┼───────────────────┼───────────────────────╣");
    println!(
        "║ Throughput      │ {:>8.1} tok/s    │ {:>8.2} tok/s         ║",
        comparison.ollama_tps, comparison.realizar_tps
    );
    println!(
        "║ P50 Latency     │ {:>8.1} ms       │ {:>8.1} ms            ║",
        comparison.ollama_p50_ms, comparison.realizar_p50_ms
    );
    println!("╠─────────────────┴───────────────────┴───────────────────────╣");
    println!(
        "║ Performance Gap: {:.1}x (target: <1.25x for parity)         ║",
        comparison.performance_gap
    );
    println!(
        "║ Parity Achieved: {}                                          ║",
        if comparison.meets_parity_target() {
            "YES ✓"
        } else {
            "NO  ✗"
        }
    );
    println!("╚══════════════════════════════════════════════════════════════╝");
}