memra-engine 0.84.0

From-scratch CUDA LLM inference engine for NVIDIA RTX 50-series (sm_120a) and Hopper (sm_90a) - custom kernels, no frameworks
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

//! rp_q4_probe: Q4_0 GGUF-block b4 vs the split-plane (rp) twin on the gemma verify-trunk
//! wq shape. Bitwise-gated inside Engine::rp_probe_q4. usage: rp_q4_probe [m]
use memra_engine::Engine;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let m: usize = std::env::args()
        .nth(1)
        .and_then(|v| v.parse().ok())
        .unwrap_or(3);
    let e = Engine::new(0)?;
    for rep in 0..3 {
        let (blk, rp) = e.rp_probe_q4(m)?;
        let bytes = (2048 * 88 * 18) as f64;
        println!(
            "rep{rep} m={m}: blk {blk:7.2}us ({:5.1} GB/s) | rp {rp:7.2}us ({:5.1} GB/s) | {:4.2}x",
            bytes / blk / 1e3,
            bytes / rp / 1e3,
            blk / rp
        );
    }
    Ok(())
}