realizar 0.8.6

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86


// =========================================================================
// T-COV-010: Additional kernel_name branch coverage (verify non-empty)
// =========================================================================

#[test]
fn test_tcov010_more_kernel_names() {
    let kernels = CudaKernels::new();

    // Verify a variety of kernel types return non-empty names
    let kernel_types: Vec<KernelType> = vec![
        KernelType::FusedRmsNormQ4KGemv {
            k: 4096,
            n: 4096,
            epsilon: 1e-6,
        },
        KernelType::FusedGateUpQ4KGemv { k: 4096, n: 11008 },
        KernelType::PackedDp4aQ4KQ8 { k: 4096, n: 4096 },
        KernelType::TrueDp4aQ4KGemv { k: 4096, n: 4096 },
        KernelType::Q4KQ8Dot { k: 4096, n: 4096 },
        KernelType::Q8Quantize { n: 4096 },
        KernelType::BatchedQ4KGemv {
            k: 4096,
            n: 4096,
            m: 4,
        },
        KernelType::MultiWarpBatchedQ4KGemv {
            k: 4096,
            n: 4096,
            warps: 4,
        },
        KernelType::TensorCoreQ4KGemm {
            m: 1,
            n: 4096,
            k: 4096,
        },
        KernelType::Rope {
            num_heads: 32,
            head_dim: 64,
            theta: 10000.0,
        },
        KernelType::RopeIndirect {
            num_heads: 32,
            head_dim: 64,
            theta: 10000.0,
        },
        KernelType::RopeNeox {
            num_heads: 32,
            head_dim: 64,
            theta: 10000.0,
        },
        KernelType::RopeNeoxIndirect {
            num_heads: 32,
            head_dim: 64,
            theta: 10000.0,
        },
        KernelType::PreciseRopeNeoxIndirect {
            num_heads: 32,
            head_dim: 64,
            theta: 10000.0,
        },
        KernelType::Silu { n: 4096 },
        KernelType::Gelu { n: 4096 },
        KernelType::ElementwiseMul { n: 4096 },
        KernelType::FusedSwiglu { n: 4096 },
        KernelType::FusedQKV {
            hidden_size: 4096,
            kv_dim: 4096,
        },
        KernelType::FusedGateUp {
            hidden_size: 4096,
            intermediate_size: 11008,
        },
        KernelType::ArgMax { length: 32000 },
        KernelType::ArgMaxFinal { num_blocks: 128 },
    ];

    for kernel_type in kernel_types {
        let name = kernels.kernel_name(&kernel_type);
        assert!(
            !name.is_empty(),
            "KernelType {:?} should have non-empty name",
            kernel_type
        );
    }
}