realizar 0.8.5

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152


#[cfg(test)]
mod tests {
    use super::*;

    // ============================================================================
    // FFN UP+GATE FUSED TESTS
    // ============================================================================

    #[test]
    fn test_fused_q4k_q8k_ffn_up_gate_into_basic() {
        let in_dim: usize = 256;
        let out_dim: usize = 64;

        let up_weights = create_q4k_weights(out_dim, in_dim);
        let gate_weights = create_q4k_weights(out_dim, in_dim);

        let num_super_blocks = in_dim.div_ceil(QK_K);
        let q8k_scales = vec![0.1f32; num_super_blocks];
        let q8k_quants = vec![10i8; in_dim];

        let mut up_output = vec![0.0f32; out_dim];
        let mut gate_output = vec![0.0f32; out_dim];

        let result = fused_q4k_q8k_ffn_up_gate_into(
            &up_weights,
            &gate_weights,
            &q8k_scales,
            &q8k_quants,
            in_dim,
            out_dim,
            &mut up_output,
            &mut gate_output,
        );
        assert!(result.is_ok());
    }

    #[test]
    fn test_fused_q4k_q8k_ffn_up_gate_into_weight_too_small() {
        let in_dim: usize = 256;
        let out_dim: usize = 64;

        let up_weights = vec![0u8; 100]; // Too small
        let gate_weights = create_q4k_weights(out_dim, in_dim);

        let q8k_scales = vec![0.1f32; 1];
        let q8k_quants = vec![10i8; in_dim];

        let mut up_output = vec![0.0f32; out_dim];
        let mut gate_output = vec![0.0f32; out_dim];

        let result = fused_q4k_q8k_ffn_up_gate_into(
            &up_weights,
            &gate_weights,
            &q8k_scales,
            &q8k_quants,
            in_dim,
            out_dim,
            &mut up_output,
            &mut gate_output,
        );
        assert!(result.is_err());
    }

    #[test]
    fn test_fused_q4k_q8k_ffn_up_gate_into_output_too_small() {
        let in_dim: usize = 256;
        let out_dim: usize = 64;

        let up_weights = create_q4k_weights(out_dim, in_dim);
        let gate_weights = create_q4k_weights(out_dim, in_dim);

        let q8k_scales = vec![0.1f32; 1];
        let q8k_quants = vec![10i8; in_dim];

        let mut up_output = vec![0.0f32; 32]; // Too small
        let mut gate_output = vec![0.0f32; out_dim];

        let result = fused_q4k_q8k_ffn_up_gate_into(
            &up_weights,
            &gate_weights,
            &q8k_scales,
            &q8k_quants,
            in_dim,
            out_dim,
            &mut up_output,
            &mut gate_output,
        );
        assert!(result.is_err());
    }

    // ============================================================================
    // EDGE CASE TESTS
    // ============================================================================

    #[test]
    fn test_parallel_matvec_multiple_super_blocks_per_row() {
        // in_dim = 512 means 2 super-blocks per row
        let in_dim: usize = 512;
        let out_dim: usize = 32;

        let weights = create_q4k_weights(out_dim, in_dim);
        let activations = vec![0.1f32; in_dim];

        let result = fused_q4k_parallel_matvec(&weights, &activations, in_dim, out_dim);
        assert!(result.is_ok());
        assert_eq!(result.expect("result").len(), out_dim);
    }

    #[test]
    fn test_parallel_matvec_large_output_dimension() {
        // Test with larger output dimension (uses parallel path)
        let in_dim: usize = 256;
        let out_dim: usize = 1024;

        let weights = create_q4k_weights(out_dim, in_dim);
        let activations = vec![0.1f32; in_dim];

        let result = fused_q4k_parallel_matvec(&weights, &activations, in_dim, out_dim);
        assert!(result.is_ok());
        assert_eq!(result.expect("result").len(), out_dim);
    }

    #[test]
    fn test_parallel_matvec_at_parallel_threshold() {
        // Exactly at the threshold (256)
        let in_dim: usize = 256;
        let out_dim: usize = 256;

        let weights = create_q4k_weights(out_dim, in_dim);
        let activations = vec![0.1f32; in_dim];

        let result = fused_q4k_parallel_matvec(&weights, &activations, in_dim, out_dim);
        assert!(result.is_ok());
        assert_eq!(result.expect("result").len(), out_dim);
    }

    #[test]
    fn test_parallel_matvec_just_below_parallel_threshold() {
        // Just below threshold (255) - uses sequential path
        let in_dim: usize = 256;
        let out_dim: usize = 255;

        let weights = create_q4k_weights(out_dim, in_dim);
        let activations = vec![0.1f32; in_dim];

        let result = fused_q4k_parallel_matvec(&weights, &activations, in_dim, out_dim);
        assert!(result.is_ok());
        assert_eq!(result.expect("result").len(), out_dim);
    }
include!("parallel_k_test_helpers.rs");
}