aprender-serve 0.64.0

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89


// =============================================================================
// Edge Cases
// =============================================================================

#[test]
fn test_phase34_embed_different_dimensions() {
    for hidden_dim in [32, 64, 128, 256] {
        let config = GGUFConfig {
            architecture: "llama".to_string(),
            constraints: crate::gguf::ArchConstraints::from_architecture("llama"),
            hidden_dim,
            intermediate_dim: hidden_dim * 2,
            num_layers: 1,
            num_heads: hidden_dim / 16,
            num_kv_heads: hidden_dim / 16,
            vocab_size: 100,
            context_length: 256,
            rope_theta: 10000.0,
            eps: 1e-5,
            rope_type: 0,
            explicit_head_dim: None,
            query_pre_attn_scalar: None,
            bos_token_id: None,
            eos_token_id: None,
        };

        let model = create_test_model_with_config(&config);
        let embeddings = model.embed(&[0, 1]);

        assert_eq!(embeddings.len(), 2 * hidden_dim);
    }
}

#[test]
fn test_phase34_fused_matmul_all_qtypes_comprehensive() {
    // Comprehensive test covering all qtypes in a single test
    let config = GGUFConfig {
        architecture: "llama".to_string(),
        constraints: crate::gguf::ArchConstraints::from_architecture("llama"),
        hidden_dim: 256,
        intermediate_dim: 512,
        num_layers: 1,
        num_heads: 4,
        num_kv_heads: 4,
        vocab_size: 100,
        context_length: 256,
        rope_theta: 10000.0,
        eps: 1e-5,
        rope_type: 0,
            explicit_head_dim: None,
            query_pre_attn_scalar: None,
        bos_token_id: None,
            eos_token_id: None,
    };

    let model = create_test_model_with_config(&config);
    let in_dim = 256;
    let out_dim = 512;
    let input = vec![1.0f32; in_dim];

    // Test each qtype
    let qtypes = [
        ("Q4_0", create_q4_0_weight(in_dim, out_dim)),
        ("Q8_0", create_q8_0_weight(in_dim, out_dim)),
        ("Q4_1", create_q4_1_weight(in_dim, out_dim)),
        ("Q5_0", create_q5_0_weight(in_dim, out_dim)),
        ("Q4_K", create_q4_k_weight(in_dim, out_dim)),
        ("Q5_K", create_q5_k_weight(in_dim, out_dim)),
        ("Q6_K", create_q6_k_weight(in_dim, out_dim)),
    ];

    for (name, weight) in qtypes {
        let result = model.fused_matmul(&input, &weight);
        assert!(
            result.is_ok(),
            "{} fused_matmul failed: {:?}",
            name,
            result.err()
        );
        let output = result.expect("test value should be present");
        assert_eq!(output.len(), out_dim, "{} output size mismatch", name);
        assert!(
            output.iter().all(|x| x.is_finite()),
            "{} produced non-finite values",
            name
        );
    }
}