aprender-serve 0.64.0

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144


// ============================================================================
// Vocabulary accessor
// ============================================================================

#[test]
fn test_vocabulary_accessor() {
    let data = build_minimal_llama_gguf(32, 64, 128, 4, 4);
    let model = GGUFModel::from_bytes(&data).expect("test value should be present");

    let vocab = model.vocabulary();
    // May or may not have vocabulary depending on metadata
    let _ = vocab;
}

// ============================================================================
// rope_type accessor
// ============================================================================

#[test]
fn test_rope_type_accessor() {
    let data = GGUFBuilder::new()
        .architecture("llama")
        .add_u32("llama.rope.scaling.type", 0)
        .build();

    let model = GGUFModel::from_bytes(&data).expect("test value should be present");
    let rope_type = model.rope_type();
    // Should have some value
    let _ = rope_type;
}

// ============================================================================
// get_tensor_f32 coverage: remaining qtype branches
// ============================================================================

#[test]
fn test_get_tensor_f32_from_q2_k() {
    let q2k_data = create_q2_k_data(256);
    let data = GGUFBuilder::new()
        .add_q2_k_tensor("test.weight", &[16, 16], &q2k_data)
        .build();

    let model = GGUFModel::from_bytes(&data).expect("test value should be present");
    let tensor = model.get_tensor_f32("test.weight", &data);
    assert!(tensor.is_ok(), "Q2_K dequant failed: {:?}", tensor.err());
    assert_eq!(tensor.expect("test value should be present").len(), 256);
}

#[test]
fn test_get_tensor_f32_from_q5_k() {
    let q5k_data = create_q5_k_data(256);
    let data = GGUFBuilder::new()
        .add_q5_k_tensor("test.weight", &[16, 16], &q5k_data)
        .build();

    let model = GGUFModel::from_bytes(&data).expect("test value should be present");
    let tensor = model.get_tensor_f32("test.weight", &data);
    assert!(tensor.is_ok(), "Q5_K dequant failed: {:?}", tensor.err());
    assert_eq!(tensor.expect("test value should be present").len(), 256);
}

#[test]
fn test_get_tensor_f32_from_q6_k() {
    let q6k_data = create_q6_k_data(256);
    let data = GGUFBuilder::new()
        .add_q6_k_tensor("test.weight", &[16, 16], &q6k_data)
        .build();

    let model = GGUFModel::from_bytes(&data).expect("test value should be present");
    let tensor = model.get_tensor_f32("test.weight", &data);
    assert!(tensor.is_ok(), "Q6_K dequant failed: {:?}", tensor.err());
    assert_eq!(tensor.expect("test value should be present").len(), 256);
}

#[test]
fn test_get_tensor_f32_from_f16() {
    let f16_data = create_f16_data(64);
    let data = GGUFBuilder::new()
        .add_f16_tensor("test.weight", &[8, 8], &f16_data)
        .build();

    let model = GGUFModel::from_bytes(&data).expect("test value should be present");
    let tensor = model.get_tensor_f32("test.weight", &data);
    assert!(tensor.is_ok(), "F16 dequant failed: {:?}", tensor.err());
    let values = tensor.expect("test value should be present");
    assert_eq!(values.len(), 64);
    // F16 data from create_f16_data: val[i] = i * 0.01
    assert!((values[0] - 0.0).abs() < 0.01);
    assert!((values[1] - 0.01).abs() < 0.01);
}

#[test]
fn test_get_tensor_f32_from_q4_1() {
    let q4_1_data = create_q4_1_data(1024);
    let data = GGUFBuilder::new()
        .add_q4_1_tensor("test.weight", &[32, 32], &q4_1_data)
        .build();

    let model = GGUFModel::from_bytes(&data).expect("test value should be present");
    let tensor = model.get_tensor_f32("test.weight", &data);
    assert!(tensor.is_ok(), "Q4_1 dequant failed: {:?}", tensor.err());
    assert_eq!(tensor.expect("test value should be present").len(), 1024);
}

#[test]
fn test_get_tensor_f32_from_q5_0() {
    let q5_0_data = create_q5_0_data(1024);
    let data = GGUFBuilder::new()
        .add_q5_0_tensor("test.weight", &[32, 32], &q5_0_data)
        .build();

    let model = GGUFModel::from_bytes(&data).expect("test value should be present");
    let tensor = model.get_tensor_f32("test.weight", &data);
    assert!(tensor.is_ok(), "Q5_0 dequant failed: {:?}", tensor.err());
    assert_eq!(tensor.expect("test value should be present").len(), 1024);
}

#[test]
fn test_get_tensor_f32_from_q5_1() {
    let q5_1_data = create_q5_1_data(1024);
    let data = GGUFBuilder::new()
        .add_q5_1_tensor("test.weight", &[32, 32], &q5_1_data)
        .build();

    let model = GGUFModel::from_bytes(&data).expect("test value should be present");
    let tensor = model.get_tensor_f32("test.weight", &data);
    assert!(tensor.is_ok(), "Q5_1 dequant failed: {:?}", tensor.err());
    assert_eq!(tensor.expect("test value should be present").len(), 1024);
}

#[test]
fn test_get_tensor_f32_unsupported_qtype() {
    // Build a GGUF with a raw tensor of unsupported qtype (type 99)
    // We construct this by manually adjusting a Q4_0 tensor's qtype
    // Since GGUFBuilder doesn't support arbitrary qtypes, use a simple test:
    // just verify the error path with a nonexistent tensor
    let data = GGUFBuilder::new().architecture("llama").build();
    let model = GGUFModel::from_bytes(&data).expect("test value should be present");
    let result = model.get_tensor_f32("nonexistent", &data);
    assert!(result.is_err());
    let err_msg = format!("{:?}", result.unwrap_err());
    assert!(err_msg.contains("not found"));
}