realizar 0.8.5

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147


#[test]
#[serial]
fn test_cov025_get_staging_buffer() {
    if !CudaExecutor::is_available() {
        return;
    }
    let mut executor = CudaExecutor::new(0).expect("CUDA executor");

    // Get a staging buffer
    let buf = executor.get_staging_buffer(1024);
    assert_eq!(buf.len(), 1024, "Staging buffer should have requested size");
    // Note: is_pinned() may return false depending on pool implementation

    // Return it
    executor.return_staging_buffer(buf);
}

#[test]
#[serial]
fn test_cov025_staging_pool_stats() {
    if !CudaExecutor::is_available() {
        return;
    }
    let mut executor = CudaExecutor::new(0).expect("CUDA executor");

    // Get initial stats
    let stats1 = executor.staging_pool_stats();
    assert_eq!(stats1.pool_hits, 0, "Initial hits should be 0");
    assert_eq!(stats1.pool_misses, 0, "Initial misses should be 0");

    // Allocate a buffer - should be a miss
    let buf = executor.get_staging_buffer(1024);
    let stats2 = executor.staging_pool_stats();
    assert_eq!(stats2.pool_misses, 1, "Should have 1 miss");

    // Return and get again - should be a hit
    executor.return_staging_buffer(buf);
    let _buf2 = executor.get_staging_buffer(1024);
    let stats3 = executor.staging_pool_stats();
    assert!(stats3.pool_hits >= 1, "Should have at least 1 hit");
}

#[test]
#[serial]
fn test_cov025_cached_weight_count() {
    if !CudaExecutor::is_available() {
        return;
    }
    let mut executor = CudaExecutor::new(0).expect("CUDA executor");

    // Initially zero
    assert_eq!(
        executor.cached_weight_count(),
        0,
        "Initial count should be 0"
    );

    // Load a weight
    let weights = vec![1.0f32; 256];
    executor
        .load_weights("test_weight", &weights)
        .expect("load");

    assert_eq!(
        executor.cached_weight_count(),
        1,
        "Count should be 1 after loading"
    );
}

#[test]
#[serial]
fn test_cov025_cached_weight_bytes() {
    if !CudaExecutor::is_available() {
        return;
    }
    let mut executor = CudaExecutor::new(0).expect("CUDA executor");

    // Initially zero
    assert_eq!(
        executor.cached_weight_bytes(),
        0,
        "Initial bytes should be 0"
    );

    // Load a weight (256 f32 = 1024 bytes)
    let weights = vec![1.0f32; 256];
    executor
        .load_weights("test_weight", &weights)
        .expect("load");

    let bytes = executor.cached_weight_bytes();
    assert!(bytes >= 1024, "Should have at least 1024 bytes cached");
}

#[test]
#[serial]
fn test_cov025_cached_quantized_weight_count() {
    if !CudaExecutor::is_available() {
        return;
    }
    let mut executor = CudaExecutor::new(0).expect("CUDA executor");

    // Initially zero
    assert_eq!(
        executor.cached_quantized_weight_count(),
        0,
        "Initial count should be 0"
    );

    // Load a quantized weight (Q4_K: 144 bytes per super-block)
    let weights = vec![0u8; 144];
    executor
        .load_quantized_weights("test_q4k", &weights)
        .expect("load");

    assert_eq!(
        executor.cached_quantized_weight_count(),
        1,
        "Count should be 1 after loading"
    );
}

#[test]
#[serial]
fn test_cov025_clear_weights() {
    if !CudaExecutor::is_available() {
        return;
    }
    let mut executor = CudaExecutor::new(0).expect("CUDA executor");

    // Load some weights
    let weights = vec![1.0f32; 256];
    executor.load_weights("test1", &weights).expect("load1");
    executor.load_weights("test2", &weights).expect("load2");
    assert_eq!(executor.cached_weight_count(), 2, "Should have 2 weights");

    // Clear
    executor.clear_weights();
    assert_eq!(
        executor.cached_weight_count(),
        0,
        "Should have 0 after clear"
    );
}