aprender-serve 0.29.0

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109


#[cfg(test)]
mod tests {
    use super::*;

    const TEST_TOKENS: &[u32] = &[1, 2, 3, 4, 5];

    #[test]
    fn t005_safetensors_cpu_forward() {
        let fixture = ModelFixture::safetensors("t005", ModelConfig::tiny());

        match forward_safetensors_cpu(&fixture, TEST_TOKENS) {
            Ok(result) => {
                eprintln!(
                    "[T005] SafeTensors:CPU produced {} logits",
                    result.logits.len()
                );
                let sum: f32 = result.logits.iter().sum();
                let max = result
                    .logits
                    .iter()
                    .cloned()
                    .fold(f32::NEG_INFINITY, f32::max);
                let min = result.logits.iter().cloned().fold(f32::INFINITY, f32::min);
                let argmax = result.argmax();
                eprintln!(
                    "[T005] SafeTensors:CPU sum={:.4}, min={:.4}, max={:.4}, argmax={:?}",
                    sum, min, max, argmax
                );
                let checks = falsify(&result);
                for check in &checks {
                    if !check.passed {
                        eprintln!("[T005] FALSIFIED {}: {}", check.id, check.details);
                    }
                }
            },
            Err(e) => {
                eprintln!("[T005] SafeTensors:CPU FAILED TO LOAD/RUN: {}", e);
            },
        }
    }

    #[test]
    #[cfg(feature = "cuda")]
    fn t002_gguf_cuda_forward() {
        use crate::cuda::CudaExecutor;
        if !CudaExecutor::is_available() {
            eprintln!("[T002] SKIPPED: CUDA not available");
            return;
        }

        let fixture = ModelFixture::gguf("t002", ModelConfig::tiny());

        match forward_gguf_cuda(&fixture, TEST_TOKENS) {
            Ok(result) => {
                eprintln!("[T002] GGUF:CUDA produced {} logits", result.logits.len());
                let sum: f32 = result.logits.iter().sum();
                let max = result
                    .logits
                    .iter()
                    .cloned()
                    .fold(f32::NEG_INFINITY, f32::max);
                let min = result.logits.iter().cloned().fold(f32::INFINITY, f32::min);
                let argmax = result.argmax();
                eprintln!(
                    "[T002] GGUF:CUDA sum={:.4}, min={:.4}, max={:.4}, argmax={:?}",
                    sum, min, max, argmax
                );
                let checks = falsify(&result);
                for check in &checks {
                    if !check.passed {
                        eprintln!("[T002] FALSIFIED {}: {}", check.id, check.details);
                    }
                }
            },
            Err(e) => {
                eprintln!("[T002] GGUF:CUDA FAILED TO LOAD/RUN: {}", e);
            },
        }
    }

    #[test]
    #[cfg(feature = "cuda")]
    fn t004_apr_cuda_forward() {
        use crate::cuda::CudaExecutor;
        if !CudaExecutor::is_available() {
            eprintln!("[T004] SKIPPED: CUDA not available");
            return;
        }

        let fixture = ModelFixture::apr("t004", ModelConfig::tiny());

        match forward_apr_cuda(&fixture, TEST_TOKENS) {
            Ok(result) => {
                eprintln!("[T004] APR:CUDA produced {} logits", result.logits.len());
                let checks = falsify(&result);
                for check in &checks {
                    if !check.passed {
                        eprintln!("[T004] FALSIFIED {}: {}", check.id, check.details);
                    }
                }
            },
            Err(e) => {
                eprintln!("[T004] APR:CUDA FAILED TO LOAD/RUN: {}", e);
            },
        }
    }
include!("falsification_tests_gguf_fixture_apr.rs");
}