trueno 0.17.5

High-performance SIMD compute library with GPU support for matrix operations
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

//! Model accuracy tests (F001-F020).

use super::super::*;
use std::time::Instant;

#[test]
fn f001_throughput_prediction_reasonable() {
    let features = TunerFeatures::builder()
        .model_params_b(1.5)
        .hidden_dim(1536)
        .batch_size(4)
        .quant_type(QuantType::Q4K)
        .cuda_graphs(true)
        .build();

    let regressor = ThroughputRegressor::new();
    let prediction = regressor.predict(&features);

    // Prediction should be positive and reasonable
    assert!(prediction.predicted_tps > 0.0);
    assert!(prediction.predicted_tps < 10000.0);
}

#[test]
fn f010_prediction_latency_under_1ms() {
    let features = TunerFeatures::builder().model_params_b(1.5).batch_size(4).build();

    let tuner = BrickTuner::new();
    let start = Instant::now();
    let _rec = tuner.recommend(&features);
    let elapsed = start.elapsed();

    assert!(elapsed.as_millis() < 1, "Prediction took {}ms", elapsed.as_millis());
}

#[test]
fn f015_batch_size_monotonic() {
    let regressor = ThroughputRegressor::new();

    let pred_m1 = regressor.predict(&TunerFeatures::builder().batch_size(1).build());
    let pred_m4 = regressor.predict(&TunerFeatures::builder().batch_size(4).build());
    let pred_m8 = regressor.predict(&TunerFeatures::builder().batch_size(8).build());

    // Higher batch size should predict higher throughput
    assert!(
        pred_m4.predicted_tps >= pred_m1.predicted_tps,
        "M=4 ({}) should be >= M=1 ({})",
        pred_m4.predicted_tps,
        pred_m1.predicted_tps
    );
    assert!(
        pred_m8.predicted_tps >= pred_m4.predicted_tps,
        "M=8 ({}) should be >= M=4 ({})",
        pred_m8.predicted_tps,
        pred_m4.predicted_tps
    );
}

#[test]
fn f019_cuda_graphs_benefit_predicted() {
    let regressor = ThroughputRegressor::new();

    let pred_no_graph =
        regressor.predict(&TunerFeatures::builder().batch_size(1).cuda_graphs(false).build());
    let pred_with_graph =
        regressor.predict(&TunerFeatures::builder().batch_size(1).cuda_graphs(true).build());

    // CUDA graphs should predict higher throughput
    assert!(
        pred_with_graph.predicted_tps >= pred_no_graph.predicted_tps,
        "With graphs ({}) should be >= without ({})",
        pred_with_graph.predicted_tps,
        pred_no_graph.predicted_tps
    );
}