instmodel_inference 0.3.0

High-performance neural network inference library with instruction-based execution
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

use instmodel_inference::instruction_model_info::{DotInstructionInfo, InstructionInfo};
use instmodel_inference::{Activation, InstructionModel, InstructionModelInfo, PredictConfig};
use std::time::Instant;

fn create_benchmark_model() -> InstructionModel {
    let input_size = 300;
    let hidden_size = 500;
    let output_size = 5;

    let weights_layer0: Vec<Vec<f32>> = (0..hidden_size)
        .map(|i| {
            (0..input_size)
                .map(|j| ((i * input_size + j) as f32 * 0.001).sin() * 0.1)
                .collect()
        })
        .collect();

    let weights_layer1: Vec<Vec<f32>> = (0..output_size)
        .map(|i| {
            (0..hidden_size)
                .map(|j| ((i * hidden_size + j) as f32 * 0.002).cos() * 0.1)
                .collect()
        })
        .collect();

    let bias_layer0: Vec<f32> = (0..hidden_size)
        .map(|i| (i as f32 * 0.01).sin() * 0.01)
        .collect();
    let bias_layer1: Vec<f32> = (0..output_size)
        .map(|i| (i as f32 * 0.01).cos() * 0.01)
        .collect();

    let model_info = InstructionModelInfo {
        features: None,
        feature_size: Some(input_size),
        computation_buffer_sizes: vec![input_size, hidden_size, output_size],
        instructions: vec![
            InstructionInfo::Dot(DotInstructionInfo {
                input: 0,
                output: 1,
                weights: 0,
                activation: Some(Activation::Relu),
            }),
            InstructionInfo::Dot(DotInstructionInfo {
                input: 1,
                output: 2,
                weights: 1,
                activation: Some(Activation::Sigmoid),
            }),
        ],
        weights: vec![weights_layer0, weights_layer1],
        bias: vec![bias_layer0, bias_layer1],
        parameters: None,
        maps: None,
        validation_data: None,
    };

    InstructionModel::new(model_info).expect("Model creation should succeed")
}

fn generate_inputs(num_samples: usize, feature_size: usize) -> Vec<f32> {
    (0..num_samples * feature_size)
        .map(|i| (i as f32 * 0.001).sin())
        .collect()
}

fn main() {
    let model = create_benchmark_model();
    let num_samples = 50_000;
    let feature_size = model.get_feature_size();
    let output_size = model.get_output_size();

    println!("Model: {} -> {} -> {}", feature_size, 500, output_size);
    println!("Samples: {}", num_samples);
    println!(
        "Required memory per inference: {} f32 values",
        model.required_memory()
    );
    println!();

    let inputs = generate_inputs(num_samples, feature_size);

    // Warmup
    let _ = model.predict(&inputs[..feature_size]);

    // Sequential benchmark
    println!("Running sequential inference...");
    let start = Instant::now();
    let mut sequential_results = Vec::with_capacity(num_samples * output_size);
    for i in 0..num_samples {
        let input_start = i * feature_size;
        let input_end = input_start + feature_size;
        let result = model.predict(&inputs[input_start..input_end]).unwrap();
        sequential_results.extend(result);
    }
    let sequential_duration = start.elapsed();
    println!(
        "Sequential: {:.3?} ({:.3} inferences/sec)",
        sequential_duration,
        num_samples as f64 / sequential_duration.as_secs_f64()
    );

    // Parallel benchmark with default config (uses all CPU cores)
    println!("\nRunning parallel inference (default threads)...");
    let config = PredictConfig::new();
    let start = Instant::now();
    let parallel_result = model.predict_parallel(&inputs, config).unwrap();
    let parallel_duration = start.elapsed();
    println!(
        "Parallel:   {:.3?} ({:.3} inferences/sec)",
        parallel_duration,
        num_samples as f64 / parallel_duration.as_secs_f64()
    );

    // Verify results match
    let parallel_buffer = parallel_result.as_slice();
    let mut max_diff: f32 = 0.0;
    for (seq, par) in sequential_results.iter().zip(parallel_buffer.iter()) {
        max_diff = max_diff.max((seq - par).abs());
    }
    println!(
        "\nMax difference between sequential and parallel: {:.2e}",
        max_diff
    );

    // Speedup
    let speedup = sequential_duration.as_secs_f64() / parallel_duration.as_secs_f64();
    println!("\nSpeedup: {:.3}x", speedup);
}