aprender-compute 0.31.1

High-performance SIMD compute library with GPU support, LLM inference engine, and GGUF model loading (was: trueno)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196

#![allow(clippy::disallowed_methods)]
//! Activation Functions Example
//!
//! Demonstrates all 11 activation functions available in Trueno for neural network
//! architectures. Each activation function is shown with its typical use case and
//! characteristics.
//!
//! Run with: cargo run --example activation_functions

use trueno::Vector;

fn main() {
    println!("🧠 Trueno Activation Functions Demo");
    println!("=====================================\n");

    // Sample input spanning negative to positive range
    let input = Vector::from_slice(&[-3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0]);
    println!("Input vector: {:?}\n", input.as_slice());

    // ========================================================================
    // Classic Activations
    // ========================================================================

    println!("📊 Classic Activation Functions");
    println!("--------------------------------\n");

    // ReLU: max(0, x) - Most common activation
    let relu_output = input.relu().expect("Example should not fail");
    println!("✓ ReLU (max(0, x))");
    println!("  Use case: Hidden layers in most neural networks");
    println!("  Output: {:?}", relu_output.as_slice());
    println!("  Characteristic: Zero for negative, identity for positive\n");

    // Leaky ReLU: Prevents dying ReLU problem
    let leaky_relu_output = input.leaky_relu(0.01).expect("Example should not fail");
    println!("✓ Leaky ReLU (x if x > 0, else 0.01*x)");
    println!("  Use case: When dying ReLU is a problem");
    println!("  Output: {:?}", leaky_relu_output.as_slice());
    println!("  Characteristic: Small negative slope prevents neuron death\n");

    // ELU: Exponential Linear Unit
    let elu_output = input.elu(1.0).expect("Example should not fail");
    println!("✓ ELU (x if x > 0, else α(e^x - 1))");
    println!("  Use case: When smooth gradients are needed");
    println!("  Output: {:?}", elu_output.as_slice());
    println!("  Characteristic: Smooth, bounded below by -α\n");

    // Sigmoid: Logistic function
    let sigmoid_output = input.sigmoid().expect("Example should not fail");
    println!("✓ Sigmoid (1 / (1 + e^(-x)))");
    println!("  Use case: Binary classification, output layer");
    println!("  Output: {:?}", sigmoid_output.as_slice());
    println!("  Characteristic: Bounded [0, 1], probabilistic interpretation\n");

    // ========================================================================
    // Modern Activations
    // ========================================================================

    println!("🚀 Modern Activation Functions");
    println!("--------------------------------\n");

    // GELU: Gaussian Error Linear Unit (used in transformers)
    let gelu_output = input.gelu().expect("Example should not fail");
    println!("✓ GELU (Gaussian Error Linear Unit)");
    println!("  Use case: BERT, GPT, T5, modern transformers");
    println!("  Output: {:?}", gelu_output.as_slice());
    println!("  Characteristic: Smooth, non-monotonic, better gradient flow\n");

    // Swish/SiLU: Self-gated activation
    let swish_output = input.swish().expect("Example should not fail");
    println!("✓ Swish/SiLU (x * sigmoid(x))");
    println!("  Use case: EfficientNet, MobileNet v3");
    println!("  Output: {:?}", swish_output.as_slice());
    println!("  Characteristic: Self-gated, smooth, unbounded above\n");

    // Hardswish: Efficient approximation for mobile
    let hardswish_output = input.hardswish().expect("Example should not fail");
    println!("✓ Hardswish (x * clip(x+3, 0, 6) / 6)");
    println!("  Use case: MobileNetV3, efficient on-device inference");
    println!("  Output: {:?}", hardswish_output.as_slice());
    println!("  Characteristic: Fast computation, no exp(), mobile-optimized\n");

    // Mish: Self-regularized non-monotonic
    let mish_output = input.mish().expect("Example should not fail");
    println!("✓ Mish (x * tanh(softplus(x)))");
    println!("  Use case: YOLOv4, modern object detection");
    println!("  Output: {:?}", mish_output.as_slice());
    println!("  Characteristic: Smooth, self-regularized, non-monotonic\n");

    // SELU: Self-normalizing
    let selu_output = input.selu().expect("Example should not fail");
    println!("✓ SELU (λ * elu(x, α))");
    println!("  Use case: Self-normalizing neural networks");
    println!("  Output: {:?}", selu_output.as_slice());
    println!("  Characteristic: Self-normalizing, reduces need for batch norm\n");

    // ========================================================================
    // Probabilistic Activations
    // ========================================================================

    println!("📈 Probabilistic Activation Functions");
    println!("--------------------------------------\n");

    // Softmax: Convert logits to probabilities
    let logits = Vector::from_slice(&[2.0, 1.0, 0.1]);
    let softmax_output = logits.softmax().expect("Example should not fail");
    println!("✓ Softmax (multi-class output layer)");
    println!("  Input logits: {:?}", logits.as_slice());
    println!("  Output probabilities: {:?}", softmax_output.as_slice());
    println!("  Sum: {:.6}", softmax_output.as_slice().iter().sum::<f32>());
    println!("  Characteristic: Outputs sum to 1.0, probabilistic\n");

    // Log-Softmax: Numerically stable for cross-entropy loss
    let log_softmax_output = logits.log_softmax().expect("Example should not fail");
    println!("✓ Log-Softmax (numerically stable cross-entropy)");
    println!("  Output: {:?}", log_softmax_output.as_slice());
    println!("  Characteristic: More stable than log(softmax(x))\n");

    // ========================================================================
    // Comparison: Different behaviors
    // ========================================================================

    println!("🔍 Behavior Comparison at Key Points");
    println!("-------------------------------------\n");

    let test_points = Vector::from_slice(&[-5.0, -1.0, 0.0, 1.0, 5.0]);
    println!("Test points: {:?}\n", test_points.as_slice());

    println!("Activation   | -5.0     | -1.0     | 0.0      | 1.0      | 5.0");
    println!("-------------|----------|----------|----------|----------|----------");

    print_activation_row("ReLU        ", &test_points.relu().expect("Example should not fail"));
    print_activation_row(
        "Leaky ReLU  ",
        &test_points.leaky_relu(0.01).expect("Example should not fail"),
    );
    print_activation_row("ELU         ", &test_points.elu(1.0).expect("Example should not fail"));
    print_activation_row("Sigmoid     ", &test_points.sigmoid().expect("Example should not fail"));
    print_activation_row("GELU        ", &test_points.gelu().expect("Example should not fail"));
    print_activation_row("Swish       ", &test_points.swish().expect("Example should not fail"));
    print_activation_row(
        "Hardswish   ",
        &test_points.hardswish().expect("Example should not fail"),
    );
    print_activation_row("Mish        ", &test_points.mish().expect("Example should not fail"));
    print_activation_row("SELU        ", &test_points.selu().expect("Example should not fail"));

    println!("\n");

    // ========================================================================
    // Practical Example: Simple Neural Network Layer
    // ========================================================================

    println!("🎯 Practical Example: Neural Network Layer");
    println!("-------------------------------------------\n");

    // Simulate a batch of pre-activations from a linear layer
    let layer_output = Vector::from_slice(&[0.5, -0.3, 1.2, -0.8, 2.1]);
    println!("Layer output (pre-activation): {:?}", layer_output.as_slice());

    // Different activation choices for different architectures
    println!("\nIf using CNN (e.g., ResNet):");
    let cnn_activation = layer_output.relu().expect("Example should not fail");
    println!("  → ReLU: {:?}", cnn_activation.as_slice());

    println!("\nIf using Transformer (e.g., BERT):");
    let transformer_activation = layer_output.gelu().expect("Example should not fail");
    println!("  → GELU: {:?}", transformer_activation.as_slice());

    println!("\nIf using EfficientNet:");
    let efficientnet_activation = layer_output.swish().expect("Example should not fail");
    println!("  → Swish: {:?}", efficientnet_activation.as_slice());

    println!("\nIf using MobileNetV3 (on-device):");
    let mobilenet_activation = layer_output.hardswish().expect("Example should not fail");
    println!("  → Hardswish: {:?}", mobilenet_activation.as_slice());

    println!("\nIf using YOLOv4 (object detection):");
    let yolo_activation = layer_output.mish().expect("Example should not fail");
    println!("  → Mish: {:?}", yolo_activation.as_slice());

    println!("\nIf using self-normalizing network:");
    let snn_activation = layer_output.selu().expect("Example should not fail");
    println!("  → SELU: {:?}", snn_activation.as_slice());

    println!("\n✨ All 11 activation functions computed successfully!");
    println!("   Note: Activations use scalar ops; SIMD optimization planned for v0.3.0");
}

fn print_activation_row(name: &str, output: &Vector<f32>) {
    print!("{}", name);
    for &val in output.as_slice() {
        print!("| {:8.4} ", val);
    }
    println!("|");
}