1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
//! A set of structs and functions that define a layer in a neural network.
/// A set of structs and functions that define a layer in a neural network.
///
/// # Example
///
/// ```
/// let layer = layer::new(Series::new("weights", vec![1.0, 2.0, 3.0]), Series::new("biases", vec![1.0, 2.0, 3.0]));
/// ```
///
pub mod layer {
use crate::model::activation_functions::{ActivationFunction, ActivationFunctionType};
use crate::model::loss_functions::{LossFunction, LossFunctionType};
use polars::prelude::*;
use rand::{thread_rng, Rng};
/// A trait that defines a set of functions for a layer in a neural network.
pub trait Layer {
/// Performs a forward pass on the layer using the given activation function, calculating the outputs of the layer.
///
/// # Example
///
/// ```
/// let layer = LinearLayer::new(Series::new("weights", vec![1.0, 2.0, 3.0]), Series::new("biases", vec![1.0, 2.0, 3.0]));
///
/// let output = layer.forward(Series::new("inputs", vec![1.0, 2.0, 3.0]), ActivationFunctionType::ReLU);
///
/// ```
///
/// # Arguments
///
/// * `inputs` - A series of inputs to the layer.
///
/// * `activation_function` - The activation function to use for the layer.
///
/// # Returns
///
/// A series of outputs from the layer.
fn forward(&self, inputs: Series, activation_function: ActivationFunctionType) -> Series;
/// Performs a backward pass on the layer using the given loss function, calculating the gradients of the weights and biases for each layer.
///
/// # Example
///
/// ```
/// let layer: LinearLayer = LinearLayer::new(Series::new("weights", vec![1.0, 2.0, 3.0]), Series::new("biases", vec![1.0, 2.0, 3.0]));
///
/// let (updated_weights, updated_biases) = layer.backward(Series::new("inputs", vec![1.0, 2.0, 3.0]), Series::new("grad_outputs", vec![1.0, 2.0, 3.0], 0.01);
/// ```
///
/// # Arguments
///
/// * `inputs` - A series of inputs to the layer.
///
/// * `grad_outputs` - A series of gradients of the outputs of the layer.
///
/// * `lr` - The learning rate to use for the update.
///
/// # Returns
///
/// A tuple of the updated weights and biases for the layer.
///
fn backward(&self, inputs: Series, grad_outputs: Series, lr: f64) -> (Series, Series);
}
/// A struct that defines a linear (fully-connected) layer in a neural network.
///
/// # Example
///
/// ```
/// let linear_layer = LinearLayer::new(Series::new("weights", vec![1.0, 2.0, 3.0]), Series::new("biases", vec![1.0, 2.0, 3.0]));
/// ```
#[derive(Clone)]
pub struct LinearLayer {
pub weights: Series,
pub biases: Series,
}
impl LinearLayer {
/// Create a new layer with the given weights and biases.
///
/// # Example
///
/// ```
/// let layer = LinearLayer::new(Series::new("weights", vec![1.0, 2.0, 3.0]), Series::new("biases", vec![1.0, 2.0, 3.0]));
/// ```
///
/// # Arguments
///
/// * `weights` - A Series of weights for the layer.
///
/// * `biases` - A Series of biases for the layer.
///
/// # Returns
///
/// A new layer with the given weights and biases.
pub fn new(weights: Series, biases: Series) -> LinearLayer {
LinearLayer { weights, biases }
}
/// Create a new linear layer with all weights and biases set to zero.
///
/// # Example
///
/// ```
/// let layer = LinearLayer::zeroes(3);
/// ```
///
/// # Arguments
///
/// * `width` - The width of the layer.
///
/// # Returns
///
/// A new linear layer of the given width with all weights and biases set to zero.
pub fn zeroes(width: usize) -> LinearLayer {
LinearLayer {
weights: Series::new("weights", vec![0.0; width]),
biases: Series::new("biases", vec![0.0; width]),
}
}
/// Create a new linear layer with all weights and biases set to random values uniformly distributed within the given range.
///
/// # Example
///
/// ```
/// let layer = LinearLayer::new_random(3, [-1.0, 1.0]);
/// ```
///
/// # Arguments
///
/// * `width` - The width of the layer.
///
/// * `range` - The range of values to use for the random weights and biases.
///
/// # Returns
///
/// A new linear layer of the given width with all weights and biases set to random values uniformly distributed within the given range (inclusive).
pub fn new_random(width: usize, range: [f64; 2]) -> LinearLayer {
let mut init_weights: Vec<f64> = Vec::with_capacity(width);
let mut init_biases: Vec<f64> = Vec::with_capacity(width);
let low: f64 = if range[0] < range[1] {
range[0]
} else {
range[1]
};
let high: f64 = if range[0] < range[1] {
range[1]
} else {
range[0]
};
let mut rng: rand::prelude::ThreadRng = thread_rng();
for _ in 0..width {
init_weights.push(rng.gen_range(low..=high));
init_biases.push(rng.gen_range(low..=high));
}
LinearLayer {
weights: Series::new("weights", init_weights),
biases: Series::new("biases", init_biases),
}
}
}
impl Layer for LinearLayer {
fn forward(&self, inputs: Series, activation_function: ActivationFunctionType) -> Series {
let dot_product: f64 = (&inputs * &self.weights).sum().unwrap();
let dot_prod_series = Series::new("dot_product", vec![dot_product; inputs.len()]);
activation_function.activate(&(dot_prod_series + self.biases.clone()))
}
fn backward(&self, inputs: Series, grad_outputs: Series, lr: f64) -> (Series, Series) {
// Calculate gradient with respect to inputs
let grad_input: Series = &inputs * &grad_outputs;
// Calculate gradient with respect to weights
let grad_weights: Series = &inputs * &grad_outputs;
let grad_biases: f64 = grad_outputs.f64().unwrap().sum().unwrap();
let new_weights: Series = self.weights.clone() - grad_weights * lr;
let new_biases: Series = self.biases.clone() - Series::new("biases", vec![grad_biases; self.biases.len()]) * lr;
(new_weights, new_biases)
}
}
}