fast_neural_network/
neural_network.rs

1//! # Neural Network
2//!
3//! This is the neural network module. It contains the `Network` struct and the `ActivationType` enum. This is the heart of the crate.
4//!
5
6use indicatif::{MultiProgress, ProgressBar, ProgressState, ProgressStyle};
7use ndarray::*;
8use rand::random;
9use rayon::prelude::*;
10use serde::{Deserialize, Serialize};
11use serde_json;
12use std::f64;
13use std::fmt::{self, Display, Formatter, Write};
14
15use crate::activation::*;
16
17/// The underlying layer struct for the heap based network.
18#[derive(Debug, Serialize, Deserialize, Clone)]
19pub struct Layer {
20    pub size: usize,
21    pub bias: Vec<f64>,
22}
23
24impl Layer {
25    /// Creates a new layer with the given size.
26    ///
27    /// ## Example
28    /// ```
29    /// use fast_neural_network::neural_network::*;
30    ///
31    /// let layer = Layer::new(3);
32    ///
33    /// assert_eq!(layer.size, 3);
34    /// assert_eq!(layer.bias.len(), 3);
35    /// ```
36    pub fn new(size: usize) -> Self {
37        Layer {
38            size,
39            bias: (0..size).into_par_iter().map(|_| random::<f64>()).collect(),
40        }
41    }
42
43    /// Creates a new layer with the given bias.
44    ///
45    /// ## Example
46    /// ```
47    /// use fast_neural_network::neural_network::*;
48    ///
49    /// let layer = Layer::from_vec(vec![0.03, 0.62, 0.85, 0.60, 0.62, 0.64]);
50    ///
51    /// assert_eq!(layer.size, 6);
52    /// assert_eq!(layer.bias.len(), 6);
53    /// ```
54    pub fn from_vec(vec: Vec<f64>) -> Self {
55        Layer {
56            size: vec.len(),
57            bias: vec,
58        }
59    }
60}
61
62/// The main heap based neural network struct.
63#[derive(Debug, Serialize, Deserialize, Clone)]
64pub struct Network {
65    inputs: Layer,             // number of neurons in input layer
66    outputs: Layer,            // number of neurons in output layer
67    hidden_layers: Vec<Layer>, // number of hidden layers (each layer has a number of neurons)
68    layer_matrices: Vec<(ndarray::Array2<f64>, ndarray::Array1<f64>)>, // (weights, biases)
69    activation_matrices: Vec<ndarray::Array1<f64>>,
70    activation: ActivationType, // activation function
71    learning_rate: f64,
72    ui_update_interval: usize,
73    compiled: bool,
74}
75
76impl Network {
77    /// Creates a new empty network with the given number of inputs and outputs.
78    ///
79    /// ## Example
80    /// ```
81    /// use fast_neural_network::{activation::*, neural_network::*};
82    ///
83    /// let mut network = Network::new(3, 1, ActivationType::Relu, 0.005);
84    ///
85    /// assert_eq!(network.dimensions().0, 3);
86    /// assert_eq!(network.dimensions().1, 1);
87    /// assert_eq!(network.hidden_layers_size(), 0);
88    /// assert_eq!(network.learning_rate(), 0.005);
89    /// ```
90    pub fn new(inputs: usize, outputs: usize, activation_func: ActivationType, alpha: f64) -> Self {
91        Network {
92            inputs: Layer::new(inputs),
93            outputs: Layer::new(outputs),
94            hidden_layers: vec![],
95            layer_matrices: vec![],
96            activation_matrices: vec![],
97            activation: activation_func,
98            learning_rate: alpha,
99            ui_update_interval: 100,
100            compiled: false,
101        }
102    }
103
104    /// Creates a new empty network with the given number of inputs and outputs and the given hidden layers.
105    ///
106    /// ## Example
107    /// ```
108    /// use fast_neural_network::{activation::*, neural_network::*};
109    ///
110    /// let mut network = Network::new_with_layers(3, 1, vec![Layer::new(4), Layer::new(4)], ActivationType::Relu, 0.005);
111    ///
112    /// assert_eq!(network.dimensions().0, 3);
113    /// assert_eq!(network.dimensions().1, 1);
114    /// assert_eq!(network.hidden_layers_size(), 2);
115    /// assert_eq!(network.learning_rate(), 0.005);
116    /// ```
117    pub fn new_with_layers(
118        inputs: usize,
119        outputs: usize,
120        hidden_layers: Vec<Layer>,
121        activation_func: ActivationType,
122        alpha: f64,
123    ) -> Self {
124        Network {
125            inputs: Layer::new(inputs),
126            outputs: Layer::new(outputs),
127            hidden_layers,
128            layer_matrices: vec![],
129            activation_matrices: vec![],
130            activation: activation_func,
131            learning_rate: alpha,
132            ui_update_interval: 100,
133            compiled: false,
134        }
135    }
136
137    /// Loads a network from a json file.
138    ///
139    /// ## Example
140    /// ```
141    /// use fast_neural_network::neural_network::*;
142    ///
143    /// let mut network = Network::load("network.json");
144    /// ```
145    pub fn load(path: &str) -> Self {
146        let json = std::fs::read_to_string(path).expect("Unable to read file");
147        serde_json::from_str(&json).unwrap()
148    }
149
150    /// Saves the network to a json file.
151    ///
152    /// ## Example
153    /// ```
154    /// use fast_neural_network::{activation::*, neural_network::*};
155    ///
156    /// let mut network = Network::new(3, 1, ActivationType::Relu, 0.005);
157    ///
158    /// network.save("network.json");
159    /// ```
160    pub fn save(&self, path: &str) {
161        let json = serde_json::to_string(&self).unwrap();
162        std::fs::write(path, json).expect("Unable to write file");
163    }
164
165    /// Creates a network from the given JSON string.
166    ///
167    /// ## Panics
168    ///
169    /// Panics if the JSON string is not valid.
170    pub fn from_json(json: &str) -> Self {
171        serde_json::from_str(&json).unwrap()
172    }
173
174    /// Adds a hidden layer to the network.
175    pub fn add_hidden_layer(&mut self, layer: Layer) {
176        self.hidden_layers.push(layer);
177    }
178
179    /// Adds a hidden layer to the network with the given size.
180    ///
181    /// ## Example
182    /// ```
183    /// use fast_neural_network::{activation::*, neural_network::*};
184    ///
185    /// let mut network = Network::new(3, 1, ActivationType::Relu, 0.005);
186    ///
187    /// network.add_hidden_layer_with_size(4);
188    /// ```
189    pub fn add_hidden_layer_with_size(&mut self, size: usize) {
190        self.hidden_layers.push(Layer::new(size));
191    }
192
193    /// Compiles the network. This prepares the random inital values of the network. Can be re-run as much as needed, if needed.
194    /// This is done automatically during training if it was not compiled before hand.
195    /// > Compilation should be done after the hidden layers are set, but before setting any custom layer values.
196    ///
197    /// ## Example
198    /// ```
199    /// use fast_neural_network::{activation::*, neural_network::*};
200    ///
201    /// let mut network = Network::new(3, 1, ActivationType::Relu, 0.005);
202    /// network.add_hidden_layer_with_size(4);
203    /// network.add_hidden_layer_with_size(4);
204    /// network.compile();
205    /// ```
206    ///
207    /// ## Panics
208    ///
209    /// Panics if any of the dimentions is 0 or if no hidden layers are present.
210    pub fn compile(&mut self) {
211        assert!(self.inputs.size > 0);
212        assert!(self.outputs.size > 0);
213
214        self.layer_matrices.clear();
215
216        let mut layers = vec![self.inputs.clone()];
217        layers.append(&mut self.hidden_layers.clone());
218        layers.push(self.outputs.clone());
219
220        for i in 0..layers.len() - 1 {
221            let mut weights = vec![];
222            let mut biases = vec![];
223            for _ in 0..layers[i + 1].size {
224                weights.append(
225                    &mut (0..layers[i].size)
226                        .into_par_iter()
227                        .map(|_| random::<f64>())
228                        .collect(),
229                );
230                biases.push(random::<f64>());
231            }
232            let weights =
233                Array2::from_shape_vec((layers[i + 1].size, layers[i].size), weights).unwrap();
234            let biases = Array1::from_shape_vec(layers[i + 1].size, biases).unwrap();
235            self.layer_matrices.push((weights, biases));
236        }
237
238        self.compiled = true;
239    }
240
241    /// Returns a Tuple with the dimentions of the Neural Network (inputs, outputs)
242    pub fn dimensions(&self) -> (usize, usize) {
243        (self.inputs.size, self.outputs.size)
244    }
245
246    /// Sets the activation function to be used by the network
247    pub fn set_activation(&mut self, activation: ActivationType) {
248        self.activation = activation;
249    }
250
251    /// Returns the activation function being used
252    pub fn activation(&self) -> ActivationType {
253        self.activation.clone()
254    }
255
256    /// Sets the weights and biases of the given layer
257    pub fn set_layer_weights(&mut self, layer: usize, weights: ndarray::Array2<f64>) {
258        assert!(layer < self.layer_matrices.len());
259        self.layer_matrices[layer].0 = weights;
260    }
261
262    /// Returns the weights of the given layer
263    pub fn layer_weights(&self, layer: usize) -> ndarray::Array2<f64> {
264        assert!(layer < self.layer_matrices.len());
265        self.layer_matrices[layer].0.clone()
266    }
267
268    /// Sets the biases of the given layer
269    pub fn set_layer_biases(&mut self, layer: usize, biases: ndarray::Array1<f64>) {
270        assert!(layer < self.layer_matrices.len());
271        assert!(biases.len() == self.layer_matrices[layer].1.len());
272
273        self.layer_matrices[layer].1 = biases.t().to_owned();
274    }
275
276    /// Returns the biases of the given layer
277    pub fn layer_biases(&self, layer: usize) -> ndarray::Array1<f64> {
278        assert!(layer < self.layer_matrices.len());
279        self.layer_matrices[layer].1.clone()
280    }
281
282    /// Returns the number of hidden layers
283    pub fn hidden_layers_size(&self) -> usize {
284        self.hidden_layers.len()
285    }
286
287    /// Sets the learning rate of the network
288    pub fn set_learning_rate(&mut self, alpha: f64) {
289        self.learning_rate = alpha;
290    }
291
292    /// Returns the learning rate of the network
293    pub fn learning_rate(&self) -> f64 {
294        self.learning_rate
295    }
296
297    /// Sets the UI progress update interval of the network
298    pub fn set_ui_update_interval(&mut self, interval: usize) {
299        self.ui_update_interval = interval;
300    }
301
302    fn activate(&self, weights: &mut ndarray::Array1<f64>) {
303        match self.activation {
304            ActivationType::Sigmoid => weights.iter_mut().for_each(|x| {
305                *x = sigm(x);
306            }),
307            ActivationType::Tanh => weights.iter_mut().for_each(|x| {
308                *x = tanh(x);
309            }),
310            ActivationType::ArcTanh => weights.iter_mut().for_each(|x| {
311                *x = arc_tanh(x);
312            }),
313            ActivationType::Relu => weights.iter_mut().for_each(|x| {
314                *x = relu(x);
315            }),
316            ActivationType::LeakyRelu => weights.iter_mut().for_each(|x| {
317                *x = leaky_relu(x);
318            }),
319            ActivationType::ELU => weights.iter_mut().for_each(|x| {
320                *x = elu(x);
321            }),
322            ActivationType::Swish => weights.iter_mut().for_each(|x| {
323                *x = swish(x);
324            }),
325            ActivationType::SoftPlus => weights.iter_mut().for_each(|x| {
326                *x = softplus(x);
327            }),
328            ActivationType::SoftMax => {
329                let w_col = weights.clone();
330                weights.iter_mut().for_each(|x| {
331                    *x = softmax(x, &w_col);
332                })
333            }
334        };
335    }
336    /// Predicts the output of the network for the given input.
337    ///
338    /// ## Example
339    /// ```
340    /// // ... imports here
341    ///
342    /// let mut network = Network::new(3, 1, ActivationType::Relu, 0.005);
343    ///
344    /// // ... training done here
345    ///
346    /// let prediction = network.forward(&array![2., 1., -1.]);
347    pub fn forward(&mut self, input: &ndarray::Array1<f64>) -> ndarray::Array2<f64> {
348        if !self.compiled {
349            self.compile();
350        }
351
352        assert!(input.len() == self.inputs.size);
353
354        self.activation_matrices.clear();
355
356        let mut output: ndarray::Array1<f64> = input.clone();
357
358        for i in 0..self.layer_matrices.len() {
359            let (weights, biases) = &self.layer_matrices[i];
360
361            output = weights.dot(&output);
362            output = output + biases;
363
364            self.activate(&mut output);
365
366            self.activation_matrices.push(output.clone());
367        }
368
369        output.to_shape((self.outputs.size, 1)).unwrap().to_owned()
370    }
371
372    fn derivate(&self, mut array: ndarray::Array1<f64>) -> ndarray::Array1<f64> {
373        match self.activation {
374            ActivationType::Sigmoid => {
375                array.iter_mut().for_each(|x| {
376                    *x = der_sigm(x);
377                });
378                array
379            }
380            ActivationType::Tanh => {
381                array.iter_mut().for_each(|x| {
382                    *x = der_tanh(x);
383                });
384                array
385            }
386            ActivationType::ArcTanh => {
387                array.iter_mut().for_each(|x| {
388                    *x = der_arc_tanh(x);
389                });
390                array
391            }
392            ActivationType::Relu => {
393                array.iter_mut().for_each(|x| {
394                    *x = der_relu(x);
395                });
396                array
397            }
398            ActivationType::LeakyRelu => {
399                array.iter_mut().for_each(|x| {
400                    *x = der_leaky_relu(x);
401                });
402                array
403            }
404            ActivationType::ELU => {
405                array.iter_mut().for_each(|x| {
406                    *x = der_elu(x);
407                });
408                array
409            }
410            ActivationType::Swish => {
411                array.iter_mut().for_each(|x| {
412                    *x = der_swish(x);
413                });
414                array
415            }
416            ActivationType::SoftPlus => {
417                array.iter_mut().for_each(|x| {
418                    *x = der_softplus(x);
419                });
420                array
421            }
422            ActivationType::SoftMax => {
423                let array_col = array.clone();
424                array.iter_mut().for_each(|x| {
425                    *x = der_softmax(x, &array_col);
426                });
427                array
428            }
429        }
430    }
431
432    /// Trains the network with the given training set for the given number of epochs.
433    ///
434    /// > It will compile the network if it was not compiled at least once
435    ///
436    /// ## Example
437    /// ```
438    /// use fast_neural_network::{activation::*, neural_network::*};
439    /// use ndarray::*;
440    ///
441    /// let mut network = Network::new(3, 1, ActivationType::Relu, 0.005);
442    ///
443    /// network.add_hidden_layer_with_size(4);
444    /// network.add_hidden_layer_with_size(4);
445    ///
446    /// network.compile();
447    ///
448    /// network.train(&[(array![2., 1., -1.], array![9.])], 100, 100);
449    /// ```
450    pub fn train(
451        &mut self,
452        training_set: &[(ndarray::Array1<f64>, ndarray::Array1<f64>)],
453        epochs: usize,
454        decay_time: usize,
455    ) {
456        if !self.compiled {
457            self.compile()
458        }
459
460        let m = MultiProgress::new();
461
462        let outer = m.add(ProgressBar::new(epochs as u64));
463        outer.set_style(
464            ProgressStyle::with_template(
465                "{spinner:.green} [{elapsed}] {wide_bar:.cyan/blue} {pos:>7}/{len:7} {msg} {eta}",
466            )
467            .unwrap()
468            .with_key("eta", |state: &ProgressState, weights: &mut dyn Write| {
469                write!(weights, "eta: {:.1}s", state.eta().as_secs_f64()).unwrap()
470            })
471            .progress_chars("#>-"),
472        );
473
474        outer.inc(0);
475
476        let mut time_since_last_decay = 0;
477        for _ in 0..epochs {
478            time_since_last_decay += 1;
479
480            let inner_pb = m.add(ProgressBar::new(training_set.len() as u64));
481            inner_pb.set_style(
482                ProgressStyle::default_bar()
483                    .template(
484                        "{spinner:.green} [{elapsed_precise}] {bar:.green} {pos:>7}/{len:7} {eta}",
485                    )
486                    .unwrap()
487                    .progress_chars("#>-")
488                    .with_key("eta", |state: &ProgressState, weights: &mut dyn Write| {
489                        write!(weights, "eta: {:.1}s", state.eta().as_secs_f64()).unwrap()
490                    }),
491            );
492
493            if time_since_last_decay >= decay_time {
494                self.learning_rate *= 0.95;
495                time_since_last_decay = 0;
496            }
497
498            let mut element_counter = 0;
499
500            inner_pb.inc(0);
501            for (input, target) in training_set {
502                element_counter += 1;
503
504                if element_counter % self.ui_update_interval == 0 {
505                    inner_pb.inc(self.ui_update_interval as u64);
506                }
507
508                let output = self.forward(input);
509
510                let output = output.to_shape(self.outputs.size).unwrap();
511
512                // dealing with the last layer
513                let mut dz = target - &output;
514
515                dz.iter_mut().zip(output.iter()).for_each(|(x, y)| {
516                    *x *= -2. * match self.activation {
517                        ActivationType::Sigmoid => der_sigm(y),
518                        ActivationType::Tanh => der_tanh(y),
519                        ActivationType::ArcTanh => der_arc_tanh(y),
520                        ActivationType::Relu => der_relu(y),
521                        ActivationType::LeakyRelu => der_leaky_relu(y),
522                        ActivationType::ELU => der_elu(y),
523                        ActivationType::Swish => der_swish(y),
524                        ActivationType::SoftMax => der_softmax(y, &output.to_owned()),
525                        ActivationType::SoftPlus => der_softplus(y),
526                    };
527                });
528
529                for i in (1..self.layer_matrices.len()).rev() {
530                    let a = &self.activation_matrices[i - 1];
531                    let (weights, bias) = &self.layer_matrices[i];
532
533                    let squared_a = a.to_shape((a.len(), 1)).unwrap();
534
535                    let dw = &dz * &squared_a;
536                    let db = &dz;
537
538                    let n_weights = weights - &dw.t().mapv(|x| x * self.learning_rate);
539                    let n_bias = bias - &db.mapv(|x| x * self.learning_rate);
540
541                    let da = weights.t().dot(&dz);
542
543                    let a_derivated = self.derivate(a.clone()); // check
544
545                    self.layer_matrices[i] = (n_weights, n_bias);
546
547                    dz = da * a_derivated;
548                }
549
550                // last iteration (first layer)
551                let (weights, bias) = &self.layer_matrices[0];
552
553                let a = input.to_shape((input.len(), 1)).unwrap();
554
555                let dz_size = dz.len();
556                let a_size = a.len();
557
558                let dw = dz.to_shape((dz_size, 1)).unwrap()
559                    * a.to_shape((a_size, 1)).unwrap().t();
560                let db = &dz;
561
562                let n_weights = weights - &dw.mapv(|x| x * self.learning_rate);
563                let n_bias = bias - &db.mapv(|x| x * self.learning_rate);
564
565                self.layer_matrices[0] = (n_weights, n_bias);
566            }
567            inner_pb.finish_and_clear();
568            outer.inc(1);
569        }
570        outer.finish_and_clear();
571        m.clear().unwrap();
572    }
573}
574
575/// Formats the network to be printed
576impl Display for Network {
577    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
578        let mut output = String::new();
579        output.push_str(&format!("Inputs: {}\n", self.inputs.size));
580        output.push_str(&format!("Outputs: {}\n", self.outputs.size));
581        output.push_str(&format!("Hidden layers: {}\n", self.hidden_layers.len()));
582        output.push_str(&format!("Activation: {:?}\n", self.activation));
583        output.push_str(&format!("Weights:\n"));
584        output.push_str(&format!("---------------------\n"));
585        for (i, (weights, biases)) in self.layer_matrices.iter().enumerate() {
586            output.push_str(&format!("Layer {}:\n", i));
587            output.push_str(&format!("Weights:{}\n", weights));
588            output.push_str(&format!("Biases:{}\n", biases));
589            output.push_str(&format!("---------------------"));
590        }
591        write!(f, "{}", output)
592    }
593}
fast_neural_network/neural_network.rs

fast_neural_network/
neural_network.rs
