newron 0.5.1

A Rust library to train and infer deep learning models.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

use crate::layers::layer::Layer;
use crate::tensor::Tensor;
use crate::layers::layer::LearnableParams;

pub struct Dense {
    input: Tensor,
    weights: Tensor,
    biases: Tensor,
    weights_grad: Tensor,
    biases_grad: Tensor
}

impl Dense {
    pub fn new(input_units: usize, output_units: usize, seed: u32) -> Dense {
        // initialize with random values following special normal distribution
        // allowing theoritical faster convergence (Xavier Initialization)
        let variance_w = 2.0 / (input_units + output_units) as f64;
        let variance_b = 2.0 / (output_units) as f64;
        Dense {
            input: Tensor::new(vec![], vec![]),
            weights: Tensor::random_normal(vec![input_units, output_units], 0.0, variance_w, seed),
            biases: Tensor::random_normal(vec![1, output_units], 1.0, variance_b, seed),
            weights_grad: Tensor::new(vec![], vec![]),
            biases_grad: Tensor::new(vec![], vec![])
        }
    }
}

impl Layer for Dense {
    fn get_info(&self) -> String {
        format!("Dense layer ({}x{} = {} params)", self.weights.shape[0], self.weights.shape[1], self.weights.shape[0] * self.weights.shape[1])
    }

    fn forward(&mut self, input: Tensor, _training: bool) -> Tensor {
        // Perform an affine transformation:
        // f(x) = <W*x> + b
        
        // input shape: [batch, input_units]
        // output shape: [batch, output units]
        self.input = input;
        // panic!("input: {:?}", &self.input * &self.weights);

        &self.input * &self.weights + &self.biases
        // &self.input.dot(&self.weights) + &self.biases
    }

    fn backward(&mut self, gradient: &Tensor) -> Tensor {
        // compute d f / d x = d f / d dense * d dense / d x
        // where d dense/ d x = weights transposed
        // panic!("input.T {:?}  grad {:?}", &self.input.get_transpose().shape, gradient.shape);
        self.weights_grad = &self.input.get_transpose() * gradient;
        self.biases_grad = gradient.get_sum(0);
        
        assert_eq!(self.weights_grad.shape, self.weights.shape, "Wrong shape for weight gradients.");
        assert_eq!(self.biases_grad.shape, self.biases.shape, "Wrong shape for biases gradients.");

        let grad_input = gradient * &self.weights.get_transpose();
        grad_input
    }

    fn get_params_list(&self) -> Vec<LearnableParams> {
        vec![LearnableParams::Weights, LearnableParams::Biases]
    }

    fn get_grad(&self, param: &LearnableParams) -> &Tensor {
        match param {
            LearnableParams::Weights => {
                &self.weights_grad
            }
            LearnableParams::Biases => {
                &self.biases_grad
            }
        }
    }

    fn get_param(&mut self, param: &LearnableParams) -> &mut Tensor {
        match param {
            LearnableParams::Weights => {
                &mut self.weights
            }
            LearnableParams::Biases => {
                &mut self.biases
            }
        }
    }

}