rail 0.3.1

A GPU-Accelerated ML Library akin to Keras
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156

use rand::distributions::Normal;
use rand::prelude::*;

use crate::{ Weight, Matrix };
use crate::utils::map_array;

use std::fmt::{ self, Display };

#[derive(Copy, Clone)]
pub enum Activation {
    Relu,
    Tanh,
    Sigmoid,
    Softmax
}

impl From<&str> for Activation {
    fn from(src: &str) -> Self {
        match src {
            "Relu"    => Activation::Relu,
            "Tanh"    => Activation::Tanh,
            "Sigmoid" => Activation::Sigmoid,
            "Softmax" => Activation::Softmax,
            other       => panic!("invalid activation function {:?}", other)
        }
    }
}

impl Display for Activation {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            Activation::Relu    => write!(f, "Relu"),
            Activation::Tanh    => write!(f, "Tanh"),
            Activation::Sigmoid => write!(f, "Sigmoid"),
            Activation::Softmax => write!(f, "Softmax")
        }
    }
}

pub fn initialize_weights(activation: Activation, inputs: usize, neurons: usize) -> (Vec<Weight>, Vec<Weight>) {
    let normal = Normal::new(0.0, 1.0);
    let mut rng = rand::thread_rng();

    let c = match activation {
        Activation::Relu => 2.0,
        Activation::Tanh => 1.0,
        Activation::Sigmoid => 1.0,
        Activation::Softmax => 1.0
    };
    
    let weight_values = (0..inputs * neurons)
        .map(|_| (normal.sample(&mut rng) * c))
        .collect::<Vec<Weight>>();

    let bias_values = (0..inputs * neurons)
        .map(|_| (normal.sample(&mut rng) * c))
        .collect::<Vec<Weight>>();

    (weight_values, bias_values)
}

pub fn relu(input: Matrix) -> Matrix {
    map_array(input, |d| Weight::max(0.0, *d))
}

pub fn drelu(input: Matrix) -> Matrix {
    let dims = input.dims();
    let size = dims[0] * dims[1] * dims[2] * dims[3];
    let buf = vec![1.0; size as usize];
    Matrix::new(&buf, dims)
}


pub fn tanh(input: Matrix) -> Matrix {
    arrayfire::tanh(&input)
}

pub fn dtanh(input: Matrix) -> Matrix {
    map_array(input, |v| 1.0 - v.powf(2.0))
}

pub fn sigmoid(input: Matrix) -> Matrix {
    arrayfire::sigmoid(&input)
}

pub fn dsigmoid(input: Matrix) -> Matrix {
    map_array(input, |v| v * ( 1.0 - v ))
}


pub fn softmax(input: Matrix) -> Matrix {
    let exps = arrayfire::exp(&input);
    let sums = arrayfire::sum(&exps, 1);
    arrayfire::div(&exps, &sums, true)
}

pub fn dsoftmax(input: Matrix) -> Matrix {
    //  zi ((i == j) - zj)
    input
}




#[cfg(test)]
mod tests {
    use arrayfire::Dim4;
    use crate::utils::array_data;
    use super::*;

    #[test]
    fn relu_test() {
        let data = Matrix::new(&[1.0, 0.0, -1.0, 0.4], Dim4::new(&[4, 1, 1, 1]));
        let data2 = array_data(relu(data));
        assert_eq!(data2, &[1.0, 0.0, 0.0, 0.4]);

        let data = Matrix::new(&[-1.0, 1.0, -0.2, 0.9], Dim4::new(&[4, 1, 1, 1]));
        let data2 = array_data(relu(data));
        assert_eq!(data2, &[0.0, 1.0, 0.0, 0.9]);
    }

    #[test]
    fn softmax_test() {
        let data = Matrix::new(&[-1.0, 1.0], Dim4::new(&[1,2,1,1]));
        let expected_1 = array_data(softmax(data));

        let data = Matrix::new(&[0.2, 0.4], Dim4::new(&[1,2,1,1]));
        let expected_2 = array_data(softmax(data));

        let data = Matrix::new(&[-1.0, 1.0, 0.2, 0.4], Dim4::new(&[1,2,2,1]));
        let data2 = softmax(data);

        let d = array_data(data2);

        let mut expected = Vec::new();
        expected.extend_from_slice(&expected_1);
        expected.extend_from_slice(&expected_2);
        assert_eq!(d, expected);
    }


    #[test]
    fn accel_softmax_text() {
        let data = Matrix::new(&[-1.0, 1.0, 0.2, 0.4], Dim4::new(&[1,2,2,1]));
        let expected = array_data(softmax(data));

        let data = Matrix::new(&[-1.0, 1.0, 0.2, 0.4], Dim4::new(&[1,2,2,1]));
        let exps = arrayfire::exp(&data);
        let sums = arrayfire::sum(&exps, 1);
        let sfmx = arrayfire::div(&exps, &sums, true);
        assert_eq!(expected, array_data(sfmx));

    }
}