hal-ml 0.2.0

HAL: a machine learning library that is able to run on Nvidia, OpenCL or CPU BLAS based compute backends. It currently provides stackable classical neural networks, RNN's and soon to be LSTM's. A differentiation of this package is that we are looking to implement RTRL (instead of just BPTT) for the recurrent layers in order to provide a solid framework for online learning. We will also (in the future) be implementing various layers such as unitary RNN's, NTM's and Adaptive Computation time based LSTM's. HAL also comes with the ability to plot and do many basic math operations on arrays.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155

use af;
use af::{Array};
use num::Complex;

use utils;
use error::HALError;

/// Returns the tanh activated value
pub fn tanh(x: &Array) -> Array {
  af::tanh(x)
}

/// Returns the sigmoid activated value
/// 1.0/(1.0 + exp(-1.0 * e))
pub fn sigmoid(x: &Array) -> Array {
  let neg_one = utils::constant(x.dims(), x.get_type(), -1.0f32);
  let one = utils::constant(x.dims(), x.get_type(), 1.0f32);
  let exp_m_e = af::exp(&af::mul(&neg_one, x, false));
  let denominator = af::add(&one, &exp_m_e, false);
  let a = af::div(&one, &denominator, false);
  utils::assert_types(vec![x, &a]);
  a
}

/// Return the softmax activated value
/// exp(x_i) / sum(exp(x))
pub fn softmax(x: &Array) -> Array {
  let exponentiated = af::exp(&x);
  let sum_epx_x = af::sum_all(&exponentiated).0 as f32;
  let sum_exp_x_vec = utils::constant(x.dims(), x.get_type(), sum_epx_x);
  let a = af::div(&exponentiated, &sum_exp_x_vec, false);
  utils::assert_types(vec![x, &a]);
  a
}

/// Return the lrelu activated value
/// max(0.01*x, x)
pub fn lrelu(x: &Array) -> Array {
  let zero_one = utils::constant(x.dims(), x.get_type(), 0.01f32);
  let scaled = af::mul(x, &zero_one, false);
  let a = af::select(&scaled                               // return 0.01x
                     , &af::lt(x, &0.0f32, false) // if x > 0.0
                     , x);                        // else x
  utils::assert_types(vec![x, &a]);
  a
}

/// Return the derivative of relu [assumes that relu has already been applied]
/// 0.01     for x <= 0
/// 1        otherwise
pub fn lrelu_derivative(x: &Array) -> Array {
  let x_lt_zero = utils::constant(x.dims(), x.get_type(), 0.01f32);
  let one = utils::constant(x.dims(), x.get_type(), 1.0f32);
  let grad = af::select(&one, &af::gt(x, &0.0f32, false), &x_lt_zero);
  utils::assert_types(vec![x, &grad]);
  grad
}

/// Return the relu activated value
/// max(0, x)
pub fn relu(x: &Array) -> Array {
  let zero = utils::constant(x.dims(), x.get_type(), 0.0f32);
  let a = af::select(&zero, &af::lt(x, &0.0, false), x);
  utils::assert_types(vec![x, &a]);
  a
}

/// Return the derivative of relu [assumes that relu has already been applied]
/// 0 for x <= 0
/// 1 otherwise
pub fn relu_derivative(x: &Array) -> Array {
  let zero = utils::constant(x.dims(), x.get_type(), 0.0f32);
  let one = utils::constant(x.dims(), x.get_type(), 1.0f32);
  let grad = af::select(&one, &af::gt(x, &0.0f32, false), &zero);
  utils::assert_types(vec![x, &grad]);
  grad
}

/// Return the derivative of tanh [assumes that tanh has already been applied]
/// 1 - tanh(x)*tanh(x)
pub fn tanh_derivative(x: &Array) -> Array {
  let one = utils::constant(x.dims(), x.get_type(), 1.0f32);
  let grad = af::sub(&one, &af::mul(x, x, false), false);
  utils::assert_types(vec![x, &grad]);
  grad
}

/// Returns the derivative of sigmoid [assumes that sigmoid is already applied]
/// x * (1 - x)
pub fn sigmoid_derivative(x: &Array) -> Array {
  let one = utils::constant(x.dims(), x.get_type(), 1.0f32);
  let grad = af::mul(x, &af::sub(&one, x, false), false);
  utils::assert_types(vec![x, &grad]);
  grad
}

/// Returns the derivative of softmax [assumes that it was already applied]
/// x * (1 - x)
pub fn softmax_derivative(x: &Array) -> Array {
  sigmoid_derivative(x)
}

/// Returns a linear activation [no non-linearity]
pub fn ones(x: &Array) -> Array {
  x.clone()
}

/// Returns a derivative of a linear activation (1's)
pub fn ones_derivative(x: &Array) -> Array {
  let grad = utils::constant(x.dims(), x.get_type(), 1.0f32);
  utils::assert_types(vec![x, &grad]);
  grad
}

/// Helper to determine whether function is smooth or non-smooth
pub fn is_smooth(name: &str) -> bool {
  match name {
    "softmax" => true,
    "sigmoid" => true,
    "relu"    => false,
    "lrelu"   => false,
    "tanh"    => true,
    "ones"    => true,
    "linear"  => true,
    _         => panic!("unknown function name provided"),
  }
}

/// Helper to get the correct activation using a string
pub fn get_activation(name: &str, x: &Array) -> Result<Array, HALError> {
  match name {
    "softmax" => Ok(softmax(x)),
    "sigmoid" => Ok(sigmoid(x)),
    "relu"    => Ok(relu(x)),
    "lrelu"   => Ok(lrelu(x)),
    "tanh"    => Ok(tanh(x)),
    "ones"    => Ok(ones(x)),
    "linear"  => Ok(ones(x)),
    _         => Err(HALError::UNKNOWN),
  }
}

/// Helper to get the correct activation derivative using a string
pub fn get_derivative(name: &str, x: &Array) -> Result<Array, HALError> {
  match name {
    "softmax" => Ok(softmax_derivative(x)),
    "sigmoid" => Ok(sigmoid_derivative(x)),
    "relu"    => Ok(relu_derivative(x)),
    "lrelu"   => Ok(lrelu_derivative(x)),
    "tanh"    => Ok(tanh_derivative(x)),
    "ones"    => Ok(ones_derivative(x)),
    "linear"  => Ok(ones_derivative(x)),
    _         => Err(HALError::UNKNOWN),
  }
}