kaffe 0.2.0

Pytorch, but make it rust :)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

//! This module contains the most used non-linear activation functions
//!
//! Sigmoid and variants of ReLU are for now the ones implemented
#![warn(missing_docs)]

use std::error::Error;
use std::str::FromStr;

use crate::constants::{E, PI};
use crate::{Tensor, TensorElement};
use rayon::prelude::*;

/// ReLU is the most used activation funcion besides Sigmoid
///
/// # Examples
///
/// ```
/// use kaffe::Tensor;
/// use kaffe::nn::activation::ReLU;
///
/// let matrix = Tensor::new(vec![-1.0, -2.0, 1.0, 2.0], vec![2,2]).unwrap();
///
/// assert_eq!(ReLU(&matrix).data, vec![0.0, 0.0, 1.0, 2.0]);
/// ```
pub fn ReLU<'a, 'b, T>(x: &Tensor<'a, T>) -> Tensor<'b, T>
where
    T: TensorElement,
    <T as FromStr>::Err: Error,
    Vec<T>: FromParallelIterator<T>,
{
    let data: Vec<T> = x
        .data
        .par_iter()
        .map(|&e| if e >= T::zero() { e } else { T::zero() })
        .collect();

    Tensor::new(data, x.shape.clone()).unwrap()
}

/// PReLU is a slight modification to ReLU
///
/// # Examples
///
/// ```
/// use kaffe::Tensor;
/// use kaffe::nn::activation::PReLU;
///
/// let matrix = Tensor::new(vec![-1.0, -2.0, 1.0, 2.0], vec![2,2]).unwrap();
///
/// assert_eq!(PReLU(&matrix, -1.0).data, vec![1.0, 2.0, 1.0, 2.0]);
/// ```
pub fn PReLU<'a, 'b, T>(x: &Tensor<'a, T>, alpha: T) -> Tensor<'b, T>
where
    T: TensorElement,
    <T as FromStr>::Err: Error,
    Vec<T>: FromParallelIterator<T>,
{
    let data: Vec<T> = x
        .data
        .par_iter()
        .map(|&e| if e >= T::zero() { e } else { e * alpha })
        .collect();

    Tensor::new(data, x.shape.clone()).unwrap()
}

/// Sigmoid function
///
/// # Examples
///
/// ```
/// use kaffe::Tensor;
/// use kaffe::nn::activation::Sigmoid;
///
/// let matrix = Tensor::new(vec![1.0, 2.0, 1.0, 2.0], vec![2,2]).unwrap();
///
/// // assert_eq!(Sigmoid(&matrix).data, vec![1.0, 2.0, 1.0, 2.0]);
/// ```
pub fn Sigmoid<'a, T>(x: &Tensor<'a, T>) -> Tensor<'a, T>
where
    T: TensorElement,
    <T as FromStr>::Err: Error,
    Vec<T>: FromParallelIterator<T>,
{
    let data: Vec<T> = x
        .data
        .par_iter()
        .map(|&e| T::from(E).unwrap().powf(e) / (T::from(E).unwrap().powf(e) + T::one()))
        .collect();

    Tensor::new(data, x.shape.clone()).unwrap()
}

/// GeLU activation function
///
/// 0.5x (1 + tanh[sqrt(2/pi) * (x + 0.044715x^3)])
///
/// # Examples
///
/// ```
/// use kaffe::Tensor;
/// use kaffe::nn::activation::GeLU;
///
/// let matrix = Tensor::new(vec![1.0, 2.0, 1.0, 2.0], vec![2,2]).unwrap();
///
/// // assert_eq!(GeLU(&matrix).data, vec![1.0, 2.0, 1.0, 2.0]);
/// ```
pub fn GeLU<'a, T>(x: &Tensor<'a, T>) -> Tensor<'a, T>
where
    T: TensorElement,
    <T as FromStr>::Err: Error,
{
    let lhs = x.mul_val(T::from(0.5).unwrap());

    let inner = x
        .pow(3)
        .mul_val(T::from(0.044715).unwrap())
        .add(x)
        .unwrap()
        .mul_val((T::from(2).unwrap() / T::from(PI).unwrap()).sqrt());

    let result = lhs.mul(&inner.tanh().add_val(T::from(1).unwrap())).unwrap();

    return result;
}