kaffe 0.2.0

Pytorch, but make it rust :)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

//! Common optimizers for neural networks
#![warn(missing_docs)]

use std::{error::Error, marker::PhantomData, str::FromStr};

use crate::{Tensor, TensorElement};
use rayon::prelude::*;

/// Trait contains all the functions needed to run an optimizer
pub trait Optimizer<'a, T>
where
    T: TensorElement,
    <T as FromStr>::Err: Error,
{
    /// Initializes a new optimizer
    fn init(learning_rate: T, momentum: f32) -> Self;

    /// Function that minimizes based on cost function
    fn minimize<F>(&mut self, cost: F, vars: &mut Vec<T>)
    where
        F: Fn(&Tensor<'a, T>, &Tensor<'a, T>) -> T;
}

/// Adam Optimizer
///
/// # Examples
///
/// ```
/// use kaffe::Tensor;
/// use kaffe::nn::optimizer::Adam;
/// use kaffe::nn::optimizer::Optimizer;
///
/// let optim = Adam::init(1e-6, 0.8);
///
/// ```
pub struct Adam<T> {
    /// Learning rate
    lr: T,
    /// Momentum
    momentum: f32,
    /// Decay rate
    decay_rate: f32,
    /// Beta 1
    beta1: f32,
    /// Beta 2:
    beta2: f32,
    /// Epsilon
    epsilon: f32,
    m_dw: f32,
    v_dw: f32,
    m_db: f32,
    v_db: f32,
}

impl<'a, T> Optimizer<'a, T> for Adam<T>
where
    T: TensorElement,
    <T as FromStr>::Err: Error,
{
    fn init(learning_rate: T, momentum: f32) -> Self {
        Self {
            lr: learning_rate,
            momentum,
            decay_rate: 0.2,
            beta1: 0.9,
            beta2: 0.999,
            epsilon: 1e-8,
            m_dw: 0.0,
            m_db: 0.0,
            v_dw: 0.0,
            v_db: 0.0,
        }
    }

    fn minimize<F>(&mut self, cost: F, vars: &mut Vec<T>)
    where
        F: Fn(&Tensor<'a, T>, &Tensor<'a, T>) -> T,
    {
        unimplemented!()
    }
}

/// SGD
///
/// # Examples
///
/// ```
/// use kaffe::Tensor;
/// use kaffe::nn::optimizer::SGD;
/// use crate::kaffe::nn::optimizer::Optimizer;
///
/// let optim = SGD::init(1e-6, 0.8);
///
/// ```
pub struct SGD<T> {
    /// Learning rate
    lr: T,
    /// Momentum
    momentum: f32,
    /// Decay rate
    decay_rate: f32,
}

impl<'a, T> Optimizer<'a, T> for SGD<T>
where
    T: TensorElement,
    <T as FromStr>::Err: Error,
{
    fn init(learning_rate: T, momentum: f32) -> Self {
        Self {
            lr: learning_rate,
            momentum,
            decay_rate: 0.2,
        }
    }

    fn minimize<F>(&mut self, cost: F, vars: &mut Vec<T>)
    where
        F: Fn(&Tensor<'a, T>, &Tensor<'a, T>) -> T,
    {
        todo!()
    }
}