neuronika 0.2.0

Tensors and dynamic neural networks.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

use super::{super::L2, Adagrad};

#[test]
fn creation() {
    let optim = Adagrad::new(Vec::new(), 1e-2, 1e-3, L2::new(1e-2), 1e-10);

    assert_eq!(optim.params.borrow().len(), 0);
    assert!((optim.get_lr() - 1e-2).abs() <= f32::EPSILON);
    assert!((optim.get_lr_decay() - 1e-3).abs() <= f32::EPSILON);
    assert!((optim.get_eps() - 1e-10).abs() <= f32::EPSILON);
}

#[test]
fn set_lr() {
    let optim = Adagrad::new(Vec::new(), 1e-2, 1e-3, L2::new(1e-2), 1e-10);

    optim.set_lr(1e-3);
    assert!((optim.get_lr() - 1e-3).abs() <= f32::EPSILON);
}

#[test]
fn set_lr_decay() {
    let optim = Adagrad::new(Vec::new(), 1e-2, 1e-3, L2::new(1e-2), 1e-10);

    optim.set_lr_decay(1e-4);
    assert!((optim.get_lr_decay() - 1e-4).abs() <= f32::EPSILON);
}

#[test]
fn set_eps() {
    let optim = Adagrad::new(Vec::new(), 1e-2, 1e-3, L2::new(1e-2), 1e-10);

    optim.set_eps(1e-9);
    assert!((optim.get_eps() - 1e-9).abs() <= f32::EPSILON);
}

const EPOCHS: usize = 200;

#[test]
fn step() {
    let x = crate::rand((3, 3));
    let y = crate::rand((3, 3));
    let z = x.clone().mm(y);

    let w = crate::rand((3, 3)).requires_grad();
    let loss = (x.mm(w) - z).pow(2).sum();
    loss.forward();

    let first_value = loss.data().clone().into_scalar();
    let optim = Adagrad::new(loss.parameters(), 0.01, 1e-9, L2::new(0.0), 1e-10);

    for _ in 0..EPOCHS {
        loss.forward();
        loss.backward(1.0);

        optim.step();
        optim.zero_grad();
    }
    assert!(loss.data().clone().into_scalar() < first_value);
}