luma-optim 0.1.0

Optimizer for update grads
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

use luma_io::lpk::LumaPack;
use luma_tensor::{Device, GradStore, Scalar, Tensor, no_grad};

use super::Optimizer;

pub struct SGD<D: Device> {
    pub params: Vec<Tensor<D>>,
    pub learning_rate: f64,
}

impl<D: Device> SGD<D> {
    pub fn new(params: impl Into<Vec<Tensor<D>>>, learning_rate: f64) -> Self {
        Self { params: params.into(), learning_rate }
    }
}

impl<D: Device> Optimizer for SGD<D> {
    type Device = D;

    fn get_lr(&self) -> f64 {
        self.learning_rate
    }

    fn set_lr(&mut self, lr: f64) {
        self.learning_rate = lr;
    }

    /// w_t = w_{t-1} - lr * g
    fn step(&mut self, grads: &GradStore<Self::Device>) -> luma_tensor::Result<()> {
        no_grad!();
        for var in self.params.iter() {
            if let Some(grad) = grads.get(var) {
                var.sub_(&grad.mul_scalar(self.learning_rate)?)?;
            }
        }
        Ok(())
    }

    fn state_dict(&self) -> luma_tensor::Result<LumaPack<Self::Device>> {
        let mut pack = LumaPack::new();
        pack.scalars.insert("lr".into(), Scalar::F64(self.learning_rate));
        Ok(pack)
    }

    fn load_state_dict(&mut self, pack: &LumaPack<Self::Device>) -> luma_tensor::Result<()> {
        if let Some(lr) = pack.scalars.get("lr").and_then(|s| s.to_f64()) {
            self.learning_rate = lr;
        }
        Ok(())
    }
}

pub struct SGDM<D: Device> {
    pub params: Vec<SGDMParam<D>>,
    pub learning_rate: f64,
    pub momentum: f64,
}

pub struct SGDMParam<D: Device> {
    pub param: Tensor<D>,
    pub velocity: Tensor<D>,
}

impl<D: Device> Optimizer for SGDM<D> {
    type Device = D;

    fn get_lr(&self) -> f64 {
        self.learning_rate
    }

    fn set_lr(&mut self, lr: f64) {
        self.learning_rate = lr;
    }

    ///
    /// v = m*v + grad
    ///
    /// w_t = w_{t-1} - lr * v
    ///
    fn step(&mut self, grads: &GradStore<Self::Device>) -> luma_tensor::Result<()> {
        no_grad!();
        for SGDMParam { param, velocity } in self.params.iter_mut() {
            if let Some(grad) = grads.get(&param) {
                // update v: v = m * v + grad
                velocity.mul_scalar_(self.momentum)?;
                velocity.add_(grad)?;
                param.sub_(&velocity.mul_scalar(self.learning_rate)?)?;
            }
        }

        Ok(())
    }

    fn state_dict(&self) -> luma_tensor::Result<LumaPack<Self::Device>> {
        let mut pack = LumaPack::new();
        for (i, p) in self.params.iter().enumerate() {
            pack.tensors.insert(format!("{i}.velocity"), luma_tensor::DynTensor::Float(p.velocity.clone()));
        }
        pack.scalars.insert("lr".into(), Scalar::F64(self.learning_rate));
        pack.scalars.insert("momentum".into(), Scalar::F64(self.momentum));
        Ok(pack)
    }

    fn load_state_dict(&mut self, pack: &LumaPack<Self::Device>) -> luma_tensor::Result<()> {
        if let Some(lr) = pack.scalars.get("lr").and_then(|s| s.to_f64()) {
            self.learning_rate = lr;
        }
        if let Some(m) = pack.scalars.get("momentum").and_then(|s| s.to_f64()) {
            self.momentum = m;
        }
        for (i, p) in self.params.iter_mut().enumerate() {
            let key = format!("{i}.velocity");
            if let Some(dt) = pack.tensors.get(&key) {
                if let Some(src) = dt.as_float() {
                    p.velocity.copy_(src)?;
                }
            }
        }
        Ok(())
    }
}