ruda-optim 0.21.72

Ruda optimizer updates, gradient state, clipping and learning-rate schedules.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86


use super::GradientsParams;
use ruda_model::module::{AutodiffModule, ModuleVisitor, Param, ParamId};
use ruda_model::tensor::{Tensor, backend::AutodiffBackend};
use ruda_model::tensor::ops::FloatTensorOps;
use core::marker::PhantomData;
use hashbrown::HashSet;

#[cfg(not(feature = "std"))]
use alloc::vec::Vec;

pub struct GradientsParamsConverter<'a, M: AutodiffModule<B>, B: AutodiffBackend> {
    grads: &'a mut B::Gradients,
    grads_params: &'a mut GradientsParams,
    phatom: PhantomData<M>,
    filter: Option<HashSet<ParamId>>,
}

impl<'a, M: AutodiffModule<B>, B: AutodiffBackend> GradientsParamsConverter<'a, M, B> {
    /// Create a gradient visitor with an optional parameter-ID selection.
    pub fn new(
        grads: &'a mut B::Gradients,
        grads_params: &'a mut GradientsParams,
        filter: Option<Vec<ParamId>>,
    ) -> Self {
        Self {
            grads,
            grads_params,
            phatom: PhantomData,
            filter: filter.map(|ids| ids.into_iter().collect()),
        }
    }
}

#[derive(new)]
pub struct GradientsParamsChangeDevice<'a, M: AutodiffModule<B>, B: AutodiffBackend> {
    device: &'a B::Device,
    grads: &'a mut GradientsParams,
    phatom: PhantomData<M>,
}

impl<B, M> ModuleVisitor<B> for GradientsParamsConverter<'_, M, B>
where
    B: AutodiffBackend,
    M: AutodiffModule<B>,
{
    fn visit_float<const D: usize>(&mut self, param: &Param<Tensor<B, D>>) {
        use ruda_model::tensor::TensorPrimitive;
        let mut extract = |id: ParamId, value: &dyn core::any::Any| {
            if self.filter.as_ref().is_some_and(|filter| !filter.contains(&id)) { return; }
            let value = value.downcast_ref::<TensorPrimitive<B>>()
                .expect("Reparameterization parameter backend mismatch");
            let grad = match value {
                TensorPrimitive::Float(value) => {
                    if !B::float_is_require_grad(value) { return; }
                    B::grad_remove(value, self.grads)
                }
                TensorPrimitive::QFloat(value) => B::q_grad_remove(value, self.grads),
            };
            let Some(grad) = grad else { return; };
            let grad = match self.grads_params.container.remove::<B::InnerBackend>(&id) {
                Some(TensorPrimitive::Float(previous)) => B::InnerBackend::float_add(previous, grad),
                Some(TensorPrimitive::QFloat(_)) => panic!("Parameter gradient cannot be quantized"),
                None => grad,
            };
            self.grads_params.container.register::<B::InnerBackend>(id, TensorPrimitive::Float(grad));
        };
        extract(param.id, &param.base().into_primitive());
        param.visit_reparameterization_primitives(&mut extract);
    }
}

impl<B, M> ModuleVisitor<B> for GradientsParamsChangeDevice<'_, M, B>
where
    B: AutodiffBackend,
    M: AutodiffModule<B>,
{
    fn visit_float<const D: usize>(&mut self, param: &Param<Tensor<B, D>>) {
        let Some(grad) = self.grads.remove::<B::InnerBackend, D>(param.id) else {
            return;
        };

        self.grads
            .register::<B::InnerBackend, D>(param.id, grad.to_device(self.device));
    }
}