ruda-optim 0.21.43

Ruda optimizer updates, gradient state, clipping and learning-rate schedules.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

/// Weight decay module for optimizers.
pub mod decay;

/// Momentum module for optimizers.
pub mod momentum;

mod adagrad;
mod adam;
mod adamw;
mod adan;
mod base;
mod grad_accum;
mod gradient_transform;
mod weighted_accum;
mod fully_sharded_accum;
mod elementwise_shard;
mod native_coordinate;
mod fully_sharded_elementwise;
mod fp32_master;
mod checkpoint_partition;
mod grads;
mod lbfgs;
mod muon;
mod rmsprop;
mod sgd;
mod simple;
mod visitor;

pub use adagrad::*;
pub use adam::*;
pub use adamw::*;
pub use adan::*;
pub use base::*;
pub use grad_accum::*;
pub use gradient_transform::*;
pub use weighted_accum::*;
pub use fully_sharded_accum::*;
pub use elementwise_shard::*;
pub use native_coordinate::*;
pub use fully_sharded_elementwise::*;
pub use fp32_master::*;
pub use checkpoint_partition::*;
pub use grads::*;
pub use lbfgs::*;
pub use muon::*;
pub use rmsprop::*;
pub use sgd::*;
pub use simple::*;