1use burn::config::Config;
41use burn::module::AutodiffModule;
42use burn::optim::{adaptor::OptimizerAdaptor, LearningRate, SimpleOptimizer};
43use burn::record::Record;
44use burn::tensor::backend::{AutodiffBackend, Backend};
45use burn::tensor::{ElementConversion, Tensor};
46
47#[derive(Clone, Debug, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
49pub enum NormDir {
50 Col,
52 Row,
54 ColRow,
56 RowCol,
58}
59
60pub const NS_COEFFS: (f32, f32, f32) = (3.4445, -4.775, 2.0315);
62
63#[derive(Config, Debug)]
65pub struct MuonPlusConfig {
66 #[config(default = 0.95)]
68 pub momentum: f64,
69 #[config(default = 0.9)]
71 pub beta_1: f64,
72 #[config(default = 0.95)]
74 pub beta_2: f64,
75 #[config(default = 1e-8)]
77 pub epsilon: f64,
78 #[config(default = 5)]
80 pub ns_steps: usize,
81 #[config(default = "NS_COEFFS")]
83 pub ns_coeffs: (f32, f32, f32),
84 #[config(default = "None")]
88 pub norm_dir: Option<NormDir>,
89 #[config(default = 0.0)]
91 pub weight_decay: f64,
92}
93
94#[derive(Record, Clone, derive_new::new)]
98pub struct MuonPlusState<B: Backend, const D: usize> {
99 pub mu_momentum: Option<Tensor<B, D>>,
101 pub ad_moment_1: Option<Tensor<B, D>>,
103 pub ad_moment_2: Option<Tensor<B, D>>,
105 pub ad_time: Option<i64>,
107}
108
109#[derive(Clone)]
111pub struct MuonPlus<B: Backend> {
112 momentum: f64,
113 beta_1: f64,
114 beta_2: f64,
115 epsilon: f64,
116 ns_steps: usize,
117 ns_coeffs: (f32, f32, f32),
118 norm_dir: Option<NormDir>,
119 weight_decay: f64,
120 _backend: core::marker::PhantomData<B>,
121}
122
123impl<B: Backend> MuonPlus<B> {
124 pub fn orthogonalize<const D: usize>(&self, g: Tensor<B, D>) -> Tensor<B, D> {
129 let dims = g.dims();
130 let (rows, cols) = (dims[D - 2], dims[D - 1]);
131 let (mut x, transposed) = if rows > cols {
132 (g.swap_dims(D - 2, D - 1), true)
133 } else {
134 (g, false)
135 };
136
137 let norm = x.clone().powf_scalar(2.0).sum().sqrt().clamp_min(1e-7);
139 x = x.div(norm.unsqueeze());
140
141 let (a, b, c) = self.ns_coeffs;
142 for _ in 0..self.ns_steps {
143 let xt = x.clone().swap_dims(D - 2, D - 1);
144 let xx = x.clone().matmul(xt); let xx2 = xx.clone().matmul(xx.clone()); let poly = xx.mul_scalar(b).add(xx2.mul_scalar(c));
147 x = x.clone().mul_scalar(a).add(poly.matmul(x.clone()));
148 }
149
150 if transposed {
151 x.swap_dims(D - 2, D - 1)
152 } else {
153 x
154 }
155 }
156
157 pub fn normalize<const D: usize>(&self, x: Tensor<B, D>) -> Tensor<B, D> {
162 match self.norm_dir {
163 None => x,
164 Some(NormDir::Col) => Self::norm_col(x),
165 Some(NormDir::Row) => Self::norm_row(x),
166 Some(NormDir::ColRow) => Self::norm_row(Self::norm_col(x)),
167 Some(NormDir::RowCol) => Self::norm_col(Self::norm_row(x)),
168 }
169 }
170
171 fn norm_col<const D: usize>(x: Tensor<B, D>) -> Tensor<B, D> {
172 let col_norms = x
174 .clone()
175 .powf_scalar(2.0)
176 .sum_dim(D - 2)
177 .sqrt()
178 .clamp_min(1e-7);
179 x.div(col_norms)
180 }
181
182 fn norm_row<const D: usize>(x: Tensor<B, D>) -> Tensor<B, D> {
183 let row_norms = x
184 .clone()
185 .powf_scalar(2.0)
186 .sum_dim(D - 1)
187 .sqrt()
188 .clamp_min(1e-7);
189 x.div(row_norms)
190 }
191}
192
193impl<B: Backend> SimpleOptimizer<B> for MuonPlus<B> {
194 type State<const D: usize> = MuonPlusState<B, D>;
195
196 fn step<const D: usize>(
197 &self,
198 lr: LearningRate,
199 tensor: Tensor<B, D>,
200 grad: Tensor<B, D>,
201 state: Option<Self::State<D>>,
202 ) -> (Tensor<B, D>, Option<Self::State<D>>) {
203 let state = state.unwrap_or_else(|| MuonPlusState::new(None, None, None, None));
204
205 let (updated, state) = if D == 2 {
206 let mu = self.momentum.elem::<f32>();
208 let momentum = match &state.mu_momentum {
210 Some(m) => m
211 .clone()
212 .mul_scalar(mu)
213 .add(grad.clone().mul_scalar(1.0 - mu)),
214 None => grad.clone().mul_scalar(1.0 - mu),
215 };
216 let update = self.normalize(self.orthogonalize(momentum.clone()));
218
219 let dims = tensor.dims();
221 let (m, n) = (dims[0] as f64, dims[1] as f64);
222 let lr_scaled = lr * (m / n).sqrt();
223
224 let decayed = tensor
225 .clone()
226 .mul_scalar(1.0 - (self.weight_decay as f32 * lr_scaled as f32).min(0.999));
227 (
228 decayed.sub(update.mul_scalar(lr_scaled as f32)),
229 MuonPlusState::new(Some(momentum), None, None, None),
230 )
231 } else {
232 let b1 = self.beta_1.elem::<f32>();
234 let b2 = self.beta_2.elem::<f32>();
235 let eps = self.epsilon.elem::<f32>();
236 let t = state.ad_time.unwrap_or(0) + 1;
237
238 let m = match &state.ad_moment_1 {
239 Some(m) => m
240 .clone()
241 .mul_scalar(b1)
242 .add(grad.clone().mul_scalar(1.0 - b1)),
243 None => grad.clone().mul_scalar(1.0 - b1),
244 };
245 let v = match &state.ad_moment_2 {
246 Some(v) => v
247 .clone()
248 .mul_scalar(b2)
249 .add(grad.clone().powf_scalar(2.0).mul_scalar(1.0 - b2)),
250 None => grad.clone().powf_scalar(2.0).mul_scalar(1.0 - b2),
251 };
252
253 let m_hat = m.clone().div_scalar(1.0 - b1.powi(t as i32));
254 let v_hat = v.clone().div_scalar(1.0 - b2.powi(t as i32));
255 let step = m_hat.div(v_hat.sqrt().add_scalar(eps));
256
257 let decayed = tensor
258 .clone()
259 .mul_scalar(1.0 - (self.weight_decay as f32 * lr as f32).min(0.999));
260 (
261 decayed.sub(step.mul_scalar(lr as f32)),
262 MuonPlusState::new(None, Some(m), Some(v), Some(t)),
263 )
264 };
265 (updated, Some(state))
266 }
267
268 fn to_device<const D: usize>(
269 mut state: Self::State<D>,
270 device: &burn::tensor::ops::Device<B>,
271 ) -> Self::State<D> {
272 state.mu_momentum = state.mu_momentum.map(|t| t.to_device(device));
273 state.ad_moment_1 = state.ad_moment_1.map(|t| t.to_device(device));
274 state.ad_moment_2 = state.ad_moment_2.map(|t| t.to_device(device));
275 state
276 }
277}
278
279impl MuonPlusConfig {
280 pub fn build<B: Backend>(&self) -> MuonPlus<B> {
282 MuonPlus {
283 momentum: self.momentum,
284 beta_1: self.beta_1,
285 beta_2: self.beta_2,
286 epsilon: self.epsilon,
287 ns_steps: self.ns_steps,
288 ns_coeffs: self.ns_coeffs,
289 norm_dir: self.norm_dir,
290 weight_decay: self.weight_decay,
291 _backend: core::marker::PhantomData,
292 }
293 }
294
295 pub fn init<B: AutodiffBackend, M: AutodiffModule<B>>(
297 &self,
298 ) -> OptimizerAdaptor<MuonPlus<B::InnerBackend>, M, B> {
299 OptimizerAdaptor::from(self.build())
300 }
301}