Skip to main content

burn_muon_plus/
lib.rs

1//! # burn-muon-plus — Muon+ optimizer for Burn
2//!
3//! | arXiv | Component | What |
4//! |-------|-----------|------|
5//! | [2602.21545](https://arxiv.org/abs/2602.21545) | `Muon+` | Muon + one post-polar normalization step (up to 37% pre-training speedup, zero extra state) |
6//!
7//! # Muon+
8//!
9//! Newton–Schulz polar iterations flatten the singular spectrum of the
10//! momentum matrix, but in practice they **amplify** column/row norm imbalance
11//! in the update ("post-polar imbalanced update problem", 2602.21545 §2.1).
12//! This imbalance tightens the second-order term in a blockwise descent
13//! analysis and shrinks Muon's largest stable step size.
14//!
15//! Muon+ inserts a single normalization step after orthogonalization:
16//!
17//! ```text
18//! M_t  = μ·M_{t-1} + (1-μ)·G_t
19//! O_t  = Norm_(d)(Ortho(M_t))      # NS polar + row/col normalization
20//! W_t  = W_{t-1} - η·√(m/n)·O_t
21//! ```
22//!
23//! - `Norm_col(X) = X·D_col⁻¹` (unit-L2 columns)
24//! - `Norm_row(X) = D_row⁻¹·X` (unit-L2 rows)
25//! - `ColRow`/`RowCol` = sequential composition (best in the paper)
26//!
27//! Non-matrix parameters (biases, norms, embeddings) use AdamW — the paper's
28//! recipe (§3.1), matching Muon's convention.
29//!
30//! # Example
31//!
32//! ```ignore
33//! use burn_muon_plus::{MuonPlusConfig, NormDir};
34//!
35//! let mut optimizer = MuonPlusConfig::new()
36//!     .with_norm_dir(NormDir::ColRow)   // paper's best
37//!     .init();
38//! let model = optimizer.step(lr, model, grads);
39//! ```
40use burn::config::Config;
41use burn::module::AutodiffModule;
42use burn::optim::{adaptor::OptimizerAdaptor, LearningRate, SimpleOptimizer};
43use burn::record::Record;
44use burn::tensor::backend::{AutodiffBackend, Backend};
45use burn::tensor::{ElementConversion, Tensor};
46
47/// Muon+ post-polar normalization direction (2602.21545).
48#[derive(Clone, Debug, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
49pub enum NormDir {
50    /// Normalize each column of the orthogonalized update to unit L2 norm.
51    Col,
52    /// Normalize each row of the orthogonalized update to unit L2 norm.
53    Row,
54    /// Column then row — paper's best single combination.
55    ColRow,
56    /// Row then column.
57    RowCol,
58}
59
60/// Newton-Schulz quintic coefficients (Keller Jordan, 2602.21545 uses them).
61pub const NS_COEFFS: (f32, f32, f32) = (3.4445, -4.775, 2.0315);
62
63/// Configuration for the hybrid Muon+ (2D) + AdamW (1D) optimizer.
64#[derive(Config, Debug)]
65pub struct MuonPlusConfig {
66    /// Momentum for 2D Muon updates. Default 0.95 (paper).
67    #[config(default = 0.95)]
68    pub momentum: f64,
69    /// AdamW beta_1 for the 1D fallback group.
70    #[config(default = 0.9)]
71    pub beta_1: f64,
72    /// AdamW beta_2 for the 1D fallback group.
73    #[config(default = 0.95)]
74    pub beta_2: f64,
75    /// AdamW epsilon for the 1D fallback group.
76    #[config(default = 1e-8)]
77    pub epsilon: f64,
78    /// Newton-Schulz iterations. Default 5 (paper).
79    #[config(default = 5)]
80    pub ns_steps: usize,
81    /// Newton-Schulz quintic coefficients.
82    #[config(default = "NS_COEFFS")]
83    pub ns_coeffs: (f32, f32, f32),
84    /// Muon+ post-polar normalization direction. `None` = plain Muon.
85    ///
86    /// The paper consistently prefers bidirectional normalization (`ColRow`).
87    #[config(default = "None")]
88    pub norm_dir: Option<NormDir>,
89    /// Decoupled weight decay (applied to both groups).
90    #[config(default = 0.0)]
91    pub weight_decay: f64,
92}
93
94/// State of the hybrid optimizer for one parameter.
95///
96/// 2D params use Muon momentum; 1D params use AdamW moments.
97#[derive(Record, Clone, derive_new::new)]
98pub struct MuonPlusState<B: Backend, const D: usize> {
99    /// Muon momentum.
100    pub mu_momentum: Option<Tensor<B, D>>,
101    /// AdamW first moment (1D group).
102    pub ad_moment_1: Option<Tensor<B, D>>,
103    /// AdamW second moment (1D group).
104    pub ad_moment_2: Option<Tensor<B, D>>,
105    /// AdamW step counter (1D group).
106    pub ad_time: Option<i64>,
107}
108
109/// Hybrid Muon+ (2D) + AdamW (1D) optimizer.
110#[derive(Clone)]
111pub struct MuonPlus<B: Backend> {
112    momentum: f64,
113    beta_1: f64,
114    beta_2: f64,
115    epsilon: f64,
116    ns_steps: usize,
117    ns_coeffs: (f32, f32, f32),
118    norm_dir: Option<NormDir>,
119    weight_decay: f64,
120    _backend: core::marker::PhantomData<B>,
121}
122
123impl<B: Backend> MuonPlus<B> {
124    /// Orthogonalize `g` via Newton-Schulz (zeroth power, tall-matrix aware).
125    ///
126    /// Public for testing: returns the nearest-orthogonal approximation of
127    /// `g` under the Frobenius norm (2602.21545 §1, Eq. 1).
128    pub fn orthogonalize<const D: usize>(&self, g: Tensor<B, D>) -> Tensor<B, D> {
129        let dims = g.dims();
130        let (rows, cols) = (dims[D - 2], dims[D - 1]);
131        let (mut x, transposed) = if rows > cols {
132            (g.swap_dims(D - 2, D - 1), true)
133        } else {
134            (g, false)
135        };
136
137        // Normalize to unit Frobenius norm.
138        let norm = x.clone().powf_scalar(2.0).sum().sqrt().clamp_min(1e-7);
139        x = x.div(norm.unsqueeze());
140
141        let (a, b, c) = self.ns_coeffs;
142        for _ in 0..self.ns_steps {
143            let xt = x.clone().swap_dims(D - 2, D - 1);
144            let xx = x.clone().matmul(xt); // X X^T
145            let xx2 = xx.clone().matmul(xx.clone()); // (X X^T)²
146            let poly = xx.mul_scalar(b).add(xx2.mul_scalar(c));
147            x = x.clone().mul_scalar(a).add(poly.matmul(x.clone()));
148        }
149
150        if transposed {
151            x.swap_dims(D - 2, D - 1)
152        } else {
153            x
154        }
155    }
156
157    /// Muon+ post-polar normalization (2602.21545 §2.3).
158    ///
159    /// `Norm_col(X) = X·D_col⁻¹`, `Norm_row(X) = D_row⁻¹·X`; compositions are
160    /// applied sequentially. Returns `x` unchanged when `norm_dir` is `None`.
161    pub fn normalize<const D: usize>(&self, x: Tensor<B, D>) -> Tensor<B, D> {
162        match self.norm_dir {
163            None => x,
164            Some(NormDir::Col) => Self::norm_col(x),
165            Some(NormDir::Row) => Self::norm_row(x),
166            Some(NormDir::ColRow) => Self::norm_row(Self::norm_col(x)),
167            Some(NormDir::RowCol) => Self::norm_col(Self::norm_row(x)),
168        }
169    }
170
171    fn norm_col<const D: usize>(x: Tensor<B, D>) -> Tensor<B, D> {
172        // sum_dim keeps the axis (size 1) → broadcasts directly.
173        let col_norms = x
174            .clone()
175            .powf_scalar(2.0)
176            .sum_dim(D - 2)
177            .sqrt()
178            .clamp_min(1e-7);
179        x.div(col_norms)
180    }
181
182    fn norm_row<const D: usize>(x: Tensor<B, D>) -> Tensor<B, D> {
183        let row_norms = x
184            .clone()
185            .powf_scalar(2.0)
186            .sum_dim(D - 1)
187            .sqrt()
188            .clamp_min(1e-7);
189        x.div(row_norms)
190    }
191}
192
193impl<B: Backend> SimpleOptimizer<B> for MuonPlus<B> {
194    type State<const D: usize> = MuonPlusState<B, D>;
195
196    fn step<const D: usize>(
197        &self,
198        lr: LearningRate,
199        tensor: Tensor<B, D>,
200        grad: Tensor<B, D>,
201        state: Option<Self::State<D>>,
202    ) -> (Tensor<B, D>, Option<Self::State<D>>) {
203        let state = state.unwrap_or_else(|| MuonPlusState::new(None, None, None, None));
204
205        let (updated, state) = if D == 2 {
206            // --- Muon+ group ---
207            let mu = self.momentum.elem::<f32>();
208            // M_t = μ·M_{t-1} + (1-μ)·G_t
209            let momentum = match &state.mu_momentum {
210                Some(m) => m
211                    .clone()
212                    .mul_scalar(mu)
213                    .add(grad.clone().mul_scalar(1.0 - mu)),
214                None => grad.clone().mul_scalar(1.0 - mu),
215            };
216            // O_t = Norm_(d)(Ortho(M_t))
217            let update = self.normalize(self.orthogonalize(momentum.clone()));
218
219            // W_t = W_{t-1} - η·√(m/n)·O_t  (Bernstein dimensional factor)
220            let dims = tensor.dims();
221            let (m, n) = (dims[0] as f64, dims[1] as f64);
222            let lr_scaled = lr * (m / n).sqrt();
223
224            let decayed = tensor
225                .clone()
226                .mul_scalar(1.0 - (self.weight_decay as f32 * lr_scaled as f32).min(0.999));
227            (
228                decayed.sub(update.mul_scalar(lr_scaled as f32)),
229                MuonPlusState::new(Some(momentum), None, None, None),
230            )
231        } else {
232            // --- AdamW group (1D: biases, norms) ---
233            let b1 = self.beta_1.elem::<f32>();
234            let b2 = self.beta_2.elem::<f32>();
235            let eps = self.epsilon.elem::<f32>();
236            let t = state.ad_time.unwrap_or(0) + 1;
237
238            let m = match &state.ad_moment_1 {
239                Some(m) => m
240                    .clone()
241                    .mul_scalar(b1)
242                    .add(grad.clone().mul_scalar(1.0 - b1)),
243                None => grad.clone().mul_scalar(1.0 - b1),
244            };
245            let v = match &state.ad_moment_2 {
246                Some(v) => v
247                    .clone()
248                    .mul_scalar(b2)
249                    .add(grad.clone().powf_scalar(2.0).mul_scalar(1.0 - b2)),
250                None => grad.clone().powf_scalar(2.0).mul_scalar(1.0 - b2),
251            };
252
253            let m_hat = m.clone().div_scalar(1.0 - b1.powi(t as i32));
254            let v_hat = v.clone().div_scalar(1.0 - b2.powi(t as i32));
255            let step = m_hat.div(v_hat.sqrt().add_scalar(eps));
256
257            let decayed = tensor
258                .clone()
259                .mul_scalar(1.0 - (self.weight_decay as f32 * lr as f32).min(0.999));
260            (
261                decayed.sub(step.mul_scalar(lr as f32)),
262                MuonPlusState::new(None, Some(m), Some(v), Some(t)),
263            )
264        };
265        (updated, Some(state))
266    }
267
268    fn to_device<const D: usize>(
269        mut state: Self::State<D>,
270        device: &burn::tensor::ops::Device<B>,
271    ) -> Self::State<D> {
272        state.mu_momentum = state.mu_momentum.map(|t| t.to_device(device));
273        state.ad_moment_1 = state.ad_moment_1.map(|t| t.to_device(device));
274        state.ad_moment_2 = state.ad_moment_2.map(|t| t.to_device(device));
275        state
276    }
277}
278
279impl MuonPlusConfig {
280    /// Build the optimizer from config.
281    pub fn build<B: Backend>(&self) -> MuonPlus<B> {
282        MuonPlus {
283            momentum: self.momentum,
284            beta_1: self.beta_1,
285            beta_2: self.beta_2,
286            epsilon: self.epsilon,
287            ns_steps: self.ns_steps,
288            ns_coeffs: self.ns_coeffs,
289            norm_dir: self.norm_dir,
290            weight_decay: self.weight_decay,
291            _backend: core::marker::PhantomData,
292        }
293    }
294
295    /// Initialize the optimizer for a module.
296    pub fn init<B: AutodiffBackend, M: AutodiffModule<B>>(
297        &self,
298    ) -> OptimizerAdaptor<MuonPlus<B::InnerBackend>, M, B> {
299        OptimizerAdaptor::from(self.build())
300    }
301}