1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
// SPDX-License-Identifier: AGPL-3.0-or-later
//! Lookahead - Lookahead Optimizer (Zhang et al.)
//!
//! Maintains two sets of weights: fast and slow.
//! Interpolates between them for better convergence.
//!
//! **Canonical Pattern**: Struct-based with Tensor support
use crate::error::{BarracudaError, Result};
use crate::tensor::Tensor;
/// Lookahead optimizer state.
pub struct LookaheadState {
/// Slow weights (updated every k steps).
pub slow_weights: Tensor,
/// Step counter for sync frequency.
pub k_counter: usize,
}
/// Lookahead optimizer wrapper
///
/// Maintains fast and slow weight sets, updating slow weights every k steps.
pub struct Lookahead {
fast_weights: Tensor,
state: LookaheadState,
k: usize, // Sync frequency
alpha: f32,
}
impl Lookahead {
/// Create a new Lookahead optimizer
/// ## Parameters
/// - `fast_weights`: Current fast weights (Tensor)
/// - `slow_weights`: Slow weights (Tensor, same shape as `fast_weights`)
/// - `k`: Sync frequency (update slow weights every k steps)
/// - `alpha`: Slow weights step size (interpolation factor)
/// # Errors
/// Returns [`Err`] if buffer allocation, GPU dispatch, or buffer
/// readback fails (e.g. device lost or out of memory).
/// Also returns [`Err`] if shape validation fails.
pub fn new(fast_weights: Tensor, slow_weights: Tensor, k: usize, alpha: f32) -> Result<Self> {
if fast_weights.len() != slow_weights.len() {
return Err(BarracudaError::Device(format!(
"State dimension mismatch: fast_weights len {}, slow_weights len {}",
fast_weights.len(),
slow_weights.len()
)));
}
if !std::ptr::eq(
fast_weights.device().as_ref(),
slow_weights.device().as_ref(),
) {
return Err(BarracudaError::Device(
"fast_weights and slow_weights must be on the same device".to_string(),
));
}
Ok(Self {
fast_weights,
state: LookaheadState {
slow_weights,
k_counter: 0,
},
k,
alpha,
})
}
/// Execute one Lookahead step
/// Updates slow weights every k steps, otherwise returns fast weights.
/// Returns the weights to use for the current step.
/// # Errors
/// Returns [`Err`] if buffer allocation, GPU dispatch, or buffer
/// readback fails (e.g. device lost or out of memory).
pub fn execute(mut self) -> Result<Tensor> {
self.state.k_counter += 1;
// Update slow weights every k steps
if self.state.k_counter.is_multiple_of(self.k) {
// slow_weights = slow_weights + alpha * (fast_weights - slow_weights)
// Rearranged: slow_weights = (1 - alpha) * slow_weights + alpha * fast_weights
//
// Using tensor ops:
// 1. diff = fast_weights - slow_weights
// 2. scaled_diff = alpha * diff
// 3. new_slow = slow_weights + scaled_diff
let diff = self.fast_weights.sub(&self.state.slow_weights)?;
let scaled_diff = diff.mul_scalar(self.alpha)?;
let new_slow = self.state.slow_weights.add(&scaled_diff)?;
// Update state for next iteration
self.state.slow_weights = new_slow.clone();
Ok(new_slow)
} else {
// Return fast weights (no slow weight update this step)
Ok(self.fast_weights)
}
}
/// Get current state (for checkpointing)
#[must_use]
pub fn state(&self) -> &LookaheadState {
&self.state
}
/// Get mutable state (for manual updates)
pub fn state_mut(&mut self) -> &mut LookaheadState {
&mut self.state
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_lookahead() {
let device = crate::device::test_pool::get_test_device().await;
let fast_weights = Tensor::from_vec_on(vec![1.0; 100], vec![100], device.clone())
.await
.unwrap();
let slow_weights = Tensor::from_vec_on(vec![0.9; 100], vec![100], device.clone())
.await
.unwrap();
let op = Lookahead::new(fast_weights, slow_weights, 5, 0.5).unwrap();
let result = op.execute().unwrap();
assert_eq!(result.len(), 100);
}
}