Skip to main content

ruvector_learning_wasm/
lora.rs

1//! MicroLoRA: Rank-2 Low-Rank Adaptation with <100us latency
2//!
3//! Implements the core LoRA algorithm: output = input + alpha * (input @ A @ B)
4//! where A: [d x 2] and B: [2 x d] for rank-2 adaptation.
5
6use wasm_bindgen::prelude::*;
7
8/// Configuration for MicroLoRA
9#[derive(Debug, Clone, Copy)]
10pub struct LoRAConfig {
11    /// Embedding dimension (typically 256)
12    pub dim: usize,
13    /// LoRA rank (1-2 for micro, default 2)
14    pub rank: usize,
15    /// Scaling factor alpha (default 0.1)
16    pub alpha: f32,
17    /// Learning rate for adaptation (default 0.01)
18    pub learning_rate: f32,
19    /// Dropout rate (0.0 = no dropout)
20    pub dropout: f32,
21}
22
23impl Default for LoRAConfig {
24    fn default() -> Self {
25        Self {
26            dim: 256,
27            rank: 2,
28            alpha: 0.1,
29            learning_rate: 0.01,
30            dropout: 0.0,
31        }
32    }
33}
34
35/// A single LoRA adapter pair (A and B matrices)
36///
37/// For rank-2:
38/// - A: [dim x 2] - Down projection
39/// - B: [2 x dim] - Up projection (initialized to zero)
40///
41/// Forward: output = input + alpha * (input @ A @ B)
42#[derive(Clone)]
43pub struct LoRAPair {
44    /// Down projection matrix A: [dim][rank]
45    /// Stored as Vec<[f32; 2]> for rank-2
46    a: Vec<[f32; 2]>,
47    /// Up projection matrix B: [rank][dim]
48    /// Stored as [[f32; 256]; 2] for fixed 256-dim embeddings
49    b: [[f32; 256]; 2],
50    /// Scaling factor
51    alpha: f32,
52    /// Learning rate
53    lr: f32,
54    /// Embedding dimension
55    dim: usize,
56    /// Adaptation count for statistics
57    adapt_count: u64,
58}
59
60impl LoRAPair {
61    /// Create a new LoRA pair with Kaiming initialization for A, zeros for B
62    pub fn new(config: &LoRAConfig) -> Self {
63        let dim = config.dim.min(256); // Cap at 256 for fixed-size B
64        let rank = config.rank.min(2); // Cap at 2 for micro
65
66        // Initialize A with small random values (Kaiming-like)
67        // Using deterministic pseudo-random for reproducibility
68        let mut a = Vec::with_capacity(dim);
69        let scale = (2.0 / dim as f32).sqrt() * 0.1; // Small initialization
70
71        for i in 0..dim {
72            let seed = i as u32;
73            let r0 = pseudo_random(seed) * scale - scale / 2.0;
74            let r1 = if rank > 1 {
75                pseudo_random(seed.wrapping_add(1000)) * scale - scale / 2.0
76            } else {
77                0.0
78            };
79            a.push([r0, r1]);
80        }
81
82        // B initialized to zeros (LoRA standard practice)
83        let b = [[0.0f32; 256]; 2];
84
85        Self {
86            a,
87            b,
88            alpha: config.alpha,
89            lr: config.learning_rate,
90            dim,
91            adapt_count: 0,
92        }
93    }
94
95    /// Forward pass: output = input + alpha * (input @ A @ B)
96    ///
97    /// Complexity: O(d * r + r * d) = O(2dr) for rank r
98    /// For rank-2, d=256: ~1024 ops = <100us
99    #[inline]
100    pub fn forward(&self, input: &[f32]) -> Vec<f32> {
101        let n = input.len().min(self.dim);
102        let mut output = input.to_vec();
103
104        // Compute low_rank = input @ A (result: [2])
105        let mut low_rank = [0.0f32; 2];
106        for i in 0..n {
107            low_rank[0] += input[i] * self.a[i][0];
108            low_rank[1] += input[i] * self.a[i][1];
109        }
110
111        // Compute delta = low_rank @ B (result: [dim])
112        // Output = input + alpha * delta
113        for i in 0..n {
114            let delta = low_rank[0] * self.b[0][i] + low_rank[1] * self.b[1][i];
115            output[i] += self.alpha * delta;
116        }
117
118        output
119    }
120
121    /// Forward pass into pre-allocated buffer (zero-allocation hot path)
122    #[inline]
123    pub fn forward_into(&self, input: &[f32], output: &mut [f32]) {
124        let n = input.len().min(self.dim).min(output.len());
125
126        // Copy input to output
127        output[..n].copy_from_slice(&input[..n]);
128
129        // Compute low_rank = input @ A
130        let mut low_rank = [0.0f32; 2];
131        for i in 0..n {
132            low_rank[0] += input[i] * self.a[i][0];
133            low_rank[1] += input[i] * self.a[i][1];
134        }
135
136        // Add delta to output
137        for i in 0..n {
138            let delta = low_rank[0] * self.b[0][i] + low_rank[1] * self.b[1][i];
139            output[i] += self.alpha * delta;
140        }
141    }
142
143    /// Adapt weights based on gradient signal
144    ///
145    /// Uses rank-1 outer product update to B matrix for instant adaptation.
146    /// Target latency: <100us
147    #[inline]
148    pub fn adapt(&mut self, gradient: &[f32]) {
149        let n = gradient.len().min(self.dim);
150
151        // Compute gradient norm for normalization
152        let mut grad_norm_sq = 0.0f32;
153        for i in 0..n {
154            grad_norm_sq += gradient[i] * gradient[i];
155        }
156
157        if grad_norm_sq < 1e-16 {
158            return; // Skip if gradient is too small
159        }
160
161        let grad_norm = fast_sqrt(grad_norm_sq);
162        let inv_norm = 1.0 / grad_norm;
163
164        // Compute column sums of A for scaling
165        let mut a_col_sum = [0.0f32; 2];
166        for i in 0..n {
167            a_col_sum[0] += self.a[i][0];
168            a_col_sum[1] += self.a[i][1];
169        }
170
171        // Update B using outer product: B += lr * a_sum * normalized_grad^T
172        for j in 0..n {
173            let normalized_grad = gradient[j] * inv_norm;
174            self.b[0][j] += self.lr * a_col_sum[0] * normalized_grad;
175            self.b[1][j] += self.lr * a_col_sum[1] * normalized_grad;
176        }
177
178        self.adapt_count += 1;
179    }
180
181    /// Adapt with improvement signal (for reinforcement learning)
182    ///
183    /// Uses the improvement ratio to scale the update magnitude.
184    #[inline]
185    pub fn adapt_with_reward(&mut self, gradient: &[f32], improvement: f32) {
186        if improvement <= 0.0 {
187            return; // Only learn from positive improvements
188        }
189
190        let n = gradient.len().min(self.dim);
191
192        // Scale learning rate by improvement (clamped)
193        let scaled_lr = self.lr * improvement.min(2.0);
194
195        // Compute gradient norm
196        let mut grad_norm_sq = 0.0f32;
197        for i in 0..n {
198            grad_norm_sq += gradient[i] * gradient[i];
199        }
200
201        if grad_norm_sq < 1e-16 {
202            return;
203        }
204
205        let inv_norm = 1.0 / fast_sqrt(grad_norm_sq);
206
207        // Compute A column sums
208        let mut a_col_sum = [0.0f32; 2];
209        for i in 0..n {
210            a_col_sum[0] += self.a[i][0];
211            a_col_sum[1] += self.a[i][1];
212        }
213
214        // Update B
215        for j in 0..n {
216            let normalized_grad = gradient[j] * inv_norm;
217            self.b[0][j] += scaled_lr * a_col_sum[0] * normalized_grad;
218            self.b[1][j] += scaled_lr * a_col_sum[1] * normalized_grad;
219        }
220
221        self.adapt_count += 1;
222    }
223
224    /// Reset B matrix to zeros (fresh start)
225    pub fn reset(&mut self) {
226        for i in 0..256 {
227            self.b[0][i] = 0.0;
228            self.b[1][i] = 0.0;
229        }
230        self.adapt_count = 0;
231    }
232
233    /// Get the number of adaptations performed
234    pub fn adapt_count(&self) -> u64 {
235        self.adapt_count
236    }
237
238    /// Get the effective weight delta norm (for monitoring)
239    pub fn delta_norm(&self) -> f32 {
240        let mut norm_sq = 0.0f32;
241        for i in 0..self.dim {
242            let delta = self.b[0][i] * self.b[0][i] + self.b[1][i] * self.b[1][i];
243            norm_sq += delta;
244        }
245        fast_sqrt(norm_sq) * self.alpha
246    }
247
248    /// Get parameter count
249    pub fn param_count(&self) -> usize {
250        self.a.len() * 2 + 256 * 2
251    }
252}
253
254/// Main MicroLoRA engine managing multiple LoRA pairs
255pub struct MicroLoRAEngine {
256    /// Default LoRA pair for unscoped operations
257    default_lora: LoRAPair,
258    /// Configuration (kept for potential future use)
259    #[allow(dead_code)]
260    config: LoRAConfig,
261    /// Total forward passes
262    forward_count: u64,
263    /// Total adaptations
264    total_adapt_count: u64,
265}
266
267impl MicroLoRAEngine {
268    /// Create a new MicroLoRA engine
269    pub fn new(config: LoRAConfig) -> Self {
270        Self {
271            default_lora: LoRAPair::new(&config),
272            config,
273            forward_count: 0,
274            total_adapt_count: 0,
275        }
276    }
277
278    /// Forward pass through the default LoRA
279    #[inline]
280    pub fn forward(&mut self, input: &[f32]) -> Vec<f32> {
281        self.forward_count += 1;
282        self.default_lora.forward(input)
283    }
284
285    /// Adapt the default LoRA with gradient
286    #[inline]
287    pub fn adapt(&mut self, gradient: &[f32]) {
288        self.default_lora.adapt(gradient);
289        self.total_adapt_count += 1;
290    }
291
292    /// Adapt with improvement reward
293    #[inline]
294    pub fn adapt_with_reward(&mut self, gradient: &[f32], improvement: f32) {
295        self.default_lora.adapt_with_reward(gradient, improvement);
296        self.total_adapt_count += 1;
297    }
298
299    /// Reset the engine
300    pub fn reset(&mut self) {
301        self.default_lora.reset();
302        self.forward_count = 0;
303        self.total_adapt_count = 0;
304    }
305
306    /// Get statistics
307    pub fn stats(&self) -> (u64, u64, f32) {
308        (
309            self.forward_count,
310            self.total_adapt_count,
311            self.default_lora.delta_norm(),
312        )
313    }
314
315    /// Get the underlying LoRA pair for advanced use
316    pub fn lora(&self) -> &LoRAPair {
317        &self.default_lora
318    }
319
320    /// Get mutable reference to underlying LoRA
321    pub fn lora_mut(&mut self) -> &mut LoRAPair {
322        &mut self.default_lora
323    }
324}
325
326impl Default for MicroLoRAEngine {
327    fn default() -> Self {
328        Self::new(LoRAConfig::default())
329    }
330}
331
332// ============ Helper Functions ============
333
334/// Fast inverse square root (Quake III style)
335#[inline(always)]
336fn fast_sqrt(x: f32) -> f32 {
337    if x <= 0.0 {
338        return 0.0;
339    }
340    let i = 0x5f3759df - (x.to_bits() >> 1);
341    let y = f32::from_bits(i);
342    x * y * (1.5 - 0.5 * x * y * y)
343}
344
345/// Deterministic pseudo-random number generator
346#[inline(always)]
347fn pseudo_random(seed: u32) -> f32 {
348    // Simple xorshift
349    let mut x = seed;
350    x ^= x << 13;
351    x ^= x >> 17;
352    x ^= x << 5;
353    (x as f32) / (u32::MAX as f32)
354}
355
356// ============ WASM Bindings ============
357
358pub mod wasm_exports {
359    use super::*;
360    #[allow(unused_imports)]
361    use wasm_bindgen::prelude::*;
362
363    /// WASM-exposed MicroLoRA engine
364    #[wasm_bindgen]
365    pub struct WasmMicroLoRA {
366        engine: MicroLoRAEngine,
367        // Pre-allocated buffers for zero-allocation hot paths
368        input_buffer: Vec<f32>,
369        output_buffer: Vec<f32>,
370    }
371
372    #[wasm_bindgen]
373    impl WasmMicroLoRA {
374        /// Create a new MicroLoRA engine
375        ///
376        /// @param dim - Embedding dimension (default 256, max 256)
377        /// @param alpha - Scaling factor (default 0.1)
378        /// @param learning_rate - Learning rate (default 0.01)
379        #[wasm_bindgen(constructor)]
380        pub fn new(dim: Option<usize>, alpha: Option<f32>, learning_rate: Option<f32>) -> Self {
381            let config = LoRAConfig {
382                dim: dim.unwrap_or(256).min(256),
383                rank: 2,
384                alpha: alpha.unwrap_or(0.1),
385                learning_rate: learning_rate.unwrap_or(0.01),
386                dropout: 0.0,
387            };
388
389            let actual_dim = config.dim;
390            Self {
391                engine: MicroLoRAEngine::new(config),
392                input_buffer: vec![0.0; actual_dim],
393                output_buffer: vec![0.0; actual_dim],
394            }
395        }
396
397        /// Get pointer to input buffer for direct memory access
398        #[wasm_bindgen]
399        pub fn get_input_ptr(&mut self) -> *mut f32 {
400            self.input_buffer.as_mut_ptr()
401        }
402
403        /// Get pointer to output buffer for direct memory access
404        #[wasm_bindgen]
405        pub fn get_output_ptr(&self) -> *const f32 {
406            self.output_buffer.as_ptr()
407        }
408
409        /// Get embedding dimension
410        #[wasm_bindgen]
411        pub fn dim(&self) -> usize {
412            self.input_buffer.len()
413        }
414
415        /// Forward pass using internal buffers (zero-allocation)
416        ///
417        /// Write input to get_input_ptr(), call forward(), read from get_output_ptr()
418        #[wasm_bindgen]
419        pub fn forward(&mut self) {
420            self.engine
421                .default_lora
422                .forward_into(&self.input_buffer, &mut self.output_buffer);
423            self.engine.forward_count += 1;
424        }
425
426        /// Forward pass with typed array input (allocates output)
427        #[wasm_bindgen]
428        pub fn forward_array(&mut self, input: &[f32]) -> Vec<f32> {
429            self.engine.forward(input)
430        }
431
432        /// Adapt using input buffer as gradient
433        #[wasm_bindgen]
434        pub fn adapt(&mut self) {
435            self.engine.adapt(&self.input_buffer.clone());
436        }
437
438        /// Adapt with typed array gradient
439        #[wasm_bindgen]
440        pub fn adapt_array(&mut self, gradient: &[f32]) {
441            self.engine.adapt(gradient);
442        }
443
444        /// Adapt with improvement reward using input buffer as gradient
445        #[wasm_bindgen]
446        pub fn adapt_with_reward(&mut self, improvement: f32) {
447            self.engine
448                .adapt_with_reward(&self.input_buffer.clone(), improvement);
449        }
450
451        /// Reset the engine
452        #[wasm_bindgen]
453        pub fn reset(&mut self) {
454            self.engine.reset();
455        }
456
457        /// Get forward pass count
458        #[wasm_bindgen]
459        pub fn forward_count(&self) -> u64 {
460            self.engine.forward_count
461        }
462
463        /// Get adaptation count
464        #[wasm_bindgen]
465        pub fn adapt_count(&self) -> u64 {
466            self.engine.total_adapt_count
467        }
468
469        /// Get delta norm (weight change magnitude)
470        #[wasm_bindgen]
471        pub fn delta_norm(&self) -> f32 {
472            self.engine.default_lora.delta_norm()
473        }
474
475        /// Get parameter count
476        #[wasm_bindgen]
477        pub fn param_count(&self) -> usize {
478            self.engine.default_lora.param_count()
479        }
480    }
481}
482
483#[cfg(test)]
484mod tests {
485    use super::*;
486
487    #[test]
488    fn test_lora_pair_creation() {
489        let config = LoRAConfig::default();
490        let lora = LoRAPair::new(&config);
491        assert_eq!(lora.dim, 256);
492        assert_eq!(lora.adapt_count, 0);
493    }
494
495    #[test]
496    fn test_lora_forward() {
497        let config = LoRAConfig::default();
498        let lora = LoRAPair::new(&config);
499
500        let input = vec![1.0; 256];
501        let output = lora.forward(&input);
502
503        assert_eq!(output.len(), 256);
504        // Initially B is zeros, so output should equal input
505        for i in 0..256 {
506            assert!((output[i] - input[i]).abs() < 1e-6);
507        }
508    }
509
510    #[test]
511    fn test_lora_adapt() {
512        let config = LoRAConfig::default();
513        let mut lora = LoRAPair::new(&config);
514
515        let gradient = vec![0.1; 256];
516        lora.adapt(&gradient);
517
518        assert_eq!(lora.adapt_count, 1);
519        assert!(lora.delta_norm() > 0.0);
520    }
521
522    #[test]
523    fn test_lora_forward_after_adapt() {
524        let config = LoRAConfig::default();
525        let mut lora = LoRAPair::new(&config);
526
527        // Adapt
528        let gradient = vec![0.1; 256];
529        lora.adapt(&gradient);
530
531        // Forward should now produce different output
532        let input = vec![1.0; 256];
533        let output = lora.forward(&input);
534
535        // Output should differ from input after adaptation
536        let mut diff = 0.0f32;
537        for i in 0..256 {
538            diff += (output[i] - input[i]).abs();
539        }
540        assert!(
541            diff > 0.0,
542            "Output should differ from input after adaptation"
543        );
544    }
545
546    #[test]
547    fn test_engine_stats() {
548        let mut engine = MicroLoRAEngine::default();
549
550        let input = vec![1.0; 256];
551        let _ = engine.forward(&input);
552        engine.adapt(&input);
553
554        let (forwards, adapts, delta) = engine.stats();
555        assert_eq!(forwards, 1);
556        assert_eq!(adapts, 1);
557        assert!(delta >= 0.0);
558    }
559}