ruvector_learning_wasm/
lora.rs1use wasm_bindgen::prelude::*;
7
8#[derive(Debug, Clone, Copy)]
10pub struct LoRAConfig {
11 pub dim: usize,
13 pub rank: usize,
15 pub alpha: f32,
17 pub learning_rate: f32,
19 pub dropout: f32,
21}
22
23impl Default for LoRAConfig {
24 fn default() -> Self {
25 Self {
26 dim: 256,
27 rank: 2,
28 alpha: 0.1,
29 learning_rate: 0.01,
30 dropout: 0.0,
31 }
32 }
33}
34
35#[derive(Clone)]
43pub struct LoRAPair {
44 a: Vec<[f32; 2]>,
47 b: [[f32; 256]; 2],
50 alpha: f32,
52 lr: f32,
54 dim: usize,
56 adapt_count: u64,
58}
59
60impl LoRAPair {
61 pub fn new(config: &LoRAConfig) -> Self {
63 let dim = config.dim.min(256); let rank = config.rank.min(2); let mut a = Vec::with_capacity(dim);
69 let scale = (2.0 / dim as f32).sqrt() * 0.1; for i in 0..dim {
72 let seed = i as u32;
73 let r0 = pseudo_random(seed) * scale - scale / 2.0;
74 let r1 = if rank > 1 {
75 pseudo_random(seed.wrapping_add(1000)) * scale - scale / 2.0
76 } else {
77 0.0
78 };
79 a.push([r0, r1]);
80 }
81
82 let b = [[0.0f32; 256]; 2];
84
85 Self {
86 a,
87 b,
88 alpha: config.alpha,
89 lr: config.learning_rate,
90 dim,
91 adapt_count: 0,
92 }
93 }
94
95 #[inline]
100 pub fn forward(&self, input: &[f32]) -> Vec<f32> {
101 let n = input.len().min(self.dim);
102 let mut output = input.to_vec();
103
104 let mut low_rank = [0.0f32; 2];
106 for i in 0..n {
107 low_rank[0] += input[i] * self.a[i][0];
108 low_rank[1] += input[i] * self.a[i][1];
109 }
110
111 for i in 0..n {
114 let delta = low_rank[0] * self.b[0][i] + low_rank[1] * self.b[1][i];
115 output[i] += self.alpha * delta;
116 }
117
118 output
119 }
120
121 #[inline]
123 pub fn forward_into(&self, input: &[f32], output: &mut [f32]) {
124 let n = input.len().min(self.dim).min(output.len());
125
126 output[..n].copy_from_slice(&input[..n]);
128
129 let mut low_rank = [0.0f32; 2];
131 for i in 0..n {
132 low_rank[0] += input[i] * self.a[i][0];
133 low_rank[1] += input[i] * self.a[i][1];
134 }
135
136 for i in 0..n {
138 let delta = low_rank[0] * self.b[0][i] + low_rank[1] * self.b[1][i];
139 output[i] += self.alpha * delta;
140 }
141 }
142
143 #[inline]
148 pub fn adapt(&mut self, gradient: &[f32]) {
149 let n = gradient.len().min(self.dim);
150
151 let mut grad_norm_sq = 0.0f32;
153 for i in 0..n {
154 grad_norm_sq += gradient[i] * gradient[i];
155 }
156
157 if grad_norm_sq < 1e-16 {
158 return; }
160
161 let grad_norm = fast_sqrt(grad_norm_sq);
162 let inv_norm = 1.0 / grad_norm;
163
164 let mut a_col_sum = [0.0f32; 2];
166 for i in 0..n {
167 a_col_sum[0] += self.a[i][0];
168 a_col_sum[1] += self.a[i][1];
169 }
170
171 for j in 0..n {
173 let normalized_grad = gradient[j] * inv_norm;
174 self.b[0][j] += self.lr * a_col_sum[0] * normalized_grad;
175 self.b[1][j] += self.lr * a_col_sum[1] * normalized_grad;
176 }
177
178 self.adapt_count += 1;
179 }
180
181 #[inline]
185 pub fn adapt_with_reward(&mut self, gradient: &[f32], improvement: f32) {
186 if improvement <= 0.0 {
187 return; }
189
190 let n = gradient.len().min(self.dim);
191
192 let scaled_lr = self.lr * improvement.min(2.0);
194
195 let mut grad_norm_sq = 0.0f32;
197 for i in 0..n {
198 grad_norm_sq += gradient[i] * gradient[i];
199 }
200
201 if grad_norm_sq < 1e-16 {
202 return;
203 }
204
205 let inv_norm = 1.0 / fast_sqrt(grad_norm_sq);
206
207 let mut a_col_sum = [0.0f32; 2];
209 for i in 0..n {
210 a_col_sum[0] += self.a[i][0];
211 a_col_sum[1] += self.a[i][1];
212 }
213
214 for j in 0..n {
216 let normalized_grad = gradient[j] * inv_norm;
217 self.b[0][j] += scaled_lr * a_col_sum[0] * normalized_grad;
218 self.b[1][j] += scaled_lr * a_col_sum[1] * normalized_grad;
219 }
220
221 self.adapt_count += 1;
222 }
223
224 pub fn reset(&mut self) {
226 for i in 0..256 {
227 self.b[0][i] = 0.0;
228 self.b[1][i] = 0.0;
229 }
230 self.adapt_count = 0;
231 }
232
233 pub fn adapt_count(&self) -> u64 {
235 self.adapt_count
236 }
237
238 pub fn delta_norm(&self) -> f32 {
240 let mut norm_sq = 0.0f32;
241 for i in 0..self.dim {
242 let delta = self.b[0][i] * self.b[0][i] + self.b[1][i] * self.b[1][i];
243 norm_sq += delta;
244 }
245 fast_sqrt(norm_sq) * self.alpha
246 }
247
248 pub fn param_count(&self) -> usize {
250 self.a.len() * 2 + 256 * 2
251 }
252}
253
254pub struct MicroLoRAEngine {
256 default_lora: LoRAPair,
258 #[allow(dead_code)]
260 config: LoRAConfig,
261 forward_count: u64,
263 total_adapt_count: u64,
265}
266
267impl MicroLoRAEngine {
268 pub fn new(config: LoRAConfig) -> Self {
270 Self {
271 default_lora: LoRAPair::new(&config),
272 config,
273 forward_count: 0,
274 total_adapt_count: 0,
275 }
276 }
277
278 #[inline]
280 pub fn forward(&mut self, input: &[f32]) -> Vec<f32> {
281 self.forward_count += 1;
282 self.default_lora.forward(input)
283 }
284
285 #[inline]
287 pub fn adapt(&mut self, gradient: &[f32]) {
288 self.default_lora.adapt(gradient);
289 self.total_adapt_count += 1;
290 }
291
292 #[inline]
294 pub fn adapt_with_reward(&mut self, gradient: &[f32], improvement: f32) {
295 self.default_lora.adapt_with_reward(gradient, improvement);
296 self.total_adapt_count += 1;
297 }
298
299 pub fn reset(&mut self) {
301 self.default_lora.reset();
302 self.forward_count = 0;
303 self.total_adapt_count = 0;
304 }
305
306 pub fn stats(&self) -> (u64, u64, f32) {
308 (
309 self.forward_count,
310 self.total_adapt_count,
311 self.default_lora.delta_norm(),
312 )
313 }
314
315 pub fn lora(&self) -> &LoRAPair {
317 &self.default_lora
318 }
319
320 pub fn lora_mut(&mut self) -> &mut LoRAPair {
322 &mut self.default_lora
323 }
324}
325
326impl Default for MicroLoRAEngine {
327 fn default() -> Self {
328 Self::new(LoRAConfig::default())
329 }
330}
331
332#[inline(always)]
336fn fast_sqrt(x: f32) -> f32 {
337 if x <= 0.0 {
338 return 0.0;
339 }
340 let i = 0x5f3759df - (x.to_bits() >> 1);
341 let y = f32::from_bits(i);
342 x * y * (1.5 - 0.5 * x * y * y)
343}
344
345#[inline(always)]
347fn pseudo_random(seed: u32) -> f32 {
348 let mut x = seed;
350 x ^= x << 13;
351 x ^= x >> 17;
352 x ^= x << 5;
353 (x as f32) / (u32::MAX as f32)
354}
355
356pub mod wasm_exports {
359 use super::*;
360 #[allow(unused_imports)]
361 use wasm_bindgen::prelude::*;
362
363 #[wasm_bindgen]
365 pub struct WasmMicroLoRA {
366 engine: MicroLoRAEngine,
367 input_buffer: Vec<f32>,
369 output_buffer: Vec<f32>,
370 }
371
372 #[wasm_bindgen]
373 impl WasmMicroLoRA {
374 #[wasm_bindgen(constructor)]
380 pub fn new(dim: Option<usize>, alpha: Option<f32>, learning_rate: Option<f32>) -> Self {
381 let config = LoRAConfig {
382 dim: dim.unwrap_or(256).min(256),
383 rank: 2,
384 alpha: alpha.unwrap_or(0.1),
385 learning_rate: learning_rate.unwrap_or(0.01),
386 dropout: 0.0,
387 };
388
389 let actual_dim = config.dim;
390 Self {
391 engine: MicroLoRAEngine::new(config),
392 input_buffer: vec![0.0; actual_dim],
393 output_buffer: vec![0.0; actual_dim],
394 }
395 }
396
397 #[wasm_bindgen]
399 pub fn get_input_ptr(&mut self) -> *mut f32 {
400 self.input_buffer.as_mut_ptr()
401 }
402
403 #[wasm_bindgen]
405 pub fn get_output_ptr(&self) -> *const f32 {
406 self.output_buffer.as_ptr()
407 }
408
409 #[wasm_bindgen]
411 pub fn dim(&self) -> usize {
412 self.input_buffer.len()
413 }
414
415 #[wasm_bindgen]
419 pub fn forward(&mut self) {
420 self.engine
421 .default_lora
422 .forward_into(&self.input_buffer, &mut self.output_buffer);
423 self.engine.forward_count += 1;
424 }
425
426 #[wasm_bindgen]
428 pub fn forward_array(&mut self, input: &[f32]) -> Vec<f32> {
429 self.engine.forward(input)
430 }
431
432 #[wasm_bindgen]
434 pub fn adapt(&mut self) {
435 self.engine.adapt(&self.input_buffer.clone());
436 }
437
438 #[wasm_bindgen]
440 pub fn adapt_array(&mut self, gradient: &[f32]) {
441 self.engine.adapt(gradient);
442 }
443
444 #[wasm_bindgen]
446 pub fn adapt_with_reward(&mut self, improvement: f32) {
447 self.engine
448 .adapt_with_reward(&self.input_buffer.clone(), improvement);
449 }
450
451 #[wasm_bindgen]
453 pub fn reset(&mut self) {
454 self.engine.reset();
455 }
456
457 #[wasm_bindgen]
459 pub fn forward_count(&self) -> u64 {
460 self.engine.forward_count
461 }
462
463 #[wasm_bindgen]
465 pub fn adapt_count(&self) -> u64 {
466 self.engine.total_adapt_count
467 }
468
469 #[wasm_bindgen]
471 pub fn delta_norm(&self) -> f32 {
472 self.engine.default_lora.delta_norm()
473 }
474
475 #[wasm_bindgen]
477 pub fn param_count(&self) -> usize {
478 self.engine.default_lora.param_count()
479 }
480 }
481}
482
483#[cfg(test)]
484mod tests {
485 use super::*;
486
487 #[test]
488 fn test_lora_pair_creation() {
489 let config = LoRAConfig::default();
490 let lora = LoRAPair::new(&config);
491 assert_eq!(lora.dim, 256);
492 assert_eq!(lora.adapt_count, 0);
493 }
494
495 #[test]
496 fn test_lora_forward() {
497 let config = LoRAConfig::default();
498 let lora = LoRAPair::new(&config);
499
500 let input = vec![1.0; 256];
501 let output = lora.forward(&input);
502
503 assert_eq!(output.len(), 256);
504 for i in 0..256 {
506 assert!((output[i] - input[i]).abs() < 1e-6);
507 }
508 }
509
510 #[test]
511 fn test_lora_adapt() {
512 let config = LoRAConfig::default();
513 let mut lora = LoRAPair::new(&config);
514
515 let gradient = vec![0.1; 256];
516 lora.adapt(&gradient);
517
518 assert_eq!(lora.adapt_count, 1);
519 assert!(lora.delta_norm() > 0.0);
520 }
521
522 #[test]
523 fn test_lora_forward_after_adapt() {
524 let config = LoRAConfig::default();
525 let mut lora = LoRAPair::new(&config);
526
527 let gradient = vec![0.1; 256];
529 lora.adapt(&gradient);
530
531 let input = vec![1.0; 256];
533 let output = lora.forward(&input);
534
535 let mut diff = 0.0f32;
537 for i in 0..256 {
538 diff += (output[i] - input[i]).abs();
539 }
540 assert!(
541 diff > 0.0,
542 "Output should differ from input after adaptation"
543 );
544 }
545
546 #[test]
547 fn test_engine_stats() {
548 let mut engine = MicroLoRAEngine::default();
549
550 let input = vec![1.0; 256];
551 let _ = engine.forward(&input);
552 engine.adapt(&input);
553
554 let (forwards, adapts, delta) = engine.stats();
555 assert_eq!(forwards, 1);
556 assert_eq!(adapts, 1);
557 assert!(delta >= 0.0);
558 }
559}