1use std::collections::HashMap;
28
29use trustformers_core::{
30 errors::{invalid_input, TrustformersError},
31 tensor::Tensor,
32};
33
34use super::{
35 Example, ExampleSet, MetaLearningConfig, MetaLearningModel, ModelGradients, ModelParameters,
36};
37
38const HVP_EPSILON: f64 = 1e-3;
41
42#[derive(Debug, Clone)]
44struct DenseLayer {
45 weight: Vec<f32>,
47 bias: Vec<f32>,
48 in_dim: usize,
49 out_dim: usize,
50}
51
52impl DenseLayer {
53 fn new(in_dim: usize, out_dim: usize, seed: usize) -> Self {
59 let scale = (1.0 / in_dim.max(1) as f32).sqrt();
60 let mut weight = Vec::with_capacity(in_dim * out_dim);
61 for index in 0..in_dim * out_dim {
62 let phase = ((index + seed * 7 + 1) as f32) * 0.754_877_7;
63 weight.push(((phase.fract() * 2.0) - 1.0) * scale);
64 }
65 Self {
66 weight,
67 bias: vec![0.0; out_dim],
68 in_dim,
69 out_dim,
70 }
71 }
72
73 fn forward(&self, input: &[f32]) -> Vec<f32> {
74 let mut output = self.bias.clone();
75 for row in 0..self.out_dim {
76 let offset = row * self.in_dim;
77 let mut sum = output[row];
78 for column in 0..self.in_dim {
79 sum += self.weight[offset + column] * input[column];
80 }
81 output[row] = sum;
82 }
83 output
84 }
85}
86
87#[derive(Debug, Clone)]
93pub struct MlpLearner {
94 layers: Vec<DenseLayer>,
95}
96
97impl MlpLearner {
98 pub fn new(
101 input_dim: usize,
102 hidden_dim: usize,
103 output_dim: usize,
104 ) -> Result<Self, TrustformersError> {
105 if input_dim == 0 || hidden_dim == 0 || output_dim == 0 {
106 return Err(invalid_input(
107 "MlpLearner dimensions must all be greater than zero",
108 ));
109 }
110 Ok(Self {
111 layers: vec![
112 DenseLayer::new(input_dim, hidden_dim, 0),
113 DenseLayer::new(hidden_dim, output_dim, 1),
114 ],
115 })
116 }
117
118 pub fn input_dim(&self) -> usize {
120 self.layers[0].in_dim
121 }
122
123 pub fn output_dim(&self) -> usize {
125 self.layers[self.layers.len() - 1].out_dim
126 }
127
128 fn forward_cached(&self, input: &[f32]) -> (Vec<Vec<f32>>, Vec<f32>) {
131 let mut activations = vec![input.to_vec()];
132 let mut current = input.to_vec();
133
134 for (index, layer) in self.layers.iter().enumerate() {
135 let mut output = layer.forward(¤t);
136 if index + 1 < self.layers.len() {
137 for value in output.iter_mut() {
138 *value = value.tanh();
139 }
140 }
141 activations.push(output.clone());
142 current = output;
143 }
144
145 (activations, current)
146 }
147
148 pub fn predict(&self, input: &[f32]) -> Vec<f32> {
150 self.forward_cached(input).1
151 }
152
153 pub fn embed(&self, input: &[f32]) -> Vec<f32> {
155 let (activations, _) = self.forward_cached(input);
156 activations[activations.len() - 2].clone()
157 }
158
159 fn backward(
162 &self,
163 activations: &[Vec<f32>],
164 output_gradient: &[f32],
165 weight_grads: &mut [Vec<f32>],
166 bias_grads: &mut [Vec<f32>],
167 ) {
168 let mut delta = output_gradient.to_vec();
169
170 for index in (0..self.layers.len()).rev() {
171 let layer = &self.layers[index];
172 let input = &activations[index];
173
174 for row in 0..layer.out_dim {
175 bias_grads[index][row] += delta[row];
176 let offset = row * layer.in_dim;
177 for column in 0..layer.in_dim {
178 weight_grads[index][offset + column] += delta[row] * input[column];
179 }
180 }
181
182 if index == 0 {
183 break;
184 }
185
186 let mut next_delta = vec![0.0f32; layer.in_dim];
188 for column in 0..layer.in_dim {
189 let mut sum = 0.0f32;
190 for row in 0..layer.out_dim {
191 sum += layer.weight[row * layer.in_dim + column] * delta[row];
192 }
193 let activation = activations[index][column];
195 next_delta[column] = sum * (1.0 - activation * activation);
196 }
197 delta = next_delta;
198 }
199 }
200
201 fn zero_gradients(&self) -> (Vec<Vec<f32>>, Vec<Vec<f32>>) {
202 (
203 self.layers.iter().map(|l| vec![0.0f32; l.weight.len()]).collect(),
204 self.layers.iter().map(|l| vec![0.0f32; l.bias.len()]).collect(),
205 )
206 }
207
208 pub fn loss(&self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
210 let (loss, _) = self.loss_and_gradients(examples, false)?;
211 Ok(loss)
212 }
213
214 fn loss_and_gradients(
216 &self,
217 examples: &ExampleSet,
218 want_gradients: bool,
219 ) -> Result<(f64, Option<ModelGradients>), TrustformersError> {
220 if examples.examples.is_empty() {
221 return Err(invalid_input("cannot score an empty example set"));
222 }
223
224 let (mut weight_grads, mut bias_grads) = self.zero_gradients();
225 let mut total_loss = 0.0f64;
226 let count = examples.examples.len() as f32;
227
228 for example in &examples.examples {
229 let input = self.example_input(example)?;
230 let (activations, output) = self.forward_cached(&input);
231
232 let output_gradient = match &example.target {
233 Some(target) => {
234 let target_values = target.to_vec_f32()?;
236 if target_values.len() != output.len() {
237 return Err(invalid_input(format!(
238 "regression target has {} values but the learner produces {}",
239 target_values.len(),
240 output.len()
241 )));
242 }
243 let mut gradient = vec![0.0f32; output.len()];
244 for ((slot, prediction), truth) in
245 gradient.iter_mut().zip(output.iter()).zip(target_values.iter())
246 {
247 let residual = prediction - truth;
248 total_loss += (residual * residual) as f64;
249 *slot = 2.0 * residual / count;
250 }
251 gradient
252 },
253 None => {
254 if example.label >= output.len() {
256 return Err(invalid_input(format!(
257 "label {} is out of range for {} outputs",
258 example.label,
259 output.len()
260 )));
261 }
262 let probabilities = softmax(&output);
263 total_loss -= (probabilities[example.label].max(1e-12) as f64).ln();
264 let mut gradient = probabilities;
265 gradient[example.label] -= 1.0;
266 for value in gradient.iter_mut() {
267 *value /= count;
268 }
269 gradient
270 },
271 };
272
273 if want_gradients {
274 self.backward(
275 &activations,
276 &output_gradient,
277 &mut weight_grads,
278 &mut bias_grads,
279 );
280 }
281 }
282
283 let mean_loss = total_loss / count as f64;
284 if !want_gradients {
285 return Ok((mean_loss, None));
286 }
287
288 let mut gradients = ModelGradients::new();
289 for (index, layer) in self.layers.iter().enumerate() {
290 gradients.gradients.insert(
291 format!("layer{}.weight", index),
292 Tensor::from_vec(
293 std::mem::take(&mut weight_grads[index]),
294 &[layer.out_dim, layer.in_dim],
295 )?,
296 );
297 gradients.gradients.insert(
298 format!("layer{}.bias", index),
299 Tensor::from_vec(std::mem::take(&mut bias_grads[index]), &[layer.out_dim])?,
300 );
301 }
302
303 Ok((mean_loss, Some(gradients)))
304 }
305
306 pub fn gradients(&self, examples: &ExampleSet) -> Result<ModelGradients, TrustformersError> {
308 let (_, gradients) = self.loss_and_gradients(examples, true)?;
309 gradients.ok_or_else(|| invalid_input("gradient computation produced no gradients"))
310 }
311
312 pub fn accuracy(&self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
315 if examples.examples.is_empty() {
316 return Err(invalid_input("cannot score an empty example set"));
317 }
318
319 let regression = examples.examples.iter().all(|e| e.target.is_some());
320 if regression {
321 let mut residual = 0.0f64;
322 let mut targets = Vec::new();
323 for example in &examples.examples {
324 let input = self.example_input(example)?;
325 let prediction = self.predict(&input);
326 let Some(target) = &example.target else {
327 return Err(invalid_input("regression example is missing its target"));
328 };
329 let truth = target.to_vec_f32()?;
330 for (p, t) in prediction.iter().zip(truth.iter()) {
331 residual += ((p - t) * (p - t)) as f64;
332 targets.push(*t as f64);
333 }
334 }
335 let mean = targets.iter().sum::<f64>() / targets.len().max(1) as f64;
336 let variance: f64 = targets.iter().map(|t| (t - mean) * (t - mean)).sum();
337 if variance <= f64::EPSILON {
338 return Ok(if residual <= f64::EPSILON { 1.0 } else { 0.0 });
339 }
340 return Ok((1.0 - residual / variance).clamp(0.0, 1.0));
341 }
342
343 let mut correct = 0usize;
344 for example in &examples.examples {
345 let input = self.example_input(example)?;
346 let prediction = self.predict(&input);
347 if argmax(&prediction) == example.label {
348 correct += 1;
349 }
350 }
351 Ok(correct as f64 / examples.examples.len() as f64)
352 }
353
354 fn example_input(&self, example: &Example) -> Result<Vec<f32>, TrustformersError> {
356 let values = example.input.to_vec_f32()?;
357 if values.len() != self.input_dim() {
358 return Err(invalid_input(format!(
359 "example has {} features but the learner expects {}",
360 values.len(),
361 self.input_dim()
362 )));
363 }
364 Ok(values)
365 }
366
367 pub fn parameters(&self) -> Result<ModelParameters, TrustformersError> {
369 let mut parameters = HashMap::new();
370 for (index, layer) in self.layers.iter().enumerate() {
371 parameters.insert(
372 format!("layer{}.weight", index),
373 Tensor::from_vec(layer.weight.clone(), &[layer.out_dim, layer.in_dim])?,
374 );
375 parameters.insert(
376 format!("layer{}.bias", index),
377 Tensor::from_vec(layer.bias.clone(), &[layer.out_dim])?,
378 );
379 }
380 Ok(ModelParameters { parameters })
381 }
382
383 pub fn set_parameters(&mut self, params: &ModelParameters) -> Result<(), TrustformersError> {
385 for (index, layer) in self.layers.iter_mut().enumerate() {
386 if let Some(weight) = params.parameters.get(&format!("layer{}.weight", index)) {
387 let values = weight.to_vec_f32()?;
388 if values.len() != layer.weight.len() {
389 return Err(invalid_input(format!(
390 "layer{}.weight has {} values, expected {}",
391 index,
392 values.len(),
393 layer.weight.len()
394 )));
395 }
396 layer.weight = values;
397 }
398 if let Some(bias) = params.parameters.get(&format!("layer{}.bias", index)) {
399 let values = bias.to_vec_f32()?;
400 if values.len() != layer.bias.len() {
401 return Err(invalid_input(format!(
402 "layer{}.bias has {} values, expected {}",
403 index,
404 values.len(),
405 layer.bias.len()
406 )));
407 }
408 layer.bias = values;
409 }
410 }
411 Ok(())
412 }
413
414 pub fn apply_gradients(
416 &mut self,
417 gradients: &ModelGradients,
418 lr: f64,
419 ) -> Result<(), TrustformersError> {
420 for (index, layer) in self.layers.iter_mut().enumerate() {
421 if let Some(gradient) = gradients.gradients.get(&format!("layer{}.weight", index)) {
422 let values = gradient.to_vec_f32()?;
423 if values.len() != layer.weight.len() {
424 return Err(invalid_input(format!(
425 "gradient for layer{}.weight has the wrong length",
426 index
427 )));
428 }
429 for (parameter, g) in layer.weight.iter_mut().zip(values) {
430 *parameter -= lr as f32 * g;
431 }
432 }
433 if let Some(gradient) = gradients.gradients.get(&format!("layer{}.bias", index)) {
434 let values = gradient.to_vec_f32()?;
435 for (parameter, g) in layer.bias.iter_mut().zip(values) {
436 *parameter -= lr as f32 * g;
437 }
438 }
439 }
440 Ok(())
441 }
442
443 pub fn apply_gradients_with_lr(
446 &mut self,
447 gradients: &ModelGradients,
448 learning_rates: &[f64],
449 ) -> Result<(), TrustformersError> {
450 let mut cursor = 0usize;
451 for (index, layer) in self.layers.iter_mut().enumerate() {
452 for (name, slots) in [
453 (format!("layer{}.weight", index), layer.weight.len()),
454 (format!("layer{}.bias", index), layer.bias.len()),
455 ] {
456 let Some(gradient) = gradients.gradients.get(&name) else {
457 cursor += slots;
458 continue;
459 };
460 let values = gradient.to_vec_f32()?;
461 for (offset, g) in values.iter().enumerate() {
462 let lr = learning_rates.get(cursor + offset).copied().ok_or_else(|| {
463 invalid_input("learned learning-rate vector is shorter than the parameters")
464 })? as f32;
465 if name.ends_with("weight") {
466 layer.weight[offset] -= lr * g;
467 } else {
468 layer.bias[offset] -= lr * g;
469 }
470 }
471 cursor += slots;
472 }
473 }
474 Ok(())
475 }
476
477 pub fn parameter_count(&self) -> usize {
479 self.layers.iter().map(|l| l.weight.len() + l.bias.len()).sum()
480 }
481}
482
483fn softmax(values: &[f32]) -> Vec<f32> {
484 let max = values.iter().copied().fold(f32::NEG_INFINITY, f32::max);
485 let exponentials: Vec<f32> = values.iter().map(|v| (v - max).exp()).collect();
486 let total: f32 = exponentials.iter().sum();
487 if total <= 0.0 {
488 return vec![1.0 / values.len().max(1) as f32; values.len()];
489 }
490 exponentials.into_iter().map(|e| e / total).collect()
491}
492
493fn argmax(values: &[f32]) -> usize {
494 values
495 .iter()
496 .enumerate()
497 .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
498 .map(|(index, _)| index)
499 .unwrap_or(0)
500}
501
502#[derive(Debug, Clone)]
504pub struct MetaMlp {
505 config: MetaLearningConfig,
506 learner: MlpLearner,
507 last_batch: Option<ExampleSet>,
509 previous_batch: Option<ExampleSet>,
511 learning_rates: Vec<f64>,
513}
514
515impl MetaMlp {
516 fn new(config: &MetaLearningConfig) -> Result<Self, TrustformersError> {
517 let learner = MlpLearner::new(
518 config.embedding_dim,
519 config.embedding_dim.clamp(4, 64),
520 config.num_ways.max(1),
521 )?;
522 let learning_rates = vec![config.inner_lr; learner.parameter_count()];
523 Ok(Self {
524 config: config.clone(),
525 learner,
526 last_batch: None,
527 previous_batch: None,
528 learning_rates,
529 })
530 }
531
532 fn cached_batch(&self) -> Result<&ExampleSet, TrustformersError> {
533 self.last_batch
534 .as_ref()
535 .ok_or_else(|| invalid_input("compute_gradients requires a preceding call to forward"))
536 }
537}
538
539impl MetaLearningModel for MetaMlp {
540 fn forward(&mut self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
541 let loss = self.learner.loss(examples)?;
542 self.previous_batch = self.last_batch.take();
543 self.last_batch = Some(examples.clone());
544 Ok(loss)
545 }
546
547 fn compute_accuracy(&self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
548 self.learner.accuracy(examples)
549 }
550
551 fn compute_gradients(&self, _loss: f64) -> Result<ModelGradients, TrustformersError> {
552 self.learner.gradients(self.cached_batch()?)
556 }
557
558 fn apply_gradients(
559 &mut self,
560 gradients: &ModelGradients,
561 lr: f64,
562 ) -> Result<(), TrustformersError> {
563 self.learner.apply_gradients(gradients, lr)
564 }
565
566 fn get_parameters(&self) -> Result<ModelParameters, TrustformersError> {
567 self.learner.parameters()
568 }
569
570 fn set_parameters(&mut self, params: ModelParameters) -> Result<(), TrustformersError> {
571 self.learner.set_parameters(¶ms)
572 }
573
574 fn embed(&self, example: &Example) -> Result<Tensor, TrustformersError> {
575 let input = self.learner.example_input(example)?;
576 let embedding = self.learner.embed(&input);
577 let length = embedding.len();
578 let tensor = Tensor::from_vec(embedding, &[length])?;
579 Ok(tensor)
580 }
581
582 fn compute_first_order_gradients(
583 &self,
584 _loss: f64,
585 ) -> Result<ModelGradients, TrustformersError> {
586 self.learner.gradients(self.cached_batch()?)
589 }
590
591 fn compute_second_order_gradients(
592 &self,
593 _initial_params: &ModelParameters,
594 _loss: f64,
595 ) -> Result<ModelGradients, TrustformersError> {
596 let query = self.cached_batch()?;
600 let Some(support) = self.previous_batch.as_ref() else {
601 return Err(invalid_input(
602 "second-order meta-gradients need both the support and the query forward pass",
603 ));
604 };
605
606 let v = self.learner.gradients(query)?;
607 let epsilon = HVP_EPSILON;
608
609 let mut plus = self.learner.clone();
610 plus.apply_gradients(&v, -epsilon)?;
611 let grad_plus = plus.gradients(support)?;
612
613 let mut minus = self.learner.clone();
614 minus.apply_gradients(&v, epsilon)?;
615 let grad_minus = minus.gradients(support)?;
616
617 let alpha = self.config.inner_lr as f32;
618 let mut meta = ModelGradients::new();
619 for (name, value) in &v.gradients {
620 let base = value.to_vec_f32()?;
621 let shape = value.shape();
622 let (Some(up), Some(down)) = (
623 grad_plus.gradients.get(name),
624 grad_minus.gradients.get(name),
625 ) else {
626 meta.gradients.insert(name.clone(), value.clone());
627 continue;
628 };
629 let up = up.to_vec_f32()?;
630 let down = down.to_vec_f32()?;
631 let mut combined = Vec::with_capacity(base.len());
632 for index in 0..base.len() {
633 let hvp = (up[index] - down[index]) / (2.0 * epsilon as f32);
634 combined.push(base[index] - alpha * hvp);
635 }
636 meta.gradients.insert(name.clone(), Tensor::from_vec(combined, &shape)?);
637 }
638
639 Ok(meta)
640 }
641
642 fn compute_relation(&self, a: &Tensor, b: &Tensor) -> Result<f64, TrustformersError> {
643 let left = a.to_vec_f32()?;
646 let right = b.to_vec_f32()?;
647 if left.len() != right.len() {
648 return Err(invalid_input(
649 "relation scores need two embeddings of the same width",
650 ));
651 }
652 let distance: f32 =
653 left.iter().zip(right.iter()).map(|(x, y)| (x - y) * (x - y)).sum::<f32>();
654 Ok((-(distance as f64)).exp())
655 }
656
657 fn get_learning_rates(&self) -> Result<Vec<f64>, TrustformersError> {
658 Ok(self.learning_rates.clone())
659 }
660
661 fn apply_gradients_with_lr(
662 &mut self,
663 gradients: &ModelGradients,
664 learning_rates: &[f64],
665 ) -> Result<(), TrustformersError> {
666 self.learner.apply_gradients_with_lr(gradients, learning_rates)
667 }
668
669 fn compute_lr_gradients(&self, _loss: f64) -> Result<Vec<f64>, TrustformersError> {
670 let query = self.cached_batch()?;
674 let Some(support) = self.previous_batch.as_ref() else {
675 return Err(invalid_input(
676 "learning-rate gradients need both the support and the query forward pass",
677 ));
678 };
679
680 let query_gradients = self.learner.gradients(query)?;
681 let support_gradients = self.learner.gradients(support)?;
682
683 let mut output = Vec::with_capacity(self.learning_rates.len());
684 for index in 0..(self.learner.layers.len()) {
685 for suffix in ["weight", "bias"] {
686 let name = format!("layer{}.{}", index, suffix);
687 let (Some(q), Some(s)) = (
688 query_gradients.gradients.get(&name),
689 support_gradients.gradients.get(&name),
690 ) else {
691 continue;
692 };
693 let q = q.to_vec_f32()?;
694 let s = s.to_vec_f32()?;
695 for (a, b) in q.iter().zip(s.iter()) {
696 output.push(-(*a as f64) * (*b as f64));
697 }
698 }
699 }
700
701 Ok(output)
702 }
703}
704
705macro_rules! meta_model {
707 ($(#[$meta:meta])* $name:ident $(, $extra:item)* $(,)?) => {
708 $(#[$meta])*
709 #[derive(Debug, Clone)]
710 pub struct $name {
711 inner: MetaMlp,
712 }
713
714 impl $name {
715 pub fn new(config: &MetaLearningConfig) -> Result<Self, TrustformersError> {
717 Ok(Self {
718 inner: MetaMlp::new(config)?,
719 })
720 }
721
722 pub fn learner(&self) -> &MlpLearner {
724 &self.inner.learner
725 }
726 }
727
728 impl MetaLearningModel for $name {
729 fn forward(&mut self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
730 self.inner.forward(examples)
731 }
732
733 fn compute_accuracy(&self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
734 self.inner.compute_accuracy(examples)
735 }
736
737 fn compute_gradients(&self, loss: f64) -> Result<ModelGradients, TrustformersError> {
738 self.inner.compute_gradients(loss)
739 }
740
741 fn apply_gradients(
742 &mut self,
743 gradients: &ModelGradients,
744 lr: f64,
745 ) -> Result<(), TrustformersError> {
746 self.inner.apply_gradients(gradients, lr)
747 }
748
749 fn get_parameters(&self) -> Result<ModelParameters, TrustformersError> {
750 self.inner.get_parameters()
751 }
752
753 fn set_parameters(
754 &mut self,
755 params: ModelParameters,
756 ) -> Result<(), TrustformersError> {
757 self.inner.set_parameters(params)
758 }
759
760 fn embed(&self, example: &Example) -> Result<Tensor, TrustformersError> {
761 self.inner.embed(example)
762 }
763
764 $($extra)*
765 }
766 };
767}
768
769meta_model!(
770 MAMLModel,
777 fn compute_first_order_gradients(&self, loss: f64) -> Result<ModelGradients, TrustformersError> {
778 self.inner.compute_first_order_gradients(loss)
779 },
780 fn compute_second_order_gradients(
781 &self,
782 initial_params: &ModelParameters,
783 loss: f64,
784 ) -> Result<ModelGradients, TrustformersError> {
785 self.inner.compute_second_order_gradients(initial_params, loss)
786 },
787);
788
789meta_model!(
790 ReptileModel,
793);
794
795meta_model!(
796 PrototypicalModel,
799);
800
801meta_model!(
802 MatchingNetModel,
804);
805
806meta_model!(
807 RelationNetModel,
809 fn compute_relation(&self, a: &Tensor, b: &Tensor) -> Result<f64, TrustformersError> {
810 self.inner.compute_relation(a, b)
811 },
812);
813
814meta_model!(
815 MemoryAugmentedModel,
822);
823
824meta_model!(
825 GradientBasedModel,
830);
831
832meta_model!(
833 MetaSGDModel,
835 fn compute_second_order_gradients(
836 &self,
837 initial_params: &ModelParameters,
838 loss: f64,
839 ) -> Result<ModelGradients, TrustformersError> {
840 self.inner.compute_second_order_gradients(initial_params, loss)
841 },
842 fn get_learning_rates(&self) -> Result<Vec<f64>, TrustformersError> {
843 self.inner.get_learning_rates()
844 },
845 fn apply_gradients_with_lr(
846 &mut self,
847 gradients: &ModelGradients,
848 learning_rates: &[f64],
849 ) -> Result<(), TrustformersError> {
850 self.inner.apply_gradients_with_lr(gradients, learning_rates)
851 },
852 fn compute_lr_gradients(&self, loss: f64) -> Result<Vec<f64>, TrustformersError> {
853 self.inner.compute_lr_gradients(loss)
854 },
855);
856
857meta_model!(
858 L2LModel,
863);