Skip to main content

trustformers_models/meta_learning/
models.rs

1//! Concrete meta-learning models.
2//!
3//! Every model in this module is backed by [`MlpLearner`], a small
4//! fully-connected network with hand-written forward and backward passes. The
5//! gradients it returns are exact analytic gradients of the loss it reports,
6//! which is what makes the inner/outer loops in
7//! [`crate::meta_learning::MetaLearner`] real learning rather than bookkeeping.
8//!
9//! ## Which algorithms are implemented
10//!
11//! | Model | Status |
12//! |---|---|
13//! | [`MAMLModel`] | full: first-order (FOMAML) and second-order meta-gradients |
14//! | [`ReptileModel`] | full: the outer step is the parameter difference |
15//! | [`PrototypicalModel`] | full: real embeddings, class prototypes, distance classification |
16//! | [`MatchingNetModel`] | full: cosine attention over support embeddings |
17//! | [`RelationNetModel`] | full: learned embedding + relation score |
18//! | [`MetaSGDModel`] | full: per-parameter learned learning rates |
19//! | [`MemoryAugmentedModel`] | learner only; the external memory is not implemented |
20//! | [`GradientBasedModel`] | learner only; the learned optimizer is not implemented |
21//! | [`L2LModel`] | learner only; the LSTM meta-learner is not implemented |
22//!
23//! The last three inherit the default trait methods, which report
24//! `unsupported_operation` for the parts that do not exist. They never return
25//! invented metrics.
26
27use std::collections::HashMap;
28
29use trustformers_core::{
30    errors::{invalid_input, TrustformersError},
31    tensor::Tensor,
32};
33
34use super::{
35    Example, ExampleSet, MetaLearningConfig, MetaLearningModel, ModelGradients, ModelParameters,
36};
37
38/// Step size for the central-difference Hessian-vector product used by the
39/// second-order MAML meta-gradient.
40const HVP_EPSILON: f64 = 1e-3;
41
42/// A dense layer with an explicit backward pass.
43#[derive(Debug, Clone)]
44struct DenseLayer {
45    /// Row-major `[out_dim, in_dim]`
46    weight: Vec<f32>,
47    bias: Vec<f32>,
48    in_dim: usize,
49    out_dim: usize,
50}
51
52impl DenseLayer {
53    /// Deterministic fan-in scaled initialisation.
54    ///
55    /// The weights follow a fixed low-discrepancy pattern rather than an RNG so
56    /// that two learners built from the same configuration start identical,
57    /// which is what the meta-learning outer loop assumes.
58    fn new(in_dim: usize, out_dim: usize, seed: usize) -> Self {
59        let scale = (1.0 / in_dim.max(1) as f32).sqrt();
60        let mut weight = Vec::with_capacity(in_dim * out_dim);
61        for index in 0..in_dim * out_dim {
62            let phase = ((index + seed * 7 + 1) as f32) * 0.754_877_7;
63            weight.push(((phase.fract() * 2.0) - 1.0) * scale);
64        }
65        Self {
66            weight,
67            bias: vec![0.0; out_dim],
68            in_dim,
69            out_dim,
70        }
71    }
72
73    fn forward(&self, input: &[f32]) -> Vec<f32> {
74        let mut output = self.bias.clone();
75        for row in 0..self.out_dim {
76            let offset = row * self.in_dim;
77            let mut sum = output[row];
78            for column in 0..self.in_dim {
79                sum += self.weight[offset + column] * input[column];
80            }
81            output[row] = sum;
82        }
83        output
84    }
85}
86
87/// A small multi-layer perceptron with analytic gradients.
88///
89/// The hidden layers use `tanh`; the output layer is linear. Classification
90/// examples are scored with softmax cross-entropy and regression examples with
91/// mean squared error.
92#[derive(Debug, Clone)]
93pub struct MlpLearner {
94    layers: Vec<DenseLayer>,
95}
96
97impl MlpLearner {
98    /// Build a learner mapping `input_dim` features to `output_dim` outputs
99    /// through one hidden layer of `hidden_dim` units.
100    pub fn new(
101        input_dim: usize,
102        hidden_dim: usize,
103        output_dim: usize,
104    ) -> Result<Self, TrustformersError> {
105        if input_dim == 0 || hidden_dim == 0 || output_dim == 0 {
106            return Err(invalid_input(
107                "MlpLearner dimensions must all be greater than zero",
108            ));
109        }
110        Ok(Self {
111            layers: vec![
112                DenseLayer::new(input_dim, hidden_dim, 0),
113                DenseLayer::new(hidden_dim, output_dim, 1),
114            ],
115        })
116    }
117
118    /// Width of the network's input.
119    pub fn input_dim(&self) -> usize {
120        self.layers[0].in_dim
121    }
122
123    /// Width of the network's output.
124    pub fn output_dim(&self) -> usize {
125        self.layers[self.layers.len() - 1].out_dim
126    }
127
128    /// Forward pass returning the pre-activation of every layer plus the
129    /// post-activation of the hidden layers.
130    fn forward_cached(&self, input: &[f32]) -> (Vec<Vec<f32>>, Vec<f32>) {
131        let mut activations = vec![input.to_vec()];
132        let mut current = input.to_vec();
133
134        for (index, layer) in self.layers.iter().enumerate() {
135            let mut output = layer.forward(&current);
136            if index + 1 < self.layers.len() {
137                for value in output.iter_mut() {
138                    *value = value.tanh();
139                }
140            }
141            activations.push(output.clone());
142            current = output;
143        }
144
145        (activations, current)
146    }
147
148    /// Network output for one input vector.
149    pub fn predict(&self, input: &[f32]) -> Vec<f32> {
150        self.forward_cached(input).1
151    }
152
153    /// Hidden representation used as the embedding by metric-based algorithms.
154    pub fn embed(&self, input: &[f32]) -> Vec<f32> {
155        let (activations, _) = self.forward_cached(input);
156        activations[activations.len() - 2].clone()
157    }
158
159    /// Backpropagate `output_gradient` through the network, accumulating into
160    /// `weight_grads` / `bias_grads`.
161    fn backward(
162        &self,
163        activations: &[Vec<f32>],
164        output_gradient: &[f32],
165        weight_grads: &mut [Vec<f32>],
166        bias_grads: &mut [Vec<f32>],
167    ) {
168        let mut delta = output_gradient.to_vec();
169
170        for index in (0..self.layers.len()).rev() {
171            let layer = &self.layers[index];
172            let input = &activations[index];
173
174            for row in 0..layer.out_dim {
175                bias_grads[index][row] += delta[row];
176                let offset = row * layer.in_dim;
177                for column in 0..layer.in_dim {
178                    weight_grads[index][offset + column] += delta[row] * input[column];
179                }
180            }
181
182            if index == 0 {
183                break;
184            }
185
186            // Propagate through the previous layer's tanh.
187            let mut next_delta = vec![0.0f32; layer.in_dim];
188            for column in 0..layer.in_dim {
189                let mut sum = 0.0f32;
190                for row in 0..layer.out_dim {
191                    sum += layer.weight[row * layer.in_dim + column] * delta[row];
192                }
193                // activations[index] is the tanh output of the previous layer.
194                let activation = activations[index][column];
195                next_delta[column] = sum * (1.0 - activation * activation);
196            }
197            delta = next_delta;
198        }
199    }
200
201    fn zero_gradients(&self) -> (Vec<Vec<f32>>, Vec<Vec<f32>>) {
202        (
203            self.layers.iter().map(|l| vec![0.0f32; l.weight.len()]).collect(),
204            self.layers.iter().map(|l| vec![0.0f32; l.bias.len()]).collect(),
205        )
206    }
207
208    /// Mean loss over a set of examples.
209    pub fn loss(&self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
210        let (loss, _) = self.loss_and_gradients(examples, false)?;
211        Ok(loss)
212    }
213
214    /// Mean loss and, when requested, the analytic gradient of that loss.
215    fn loss_and_gradients(
216        &self,
217        examples: &ExampleSet,
218        want_gradients: bool,
219    ) -> Result<(f64, Option<ModelGradients>), TrustformersError> {
220        if examples.examples.is_empty() {
221            return Err(invalid_input("cannot score an empty example set"));
222        }
223
224        let (mut weight_grads, mut bias_grads) = self.zero_gradients();
225        let mut total_loss = 0.0f64;
226        let count = examples.examples.len() as f32;
227
228        for example in &examples.examples {
229            let input = self.example_input(example)?;
230            let (activations, output) = self.forward_cached(&input);
231
232            let output_gradient = match &example.target {
233                Some(target) => {
234                    // Regression: mean squared error.
235                    let target_values = target.to_vec_f32()?;
236                    if target_values.len() != output.len() {
237                        return Err(invalid_input(format!(
238                            "regression target has {} values but the learner produces {}",
239                            target_values.len(),
240                            output.len()
241                        )));
242                    }
243                    let mut gradient = vec![0.0f32; output.len()];
244                    for ((slot, prediction), truth) in
245                        gradient.iter_mut().zip(output.iter()).zip(target_values.iter())
246                    {
247                        let residual = prediction - truth;
248                        total_loss += (residual * residual) as f64;
249                        *slot = 2.0 * residual / count;
250                    }
251                    gradient
252                },
253                None => {
254                    // Classification: softmax cross-entropy.
255                    if example.label >= output.len() {
256                        return Err(invalid_input(format!(
257                            "label {} is out of range for {} outputs",
258                            example.label,
259                            output.len()
260                        )));
261                    }
262                    let probabilities = softmax(&output);
263                    total_loss -= (probabilities[example.label].max(1e-12) as f64).ln();
264                    let mut gradient = probabilities;
265                    gradient[example.label] -= 1.0;
266                    for value in gradient.iter_mut() {
267                        *value /= count;
268                    }
269                    gradient
270                },
271            };
272
273            if want_gradients {
274                self.backward(
275                    &activations,
276                    &output_gradient,
277                    &mut weight_grads,
278                    &mut bias_grads,
279                );
280            }
281        }
282
283        let mean_loss = total_loss / count as f64;
284        if !want_gradients {
285            return Ok((mean_loss, None));
286        }
287
288        let mut gradients = ModelGradients::new();
289        for (index, layer) in self.layers.iter().enumerate() {
290            gradients.gradients.insert(
291                format!("layer{}.weight", index),
292                Tensor::from_vec(
293                    std::mem::take(&mut weight_grads[index]),
294                    &[layer.out_dim, layer.in_dim],
295                )?,
296            );
297            gradients.gradients.insert(
298                format!("layer{}.bias", index),
299                Tensor::from_vec(std::mem::take(&mut bias_grads[index]), &[layer.out_dim])?,
300            );
301        }
302
303        Ok((mean_loss, Some(gradients)))
304    }
305
306    /// Analytic gradient of the mean loss over `examples`.
307    pub fn gradients(&self, examples: &ExampleSet) -> Result<ModelGradients, TrustformersError> {
308        let (_, gradients) = self.loss_and_gradients(examples, true)?;
309        gradients.ok_or_else(|| invalid_input("gradient computation produced no gradients"))
310    }
311
312    /// Classification accuracy, or the coefficient of determination for
313    /// regression example sets.
314    pub fn accuracy(&self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
315        if examples.examples.is_empty() {
316            return Err(invalid_input("cannot score an empty example set"));
317        }
318
319        let regression = examples.examples.iter().all(|e| e.target.is_some());
320        if regression {
321            let mut residual = 0.0f64;
322            let mut targets = Vec::new();
323            for example in &examples.examples {
324                let input = self.example_input(example)?;
325                let prediction = self.predict(&input);
326                let Some(target) = &example.target else {
327                    return Err(invalid_input("regression example is missing its target"));
328                };
329                let truth = target.to_vec_f32()?;
330                for (p, t) in prediction.iter().zip(truth.iter()) {
331                    residual += ((p - t) * (p - t)) as f64;
332                    targets.push(*t as f64);
333                }
334            }
335            let mean = targets.iter().sum::<f64>() / targets.len().max(1) as f64;
336            let variance: f64 = targets.iter().map(|t| (t - mean) * (t - mean)).sum();
337            if variance <= f64::EPSILON {
338                return Ok(if residual <= f64::EPSILON { 1.0 } else { 0.0 });
339            }
340            return Ok((1.0 - residual / variance).clamp(0.0, 1.0));
341        }
342
343        let mut correct = 0usize;
344        for example in &examples.examples {
345            let input = self.example_input(example)?;
346            let prediction = self.predict(&input);
347            if argmax(&prediction) == example.label {
348                correct += 1;
349            }
350        }
351        Ok(correct as f64 / examples.examples.len() as f64)
352    }
353
354    /// Flattened input vector of an example, validated against the input width.
355    fn example_input(&self, example: &Example) -> Result<Vec<f32>, TrustformersError> {
356        let values = example.input.to_vec_f32()?;
357        if values.len() != self.input_dim() {
358            return Err(invalid_input(format!(
359                "example has {} features but the learner expects {}",
360                values.len(),
361                self.input_dim()
362            )));
363        }
364        Ok(values)
365    }
366
367    /// Every parameter tensor, named `layer{i}.weight` / `layer{i}.bias`.
368    pub fn parameters(&self) -> Result<ModelParameters, TrustformersError> {
369        let mut parameters = HashMap::new();
370        for (index, layer) in self.layers.iter().enumerate() {
371            parameters.insert(
372                format!("layer{}.weight", index),
373                Tensor::from_vec(layer.weight.clone(), &[layer.out_dim, layer.in_dim])?,
374            );
375            parameters.insert(
376                format!("layer{}.bias", index),
377                Tensor::from_vec(layer.bias.clone(), &[layer.out_dim])?,
378            );
379        }
380        Ok(ModelParameters { parameters })
381    }
382
383    /// Overwrite the parameters from a [`ModelParameters`] map.
384    pub fn set_parameters(&mut self, params: &ModelParameters) -> Result<(), TrustformersError> {
385        for (index, layer) in self.layers.iter_mut().enumerate() {
386            if let Some(weight) = params.parameters.get(&format!("layer{}.weight", index)) {
387                let values = weight.to_vec_f32()?;
388                if values.len() != layer.weight.len() {
389                    return Err(invalid_input(format!(
390                        "layer{}.weight has {} values, expected {}",
391                        index,
392                        values.len(),
393                        layer.weight.len()
394                    )));
395                }
396                layer.weight = values;
397            }
398            if let Some(bias) = params.parameters.get(&format!("layer{}.bias", index)) {
399                let values = bias.to_vec_f32()?;
400                if values.len() != layer.bias.len() {
401                    return Err(invalid_input(format!(
402                        "layer{}.bias has {} values, expected {}",
403                        index,
404                        values.len(),
405                        layer.bias.len()
406                    )));
407                }
408                layer.bias = values;
409            }
410        }
411        Ok(())
412    }
413
414    /// Apply `theta -= lr * gradient` for every named parameter.
415    pub fn apply_gradients(
416        &mut self,
417        gradients: &ModelGradients,
418        lr: f64,
419    ) -> Result<(), TrustformersError> {
420        for (index, layer) in self.layers.iter_mut().enumerate() {
421            if let Some(gradient) = gradients.gradients.get(&format!("layer{}.weight", index)) {
422                let values = gradient.to_vec_f32()?;
423                if values.len() != layer.weight.len() {
424                    return Err(invalid_input(format!(
425                        "gradient for layer{}.weight has the wrong length",
426                        index
427                    )));
428                }
429                for (parameter, g) in layer.weight.iter_mut().zip(values) {
430                    *parameter -= lr as f32 * g;
431                }
432            }
433            if let Some(gradient) = gradients.gradients.get(&format!("layer{}.bias", index)) {
434                let values = gradient.to_vec_f32()?;
435                for (parameter, g) in layer.bias.iter_mut().zip(values) {
436                    *parameter -= lr as f32 * g;
437                }
438            }
439        }
440        Ok(())
441    }
442
443    /// Apply per-parameter learning rates, consumed in the deterministic order
444    /// `layer0.weight, layer0.bias, layer1.weight, …`.
445    pub fn apply_gradients_with_lr(
446        &mut self,
447        gradients: &ModelGradients,
448        learning_rates: &[f64],
449    ) -> Result<(), TrustformersError> {
450        let mut cursor = 0usize;
451        for (index, layer) in self.layers.iter_mut().enumerate() {
452            for (name, slots) in [
453                (format!("layer{}.weight", index), layer.weight.len()),
454                (format!("layer{}.bias", index), layer.bias.len()),
455            ] {
456                let Some(gradient) = gradients.gradients.get(&name) else {
457                    cursor += slots;
458                    continue;
459                };
460                let values = gradient.to_vec_f32()?;
461                for (offset, g) in values.iter().enumerate() {
462                    let lr = learning_rates.get(cursor + offset).copied().ok_or_else(|| {
463                        invalid_input("learned learning-rate vector is shorter than the parameters")
464                    })? as f32;
465                    if name.ends_with("weight") {
466                        layer.weight[offset] -= lr * g;
467                    } else {
468                        layer.bias[offset] -= lr * g;
469                    }
470                }
471                cursor += slots;
472            }
473        }
474        Ok(())
475    }
476
477    /// Total number of scalar parameters.
478    pub fn parameter_count(&self) -> usize {
479        self.layers.iter().map(|l| l.weight.len() + l.bias.len()).sum()
480    }
481}
482
483fn softmax(values: &[f32]) -> Vec<f32> {
484    let max = values.iter().copied().fold(f32::NEG_INFINITY, f32::max);
485    let exponentials: Vec<f32> = values.iter().map(|v| (v - max).exp()).collect();
486    let total: f32 = exponentials.iter().sum();
487    if total <= 0.0 {
488        return vec![1.0 / values.len().max(1) as f32; values.len()];
489    }
490    exponentials.into_iter().map(|e| e / total).collect()
491}
492
493fn argmax(values: &[f32]) -> usize {
494    values
495        .iter()
496        .enumerate()
497        .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
498        .map(|(index, _)| index)
499        .unwrap_or(0)
500}
501
502/// Shared state behind every concrete meta-learning model.
503#[derive(Debug, Clone)]
504pub struct MetaMlp {
505    config: MetaLearningConfig,
506    learner: MlpLearner,
507    /// Most recent batch passed to `forward` (the query set in MAML).
508    last_batch: Option<ExampleSet>,
509    /// Batch passed to `forward` before that (the support set in MAML).
510    previous_batch: Option<ExampleSet>,
511    /// Per-parameter learned learning rates (Meta-SGD).
512    learning_rates: Vec<f64>,
513}
514
515impl MetaMlp {
516    fn new(config: &MetaLearningConfig) -> Result<Self, TrustformersError> {
517        let learner = MlpLearner::new(
518            config.embedding_dim,
519            config.embedding_dim.clamp(4, 64),
520            config.num_ways.max(1),
521        )?;
522        let learning_rates = vec![config.inner_lr; learner.parameter_count()];
523        Ok(Self {
524            config: config.clone(),
525            learner,
526            last_batch: None,
527            previous_batch: None,
528            learning_rates,
529        })
530    }
531
532    fn cached_batch(&self) -> Result<&ExampleSet, TrustformersError> {
533        self.last_batch
534            .as_ref()
535            .ok_or_else(|| invalid_input("compute_gradients requires a preceding call to forward"))
536    }
537}
538
539impl MetaLearningModel for MetaMlp {
540    fn forward(&mut self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
541        let loss = self.learner.loss(examples)?;
542        self.previous_batch = self.last_batch.take();
543        self.last_batch = Some(examples.clone());
544        Ok(loss)
545    }
546
547    fn compute_accuracy(&self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
548        self.learner.accuracy(examples)
549    }
550
551    fn compute_gradients(&self, _loss: f64) -> Result<ModelGradients, TrustformersError> {
552        // Gradients are the analytic gradients of the loss reported by the most
553        // recent `forward`; `_loss` is that scalar and carries no extra
554        // information.
555        self.learner.gradients(self.cached_batch()?)
556    }
557
558    fn apply_gradients(
559        &mut self,
560        gradients: &ModelGradients,
561        lr: f64,
562    ) -> Result<(), TrustformersError> {
563        self.learner.apply_gradients(gradients, lr)
564    }
565
566    fn get_parameters(&self) -> Result<ModelParameters, TrustformersError> {
567        self.learner.parameters()
568    }
569
570    fn set_parameters(&mut self, params: ModelParameters) -> Result<(), TrustformersError> {
571        self.learner.set_parameters(&params)
572    }
573
574    fn embed(&self, example: &Example) -> Result<Tensor, TrustformersError> {
575        let input = self.learner.example_input(example)?;
576        let embedding = self.learner.embed(&input);
577        let length = embedding.len();
578        let tensor = Tensor::from_vec(embedding, &[length])?;
579        Ok(tensor)
580    }
581
582    fn compute_first_order_gradients(
583        &self,
584        _loss: f64,
585    ) -> Result<ModelGradients, TrustformersError> {
586        // FOMAML: the meta-gradient is the query-set gradient evaluated at the
587        // adapted parameters, ignoring the second-order term.
588        self.learner.gradients(self.cached_batch()?)
589    }
590
591    fn compute_second_order_gradients(
592        &self,
593        _initial_params: &ModelParameters,
594        _loss: f64,
595    ) -> Result<ModelGradients, TrustformersError> {
596        // One-step MAML meta-gradient  g = v - alpha * H v  with
597        // v = grad L_query(theta') and H the Hessian of the support loss,
598        // estimated by a central difference of the support gradient.
599        let query = self.cached_batch()?;
600        let Some(support) = self.previous_batch.as_ref() else {
601            return Err(invalid_input(
602                "second-order meta-gradients need both the support and the query forward pass",
603            ));
604        };
605
606        let v = self.learner.gradients(query)?;
607        let epsilon = HVP_EPSILON;
608
609        let mut plus = self.learner.clone();
610        plus.apply_gradients(&v, -epsilon)?;
611        let grad_plus = plus.gradients(support)?;
612
613        let mut minus = self.learner.clone();
614        minus.apply_gradients(&v, epsilon)?;
615        let grad_minus = minus.gradients(support)?;
616
617        let alpha = self.config.inner_lr as f32;
618        let mut meta = ModelGradients::new();
619        for (name, value) in &v.gradients {
620            let base = value.to_vec_f32()?;
621            let shape = value.shape();
622            let (Some(up), Some(down)) = (
623                grad_plus.gradients.get(name),
624                grad_minus.gradients.get(name),
625            ) else {
626                meta.gradients.insert(name.clone(), value.clone());
627                continue;
628            };
629            let up = up.to_vec_f32()?;
630            let down = down.to_vec_f32()?;
631            let mut combined = Vec::with_capacity(base.len());
632            for index in 0..base.len() {
633                let hvp = (up[index] - down[index]) / (2.0 * epsilon as f32);
634                combined.push(base[index] - alpha * hvp);
635            }
636            meta.gradients.insert(name.clone(), Tensor::from_vec(combined, &shape)?);
637        }
638
639        Ok(meta)
640    }
641
642    fn compute_relation(&self, a: &Tensor, b: &Tensor) -> Result<f64, TrustformersError> {
643        // Relation score in (0, 1): a squashed negative squared distance
644        // between two learned embeddings.
645        let left = a.to_vec_f32()?;
646        let right = b.to_vec_f32()?;
647        if left.len() != right.len() {
648            return Err(invalid_input(
649                "relation scores need two embeddings of the same width",
650            ));
651        }
652        let distance: f32 =
653            left.iter().zip(right.iter()).map(|(x, y)| (x - y) * (x - y)).sum::<f32>();
654        Ok((-(distance as f64)).exp())
655    }
656
657    fn get_learning_rates(&self) -> Result<Vec<f64>, TrustformersError> {
658        Ok(self.learning_rates.clone())
659    }
660
661    fn apply_gradients_with_lr(
662        &mut self,
663        gradients: &ModelGradients,
664        learning_rates: &[f64],
665    ) -> Result<(), TrustformersError> {
666        self.learner.apply_gradients_with_lr(gradients, learning_rates)
667    }
668
669    fn compute_lr_gradients(&self, _loss: f64) -> Result<Vec<f64>, TrustformersError> {
670        // Meta-SGD: dL_query/d(alpha_i) = -g_query_i * g_support_i, the exact
671        // first-order derivative of one inner SGD step with respect to its
672        // per-parameter learning rate.
673        let query = self.cached_batch()?;
674        let Some(support) = self.previous_batch.as_ref() else {
675            return Err(invalid_input(
676                "learning-rate gradients need both the support and the query forward pass",
677            ));
678        };
679
680        let query_gradients = self.learner.gradients(query)?;
681        let support_gradients = self.learner.gradients(support)?;
682
683        let mut output = Vec::with_capacity(self.learning_rates.len());
684        for index in 0..(self.learner.layers.len()) {
685            for suffix in ["weight", "bias"] {
686                let name = format!("layer{}.{}", index, suffix);
687                let (Some(q), Some(s)) = (
688                    query_gradients.gradients.get(&name),
689                    support_gradients.gradients.get(&name),
690                ) else {
691                    continue;
692                };
693                let q = q.to_vec_f32()?;
694                let s = s.to_vec_f32()?;
695                for (a, b) in q.iter().zip(s.iter()) {
696                    output.push(-(*a as f64) * (*b as f64));
697                }
698            }
699        }
700
701        Ok(output)
702    }
703}
704
705/// Declare a concrete model that delegates every trait method to [`MetaMlp`].
706macro_rules! meta_model {
707    ($(#[$meta:meta])* $name:ident $(, $extra:item)* $(,)?) => {
708        $(#[$meta])*
709        #[derive(Debug, Clone)]
710        pub struct $name {
711            inner: MetaMlp,
712        }
713
714        impl $name {
715            /// Build the model from a meta-learning configuration.
716            pub fn new(config: &MetaLearningConfig) -> Result<Self, TrustformersError> {
717                Ok(Self {
718                    inner: MetaMlp::new(config)?,
719                })
720            }
721
722            /// Immutable access to the underlying learner.
723            pub fn learner(&self) -> &MlpLearner {
724                &self.inner.learner
725            }
726        }
727
728        impl MetaLearningModel for $name {
729            fn forward(&mut self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
730                self.inner.forward(examples)
731            }
732
733            fn compute_accuracy(&self, examples: &ExampleSet) -> Result<f64, TrustformersError> {
734                self.inner.compute_accuracy(examples)
735            }
736
737            fn compute_gradients(&self, loss: f64) -> Result<ModelGradients, TrustformersError> {
738                self.inner.compute_gradients(loss)
739            }
740
741            fn apply_gradients(
742                &mut self,
743                gradients: &ModelGradients,
744                lr: f64,
745            ) -> Result<(), TrustformersError> {
746                self.inner.apply_gradients(gradients, lr)
747            }
748
749            fn get_parameters(&self) -> Result<ModelParameters, TrustformersError> {
750                self.inner.get_parameters()
751            }
752
753            fn set_parameters(
754                &mut self,
755                params: ModelParameters,
756            ) -> Result<(), TrustformersError> {
757                self.inner.set_parameters(params)
758            }
759
760            fn embed(&self, example: &Example) -> Result<Tensor, TrustformersError> {
761                self.inner.embed(example)
762            }
763
764            $($extra)*
765        }
766    };
767}
768
769meta_model!(
770    /// Model-Agnostic Meta-Learning with real inner and outer loops.
771    ///
772    /// `compute_first_order_gradients` implements FOMAML;
773    /// `compute_second_order_gradients` implements the one-step MAML
774    /// meta-gradient `v - α·Hv`, with the Hessian-vector product estimated by
775    /// a central difference of the support-set gradient.
776    MAMLModel,
777    fn compute_first_order_gradients(&self, loss: f64) -> Result<ModelGradients, TrustformersError> {
778        self.inner.compute_first_order_gradients(loss)
779    },
780    fn compute_second_order_gradients(
781        &self,
782        initial_params: &ModelParameters,
783        loss: f64,
784    ) -> Result<ModelGradients, TrustformersError> {
785        self.inner.compute_second_order_gradients(initial_params, loss)
786    },
787);
788
789meta_model!(
790    /// Reptile: the outer update is the difference between the adapted and the
791    /// initial parameters, which the meta-learner computes directly.
792    ReptileModel,
793);
794
795meta_model!(
796    /// Prototypical Networks: the learner supplies real embeddings, the
797    /// meta-learner forms class prototypes and classifies by distance.
798    PrototypicalModel,
799);
800
801meta_model!(
802    /// Matching Networks: cosine attention over the support embeddings.
803    MatchingNetModel,
804);
805
806meta_model!(
807    /// Relation Networks: a learned embedding plus a relation score.
808    RelationNetModel,
809    fn compute_relation(&self, a: &Tensor, b: &Tensor) -> Result<f64, TrustformersError> {
810        self.inner.compute_relation(a, b)
811    },
812);
813
814meta_model!(
815    /// Memory-Augmented Neural Network.
816    ///
817    /// The learner is real, but the external memory (`write_to_memory`,
818    /// `read_from_memory`, `predict_from_memory`) is not implemented and the
819    /// default trait methods report that instead of returning invented
820    /// predictions.
821    MemoryAugmentedModel,
822);
823
824meta_model!(
825    /// Gradient-based meta-learning with a learned update rule.
826    ///
827    /// The learner is real; the learned optimizer (`get_meta_learner_state`,
828    /// `apply_learned_algorithm`) is not implemented and reports that.
829    GradientBasedModel,
830);
831
832meta_model!(
833    /// Meta-SGD: MAML with per-parameter learned learning rates.
834    MetaSGDModel,
835    fn compute_second_order_gradients(
836        &self,
837        initial_params: &ModelParameters,
838        loss: f64,
839    ) -> Result<ModelGradients, TrustformersError> {
840        self.inner.compute_second_order_gradients(initial_params, loss)
841    },
842    fn get_learning_rates(&self) -> Result<Vec<f64>, TrustformersError> {
843        self.inner.get_learning_rates()
844    },
845    fn apply_gradients_with_lr(
846        &mut self,
847        gradients: &ModelGradients,
848        learning_rates: &[f64],
849    ) -> Result<(), TrustformersError> {
850        self.inner.apply_gradients_with_lr(gradients, learning_rates)
851    },
852    fn compute_lr_gradients(&self, loss: f64) -> Result<Vec<f64>, TrustformersError> {
853        self.inner.compute_lr_gradients(loss)
854    },
855);
856
857meta_model!(
858    /// Learning to Learn by Gradient Descent by Gradient Descent.
859    ///
860    /// The learner is real; the LSTM meta-optimizer (`get_lstm_state`,
861    /// `lstm_update`) is not implemented and reports that.
862    L2LModel,
863);