Skip to main content

mathverse_ai/
models.rs

1//! Sequential model container: chains layers + activations for forward pass.
2
3use crate::tensor::Tensor;
4use crate::layers::{Linear, LayerNorm, Dropout};
5use mathverse_core::error::MathResult;
6
7/// Activation type enum.
8#[derive(Clone, Copy)]
9pub enum Activation {
10    /// Rectified Linear Unit.
11    ReLU,
12    /// Logistic sigmoid.
13    Sigmoid,
14    /// Hyperbolic tangent.
15    Tanh,
16    /// Gaussian Error Linear Unit (tanh approximation).
17    GELU,
18    /// Swish / SiLU.
19    Swish,
20    /// Softmax along a given `axis`.
21    Softmax {
22        /// Axis along which softmax is applied.
23        axis: usize,
24    },
25    /// No activation (identity).
26    None,
27}
28
29/// Layer enum for sequential models.
30pub enum Layer {
31    /// Fully connected linear layer.
32    Linear(Linear),
33    /// Layer normalization.
34    LayerNorm(LayerNorm),
35    /// Dropout layer.
36    Dropout(Dropout),
37    /// Activation function.
38    Activation(Activation),
39}
40
41/// Sequential model: chains layers in order.
42pub struct Sequential {
43    /// Layers executed in order.
44    pub layers: Vec<Layer>,
45}
46
47impl Default for Sequential {
48    fn default() -> Self {
49        Self::new()
50    }
51}
52
53impl Sequential {
54    /// Create an empty sequential model.
55    pub fn new() -> Self { Self { layers: Vec::new() } }
56
57    /// Append a [`Linear`] layer.
58    pub fn add_linear(mut self, in_f: usize, out_f: usize) -> Self {
59        self.layers.push(Layer::Linear(Linear::new(in_f, out_f)));
60        self
61    }
62
63    /// Append a [`LayerNorm`] layer.
64    pub fn add_layer_norm(mut self, normalized_shape: usize) -> Self {
65        self.layers.push(Layer::LayerNorm(LayerNorm::new(normalized_shape, 1e-5)));
66        self
67    }
68
69    /// Append a [`Dropout`] layer.
70    pub fn add_dropout(mut self, p: f64) -> Self {
71        self.layers.push(Layer::Dropout(Dropout::new(p)));
72        self
73    }
74
75    /// Append an activation.
76    pub fn add_activation(mut self, act: Activation) -> Self {
77        self.layers.push(Layer::Activation(act));
78        self
79    }
80
81    /// Forward pass through all layers.
82    pub fn forward(&mut self, x: &Tensor, training: bool) -> MathResult<Tensor> {
83        let mut out = x.clone();
84        for layer in &mut self.layers {
85            match layer {
86                Layer::Linear(l) => out = l.forward(&out)?,
87                Layer::LayerNorm(l) => out = l.forward(&out)?,
88                Layer::Dropout(d) => out = d.forward(&out, training),
89                Layer::Activation(act) => {
90                    out = match act {
91                        Activation::ReLU => crate::activations::relu(&out),
92                        Activation::Sigmoid => crate::activations::sigmoid(&out),
93                        Activation::Tanh => crate::activations::tanh(&out),
94                        Activation::GELU => crate::activations::gelu(&out),
95                        Activation::Swish => crate::activations::swish(&out),
96                        Activation::Softmax { axis } => crate::activations::softmax(&out, *axis)?,
97                        Activation::None => out,
98                    };
99                }
100            }
101        }
102        Ok(out)
103    }
104
105    /// Total number of parameters.
106    pub fn num_params(&self) -> usize {
107        self.layers.iter().map(|l| match l {
108            Layer::Linear(l) => l.num_params(),
109            Layer::LayerNorm(l) => l.normalized_shape * 2,
110            _ => 0,
111        }).sum()
112    }
113}
114
115/// Multi-layer perceptron (MLP) for classification.
116pub struct MLP {
117    /// Hidden linear layers.
118    pub hidden_layers: Vec<Linear>,
119    /// Output linear layer.
120    pub output_layer: Linear,
121    /// Activation used between hidden layers.
122    pub activation: Activation,
123}
124
125impl MLP {
126    /// Create a new MLP.
127    pub fn new(input_size: usize, hidden_sizes: &[usize], output_size: usize) -> Self {
128        let mut hidden_layers = Vec::new();
129        let mut prev = input_size;
130        for &h in hidden_sizes {
131            hidden_layers.push(Linear::new(prev, h));
132            prev = h;
133        }
134        Self { hidden_layers, output_layer: Linear::new(prev, output_size), activation: Activation::ReLU }
135    }
136
137    /// Forward pass through all layers.
138    pub fn forward(&self, x: &Tensor) -> MathResult<Tensor> {
139        let mut out = x.clone();
140        for layer in &self.hidden_layers {
141            out = layer.forward(&out)?;
142            out = match self.activation {
143                Activation::ReLU => crate::activations::relu(&out),
144                Activation::Sigmoid => crate::activations::sigmoid(&out),
145                Activation::Tanh => crate::activations::tanh(&out),
146                _ => out,
147            };
148        }
149        self.output_layer.forward(&out)
150    }
151}
152
153/// Transformer encoder block.
154pub struct TransformerBlock {
155    /// Query projection weight.
156    pub attn_wq: Tensor,
157    /// Key projection weight.
158    pub attn_wk: Tensor,
159    /// Value projection weight.
160    pub attn_wv: Tensor,
161    /// Output projection weight.
162    pub attn_wo: Tensor,
163    /// First feed-forward layer.
164    pub ff_w1: Linear,
165    /// Second feed-forward layer.
166    pub ff_w2: Linear,
167    /// First layer norm (pre-attention).
168    pub ln1: LayerNorm,
169    /// Second layer norm (pre-FFN).
170    pub ln2: LayerNorm,
171    /// Model width.
172    pub d_model: usize,
173    /// Number of attention heads.
174    pub num_heads: usize,
175}
176
177impl TransformerBlock {
178    /// Create a transformer block with random-initialized parameters.
179    pub fn new(d_model: usize, num_heads: usize, d_ff: usize) -> Self {
180        assert!(d_model % num_heads == 0, "TransformerBlock: d_model ({d_model}) must be divisible by num_heads ({num_heads})");
181        let _d_k = d_model / num_heads;
182        Self {
183            attn_wq: Tensor::randn(&[d_model, d_model]).mul_scalar((2.0 / d_model as f64).sqrt()),
184            attn_wk: Tensor::randn(&[d_model, d_model]).mul_scalar((2.0 / d_model as f64).sqrt()),
185            attn_wv: Tensor::randn(&[d_model, d_model]).mul_scalar((2.0 / d_model as f64).sqrt()),
186            attn_wo: Tensor::randn(&[d_model, d_model]).mul_scalar((2.0 / d_model as f64).sqrt()),
187            ff_w1: Linear::new(d_model, d_ff),
188            ff_w2: Linear::new(d_ff, d_model),
189            ln1: LayerNorm::new(d_model, 1e-5),
190            ln2: LayerNorm::new(d_model, 1e-5),
191            d_model,
192            num_heads,
193        }
194    }
195
196    /// Forward pass with optional attention mask.
197    pub fn forward(&self, x: &Tensor, mask: Option<&Tensor>) -> MathResult<Tensor> {
198        // Self-attention with residual
199        let normed = self.ln1.forward(x)?;
200        let attn_out = crate::attention::multi_head_attention(
201            &normed, &self.attn_wq, &self.attn_wk, &self.attn_wv, &self.attn_wo,
202            self.num_heads, mask,
203        )?;
204        let h = x.add(&attn_out)?;
205
206        // Feed-forward with residual
207        let normed2 = self.ln2.forward(&h)?;
208        let ff_out = self.ff_w1.forward(&normed2)?;
209        let ff_out = crate::activations::gelu(&ff_out);
210        let ff_out = self.ff_w2.forward(&ff_out)?;
211        h.add(&ff_out)
212    }
213}
214
215#[cfg(test)]
216mod tests {
217    use super::*;
218
219    #[test]
220    fn sequential_test() {
221        let mut model = Sequential::new()
222            .add_linear(4, 8)
223            .add_activation(Activation::ReLU)
224            .add_linear(8, 2);
225        let x = Tensor::zeros(&[3, 4]);
226        let out = model.forward(&x, false).unwrap();
227        assert_eq!(out.shape, vec![3, 2]);
228        assert_eq!(model.num_params(), 4 * 8 + 8 + 8 * 2 + 2);
229    }
230
231    #[test]
232    fn mlp_test() {
233        let mlp = MLP::new(10, &[32, 16], 3);
234        let x = Tensor::zeros(&[5, 10]);
235        let out = mlp.forward(&x).unwrap();
236        assert_eq!(out.shape, vec![5, 3]);
237    }
238
239    #[test]
240    fn transformer_block_test() {
241        let block = TransformerBlock::new(16, 4, 32);
242        let x = Tensor::randn(&[2, 8, 16]);
243        let out = block.forward(&x, None).unwrap();
244        assert_eq!(out.shape, vec![2, 8, 16]);
245    }
246}
247