1use crate::tensor::Tensor;
4use crate::layers::{Linear, LayerNorm, Dropout};
5use mathverse_core::error::MathResult;
6
7#[derive(Clone, Copy)]
9pub enum Activation {
10 ReLU,
12 Sigmoid,
14 Tanh,
16 GELU,
18 Swish,
20 Softmax {
22 axis: usize,
24 },
25 None,
27}
28
29pub enum Layer {
31 Linear(Linear),
33 LayerNorm(LayerNorm),
35 Dropout(Dropout),
37 Activation(Activation),
39}
40
41pub struct Sequential {
43 pub layers: Vec<Layer>,
45}
46
47impl Default for Sequential {
48 fn default() -> Self {
49 Self::new()
50 }
51}
52
53impl Sequential {
54 pub fn new() -> Self { Self { layers: Vec::new() } }
56
57 pub fn add_linear(mut self, in_f: usize, out_f: usize) -> Self {
59 self.layers.push(Layer::Linear(Linear::new(in_f, out_f)));
60 self
61 }
62
63 pub fn add_layer_norm(mut self, normalized_shape: usize) -> Self {
65 self.layers.push(Layer::LayerNorm(LayerNorm::new(normalized_shape, 1e-5)));
66 self
67 }
68
69 pub fn add_dropout(mut self, p: f64) -> Self {
71 self.layers.push(Layer::Dropout(Dropout::new(p)));
72 self
73 }
74
75 pub fn add_activation(mut self, act: Activation) -> Self {
77 self.layers.push(Layer::Activation(act));
78 self
79 }
80
81 pub fn forward(&mut self, x: &Tensor, training: bool) -> MathResult<Tensor> {
83 let mut out = x.clone();
84 for layer in &mut self.layers {
85 match layer {
86 Layer::Linear(l) => out = l.forward(&out)?,
87 Layer::LayerNorm(l) => out = l.forward(&out)?,
88 Layer::Dropout(d) => out = d.forward(&out, training),
89 Layer::Activation(act) => {
90 out = match act {
91 Activation::ReLU => crate::activations::relu(&out),
92 Activation::Sigmoid => crate::activations::sigmoid(&out),
93 Activation::Tanh => crate::activations::tanh(&out),
94 Activation::GELU => crate::activations::gelu(&out),
95 Activation::Swish => crate::activations::swish(&out),
96 Activation::Softmax { axis } => crate::activations::softmax(&out, *axis)?,
97 Activation::None => out,
98 };
99 }
100 }
101 }
102 Ok(out)
103 }
104
105 pub fn num_params(&self) -> usize {
107 self.layers.iter().map(|l| match l {
108 Layer::Linear(l) => l.num_params(),
109 Layer::LayerNorm(l) => l.normalized_shape * 2,
110 _ => 0,
111 }).sum()
112 }
113}
114
115pub struct MLP {
117 pub hidden_layers: Vec<Linear>,
119 pub output_layer: Linear,
121 pub activation: Activation,
123}
124
125impl MLP {
126 pub fn new(input_size: usize, hidden_sizes: &[usize], output_size: usize) -> Self {
128 let mut hidden_layers = Vec::new();
129 let mut prev = input_size;
130 for &h in hidden_sizes {
131 hidden_layers.push(Linear::new(prev, h));
132 prev = h;
133 }
134 Self { hidden_layers, output_layer: Linear::new(prev, output_size), activation: Activation::ReLU }
135 }
136
137 pub fn forward(&self, x: &Tensor) -> MathResult<Tensor> {
139 let mut out = x.clone();
140 for layer in &self.hidden_layers {
141 out = layer.forward(&out)?;
142 out = match self.activation {
143 Activation::ReLU => crate::activations::relu(&out),
144 Activation::Sigmoid => crate::activations::sigmoid(&out),
145 Activation::Tanh => crate::activations::tanh(&out),
146 _ => out,
147 };
148 }
149 self.output_layer.forward(&out)
150 }
151}
152
153pub struct TransformerBlock {
155 pub attn_wq: Tensor,
157 pub attn_wk: Tensor,
159 pub attn_wv: Tensor,
161 pub attn_wo: Tensor,
163 pub ff_w1: Linear,
165 pub ff_w2: Linear,
167 pub ln1: LayerNorm,
169 pub ln2: LayerNorm,
171 pub d_model: usize,
173 pub num_heads: usize,
175}
176
177impl TransformerBlock {
178 pub fn new(d_model: usize, num_heads: usize, d_ff: usize) -> Self {
180 assert!(d_model % num_heads == 0, "TransformerBlock: d_model ({d_model}) must be divisible by num_heads ({num_heads})");
181 let _d_k = d_model / num_heads;
182 Self {
183 attn_wq: Tensor::randn(&[d_model, d_model]).mul_scalar((2.0 / d_model as f64).sqrt()),
184 attn_wk: Tensor::randn(&[d_model, d_model]).mul_scalar((2.0 / d_model as f64).sqrt()),
185 attn_wv: Tensor::randn(&[d_model, d_model]).mul_scalar((2.0 / d_model as f64).sqrt()),
186 attn_wo: Tensor::randn(&[d_model, d_model]).mul_scalar((2.0 / d_model as f64).sqrt()),
187 ff_w1: Linear::new(d_model, d_ff),
188 ff_w2: Linear::new(d_ff, d_model),
189 ln1: LayerNorm::new(d_model, 1e-5),
190 ln2: LayerNorm::new(d_model, 1e-5),
191 d_model,
192 num_heads,
193 }
194 }
195
196 pub fn forward(&self, x: &Tensor, mask: Option<&Tensor>) -> MathResult<Tensor> {
198 let normed = self.ln1.forward(x)?;
200 let attn_out = crate::attention::multi_head_attention(
201 &normed, &self.attn_wq, &self.attn_wk, &self.attn_wv, &self.attn_wo,
202 self.num_heads, mask,
203 )?;
204 let h = x.add(&attn_out)?;
205
206 let normed2 = self.ln2.forward(&h)?;
208 let ff_out = self.ff_w1.forward(&normed2)?;
209 let ff_out = crate::activations::gelu(&ff_out);
210 let ff_out = self.ff_w2.forward(&ff_out)?;
211 h.add(&ff_out)
212 }
213}
214
215#[cfg(test)]
216mod tests {
217 use super::*;
218
219 #[test]
220 fn sequential_test() {
221 let mut model = Sequential::new()
222 .add_linear(4, 8)
223 .add_activation(Activation::ReLU)
224 .add_linear(8, 2);
225 let x = Tensor::zeros(&[3, 4]);
226 let out = model.forward(&x, false).unwrap();
227 assert_eq!(out.shape, vec![3, 2]);
228 assert_eq!(model.num_params(), 4 * 8 + 8 + 8 * 2 + 2);
229 }
230
231 #[test]
232 fn mlp_test() {
233 let mlp = MLP::new(10, &[32, 16], 3);
234 let x = Tensor::zeros(&[5, 10]);
235 let out = mlp.forward(&x).unwrap();
236 assert_eq!(out.shape, vec![5, 3]);
237 }
238
239 #[test]
240 fn transformer_block_test() {
241 let block = TransformerBlock::new(16, 4, 32);
242 let x = Tensor::randn(&[2, 8, 16]);
243 let out = block.forward(&x, None).unwrap();
244 assert_eq!(out.shape, vec![2, 8, 16]);
245 }
246}
247