1use crate::dropout::Dropout;
4use crate::linear::Linear;
5use crate::module::{prefixed_parameters, Module, ModuleError};
6use coeus_autograd::Var;
7use coeus_core::{Float, MoiraiBackend};
8use std::marker::PhantomData;
9
10pub fn feed_forward<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
14 input: &Var<T, B>,
15 w1: &Var<T, B>,
16 b1: Option<&Var<T, B>>,
17 w2: &Var<T, B>,
18 b2: Option<&Var<T, B>>,
19 dropout_p: f64,
20) -> Result<Var<T, B>, ModuleError<B::Error>> {
21 feed_forward_with_training(input, w1, b1, w2, b2, dropout_p, dropout_p > 0.0)
22}
23
24pub(super) fn feed_forward_with_training<
25 T: Float + coeus_cpu::RealScalar,
26 B: coeus_ops::BackendOps<T> + Default,
27>(
28 input: &Var<T, B>,
29 w1: &Var<T, B>,
30 b1: Option<&Var<T, B>>,
31 w2: &Var<T, B>,
32 b2: Option<&Var<T, B>>,
33 dropout_p: f64,
34 is_training: bool,
35) -> Result<Var<T, B>, ModuleError<B::Error>> {
36 let shape = input.tensor.shape();
37 if shape.len() < 2 {
38 return Err(ModuleError::InvalidRank {
39 module: "FeedForward",
40 expected: "at least 2",
41 actual: shape.len(),
42 });
43 }
44 let d_model = *shape
45 .last()
46 .expect("invariant: feed-forward rank was validated as at least two");
47 super::validation::feed_forward("FeedForward", d_model, w1, b1, w2, b2)?;
48
49 let x = linear_from_parts(input, w1, b1)?;
50 let x = coeus_autograd::gelu(&x)?;
51 let mut dropout = Dropout::new(dropout_p);
52 dropout.set_training(is_training);
53 let x = dropout.forward(&x)?;
54 Ok(linear_from_parts(&x, w2, b2)?)
55}
56
57fn linear_from_parts<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
58 input: &Var<T, B>,
59 weight: &Var<T, B>,
60 bias: Option<&Var<T, B>>,
61) -> Result<Var<T, B>, B::Error> {
62 let input_shape = input.tensor.shape();
63 let in_features = weight.tensor.shape()[1];
64 let out_features = weight.tensor.shape()[0];
65 let rows = input_shape[..input_shape.len() - 1]
66 .iter()
67 .copied()
68 .product::<usize>();
69 let flattened = coeus_autograd::reshape(input, [rows, in_features])?;
70 let weight_transposed = coeus_autograd::transpose_2d(weight)?;
71 let projected = coeus_autograd::matmul(&flattened, &weight_transposed)?;
72 let projected = match bias {
73 Some(bias) => coeus_autograd::add(&projected, bias)?,
74 None => projected,
75 };
76
77 let mut output_shape = input_shape.to_vec();
78 *output_shape
79 .last_mut()
80 .expect("invariant: feed-forward rank was validated as at least two") = out_features;
81 coeus_autograd::reshape(&projected, output_shape)
82}
83
84#[derive(Clone)]
92pub struct FeedForward<T: coeus_core::Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend>
93{
94 pub linear1: Linear<T, B>,
96 pub linear2: Linear<T, B>,
98 pub dropout: Dropout,
100 _marker: PhantomData<(T, B)>,
101}
102
103impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> FeedForward<T, B> {
104 pub fn new(
116 d_model: usize,
117 d_ff: usize,
118 dropout_p: f64,
119 ) -> Result<Self, crate::init::InitializationError<B::Error>>
120 where
121 T: coeus_cpu::RealScalar,
122 B: coeus_ops::RandomInitOps<T>,
123 {
124 Ok(Self {
125 linear1: Linear::new(d_model, d_ff, true)?,
126 linear2: Linear::new(d_ff, d_model, true)?,
127 dropout: Dropout::new(dropout_p),
128 _marker: PhantomData,
129 })
130 }
131}
132
133impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
134 for FeedForward<T, B>
135{
136 fn parameters(&self) -> Vec<Var<T, B>> {
137 let mut p = self.linear1.parameters();
138 p.extend(self.linear2.parameters());
139 p
140 }
141
142 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
143 let mut parameters = prefixed_parameters("input", &self.linear1);
144 parameters.extend(prefixed_parameters("output", &self.linear2));
145 parameters
146 }
147
148 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
153 feed_forward(
154 input,
155 &self.linear1.weight,
156 self.linear1.bias.as_ref(),
157 &self.linear2.weight,
158 self.linear2.bias.as_ref(),
159 self.dropout.p,
160 )
161 }
162}