Skip to main content

coeus_nn/transformer/
ffn.rs

1// ── Feed-Forward Network sub-layer ──
2
3use crate::dropout::Dropout;
4use crate::linear::Linear;
5use crate::module::{prefixed_parameters, Module, ModuleError};
6use coeus_autograd::Var;
7use coeus_core::{Float, MoiraiBackend};
8use std::marker::PhantomData;
9
10/// Functional (stateless) transformer feed-forward block.
11///
12/// Computes: `Linear1(d_model→d_ff) → GELU → Dropout(p) → Linear2(d_ff→d_model)`.
13pub fn feed_forward<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
14    input: &Var<T, B>,
15    w1: &Var<T, B>,
16    b1: Option<&Var<T, B>>,
17    w2: &Var<T, B>,
18    b2: Option<&Var<T, B>>,
19    dropout_p: f64,
20) -> Result<Var<T, B>, ModuleError<B::Error>> {
21    feed_forward_with_training(input, w1, b1, w2, b2, dropout_p, dropout_p > 0.0)
22}
23
24pub(super) fn feed_forward_with_training<
25    T: Float + coeus_cpu::RealScalar,
26    B: coeus_ops::BackendOps<T> + Default,
27>(
28    input: &Var<T, B>,
29    w1: &Var<T, B>,
30    b1: Option<&Var<T, B>>,
31    w2: &Var<T, B>,
32    b2: Option<&Var<T, B>>,
33    dropout_p: f64,
34    is_training: bool,
35) -> Result<Var<T, B>, ModuleError<B::Error>> {
36    let shape = input.tensor.shape();
37    if shape.len() < 2 {
38        return Err(ModuleError::InvalidRank {
39            module: "FeedForward",
40            expected: "at least 2",
41            actual: shape.len(),
42        });
43    }
44    let d_model = *shape
45        .last()
46        .expect("invariant: feed-forward rank was validated as at least two");
47    super::validation::feed_forward("FeedForward", d_model, w1, b1, w2, b2)?;
48
49    let x = linear_from_parts(input, w1, b1)?;
50    let x = coeus_autograd::gelu(&x)?;
51    let mut dropout = Dropout::new(dropout_p);
52    dropout.set_training(is_training);
53    let x = dropout.forward(&x)?;
54    Ok(linear_from_parts(&x, w2, b2)?)
55}
56
57fn linear_from_parts<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
58    input: &Var<T, B>,
59    weight: &Var<T, B>,
60    bias: Option<&Var<T, B>>,
61) -> Result<Var<T, B>, B::Error> {
62    let input_shape = input.tensor.shape();
63    let in_features = weight.tensor.shape()[1];
64    let out_features = weight.tensor.shape()[0];
65    let rows = input_shape[..input_shape.len() - 1]
66        .iter()
67        .copied()
68        .product::<usize>();
69    let flattened = coeus_autograd::reshape(input, [rows, in_features])?;
70    let weight_transposed = coeus_autograd::transpose_2d(weight)?;
71    let projected = coeus_autograd::matmul(&flattened, &weight_transposed)?;
72    let projected = match bias {
73        Some(bias) => coeus_autograd::add(&projected, bias)?,
74        None => projected,
75    };
76
77    let mut output_shape = input_shape.to_vec();
78    *output_shape
79        .last_mut()
80        .expect("invariant: feed-forward rank was validated as at least two") = out_features;
81    coeus_autograd::reshape(&projected, output_shape)
82}
83
84/// Two-layer feed-forward sub-layer.
85///
86/// Computes: `Linear(d_model → d_ff) → GELU → Dropout → Linear(d_ff → d_model)`.
87///
88/// Accepts inputs of any rank ≥ 2 (`[..., d_model]`), including the standard
89/// transformer shape `[batch, seq, d_model]`, via the batched matmul support
90/// in `coeus_autograd::matmul`.
91#[derive(Clone)]
92pub struct FeedForward<T: coeus_core::Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend>
93{
94    /// First linear projection: `d_model → d_ff`.
95    pub linear1: Linear<T, B>,
96    /// Second linear projection: `d_ff → d_model`.
97    pub linear2: Linear<T, B>,
98    /// Dropout between the two projections.
99    pub dropout: Dropout,
100    _marker: PhantomData<(T, B)>,
101}
102
103impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> FeedForward<T, B> {
104    /// Create a FeedForward sub-layer.
105    ///
106    /// # Arguments
107    /// - `d_model`: input and output feature dimension
108    /// - `d_ff`:    hidden (inner) feature dimension
109    /// - `dropout_p`: dropout probability
110    ///
111    /// # Errors
112    ///
113    /// Returns [`crate::init::InitializationError`] when a size is zero or the
114    /// backend's draw fails.
115    pub fn new(
116        d_model: usize,
117        d_ff: usize,
118        dropout_p: f64,
119    ) -> Result<Self, crate::init::InitializationError<B::Error>>
120    where
121        T: coeus_cpu::RealScalar,
122        B: coeus_ops::RandomInitOps<T>,
123    {
124        Ok(Self {
125            linear1: Linear::new(d_model, d_ff, true)?,
126            linear2: Linear::new(d_ff, d_model, true)?,
127            dropout: Dropout::new(dropout_p),
128            _marker: PhantomData,
129        })
130    }
131}
132
133impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
134    for FeedForward<T, B>
135{
136    fn parameters(&self) -> Vec<Var<T, B>> {
137        let mut p = self.linear1.parameters();
138        p.extend(self.linear2.parameters());
139        p
140    }
141
142    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
143        let mut parameters = prefixed_parameters("input", &self.linear1);
144        parameters.extend(prefixed_parameters("output", &self.linear2));
145        parameters
146    }
147
148    /// Forward pass: `Linear1 → GELU → Dropout → Linear2`.
149    ///
150    /// Works for any input rank (`[batch, seq, d_model]`, `[batch, d_model]`, etc.)
151    /// because `Linear::forward` dispatches to the batched matmul kernel.
152    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
153        feed_forward(
154            input,
155            &self.linear1.weight,
156            self.linear1.bias.as_ref(),
157            &self.linear2.weight,
158            self.linear2.bias.as_ref(),
159            self.dropout.p,
160        )
161    }
162}