Skip to main content

coeus_nn/
swiglu.rs

1//! SwiGLU gated feed-forward unit.
2//!
3//! `SwiGLU(x) = silu(W_inner · x) * (W_outer · x)` — the Swish/SiLU-gated
4//! linear unit used in modern transformer feed-forward blocks (PaLM, LLaMA).
5//! Two parallel linear projections of the same input share the input tensor;
6//! the inner projection is SiLU-gated and multiplied element-wise by the outer
7//! projection.
8
9use crate::activation::silu;
10use crate::linear::Linear;
11use crate::module::{prefixed_parameters, Module, ModuleError};
12use coeus_autograd::Var;
13use coeus_core::{Float, MoiraiBackend};
14
15/// SwiGLU gated linear unit projecting `d_input -> d_output`.
16///
17/// Composed of two [`Linear`] layers over the same input: the inner projection
18/// is SiLU-gated, the outer is the gate's value path, combined by an
19/// element-wise product. Generic over any [`Float`] scalar and backend.
20#[derive(Clone)]
21pub struct SwiGlu<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
22    /// Inner (SiLU-gated) projection, weight `[d_output × d_input]`.
23    pub linear_inner: Linear<T, B>,
24    /// Outer (value) projection, weight `[d_output × d_input]`.
25    pub linear_outer: Linear<T, B>,
26}
27
28impl<T: Float, B: coeus_ops::BackendOps<T> + Default> SwiGlu<T, B> {
29    /// Create a SwiGLU unit projecting `d_input -> d_output`, with optional bias
30    /// on both linear layers.
31    ///
32    /// # Errors
33    ///
34    /// Returns [`crate::init::InitializationError`] when a size is zero or the
35    /// backend's draw fails.
36    pub fn new(
37        d_input: usize,
38        d_output: usize,
39        bias: bool,
40    ) -> Result<Self, crate::init::InitializationError<B::Error>>
41    where
42        T: coeus_cpu::RealScalar,
43        B: coeus_ops::RandomInitOps<T>,
44    {
45        Ok(Self {
46            linear_inner: Linear::new(d_input, d_output, bias)?,
47            linear_outer: Linear::new(d_input, d_output, bias)?,
48        })
49    }
50}
51
52impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
53    for SwiGlu<T, B>
54{
55    fn parameters(&self) -> Vec<Var<T, B>> {
56        let mut params = self.linear_inner.parameters();
57        params.extend(self.linear_outer.parameters());
58        params
59    }
60
61    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
62        let mut parameters = prefixed_parameters("inner", &self.linear_inner);
63        parameters.extend(prefixed_parameters("outer", &self.linear_outer));
64        parameters
65    }
66
67    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
68        let gated = silu(&self.linear_inner.forward(input)?)?;
69        let outer = self.linear_outer.forward(input)?;
70        Ok(coeus_autograd::mul(&gated, &outer)?)
71    }
72}