1use crate::activation::silu;
10use crate::linear::Linear;
11use crate::module::{prefixed_parameters, Module, ModuleError};
12use coeus_autograd::Var;
13use coeus_core::{Float, MoiraiBackend};
14
15#[derive(Clone)]
21pub struct SwiGlu<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
22 pub linear_inner: Linear<T, B>,
24 pub linear_outer: Linear<T, B>,
26}
27
28impl<T: Float, B: coeus_ops::BackendOps<T> + Default> SwiGlu<T, B> {
29 pub fn new(
37 d_input: usize,
38 d_output: usize,
39 bias: bool,
40 ) -> Result<Self, crate::init::InitializationError<B::Error>>
41 where
42 T: coeus_cpu::RealScalar,
43 B: coeus_ops::RandomInitOps<T>,
44 {
45 Ok(Self {
46 linear_inner: Linear::new(d_input, d_output, bias)?,
47 linear_outer: Linear::new(d_input, d_output, bias)?,
48 })
49 }
50}
51
52impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
53 for SwiGlu<T, B>
54{
55 fn parameters(&self) -> Vec<Var<T, B>> {
56 let mut params = self.linear_inner.parameters();
57 params.extend(self.linear_outer.parameters());
58 params
59 }
60
61 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
62 let mut parameters = prefixed_parameters("inner", &self.linear_inner);
63 parameters.extend(prefixed_parameters("outer", &self.linear_outer));
64 parameters
65 }
66
67 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
68 let gated = silu(&self.linear_inner.forward(input)?)?;
69 let outer = self.linear_outer.forward(input)?;
70 Ok(coeus_autograd::mul(&gated, &outer)?)
71 }
72}