use crate::activation::silu;
use crate::linear::Linear;
use crate::module::{prefixed_parameters, Module, ModuleError};
use coeus_autograd::Var;
use coeus_core::{Float, MoiraiBackend};
#[derive(Clone)]
pub struct SwiGlu<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
pub linear_inner: Linear<T, B>,
pub linear_outer: Linear<T, B>,
}
impl<T: Float, B: coeus_ops::BackendOps<T> + Default> SwiGlu<T, B> {
pub fn new(
d_input: usize,
d_output: usize,
bias: bool,
) -> Result<Self, crate::init::InitializationError<B::Error>>
where
T: coeus_cpu::RealScalar,
B: coeus_ops::RandomInitOps<T>,
{
Ok(Self {
linear_inner: Linear::new(d_input, d_output, bias)?,
linear_outer: Linear::new(d_input, d_output, bias)?,
})
}
}
impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
for SwiGlu<T, B>
{
fn parameters(&self) -> Vec<Var<T, B>> {
let mut params = self.linear_inner.parameters();
params.extend(self.linear_outer.parameters());
params
}
fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
let mut parameters = prefixed_parameters("inner", &self.linear_inner);
parameters.extend(prefixed_parameters("outer", &self.linear_outer));
parameters
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
let gated = silu(&self.linear_inner.forward(input)?)?;
let outer = self.linear_outer.forward(input)?;
Ok(coeus_autograd::mul(&gated, &outer)?)
}
}