use burn::{
Tensor,
tensor::FloatDType,
};
use super::{
Forward1Module,
InputDTypes,
ResultDType,
};
use crate::burner::tensor::Widening;
#[derive(Debug)]
pub struct Gate<'a, A, B> {
a: &'a A,
b: &'a B,
widening: Widening,
}
impl<A, B> Clone for Gate<'_, A, B> {
fn clone(&self) -> Self {
*self
}
}
impl<A, B> Copy for Gate<'_, A, B> {}
impl<'a, A, B> Gate<'a, A, B> {
pub fn new(
a: &'a A,
b: &'a B,
) -> Self {
Self {
a,
b,
widening: Widening::NONE,
}
}
pub fn with_widening(
self,
widening: Widening,
) -> Self {
Self { widening, ..self }
}
pub fn a(&self) -> &'a A {
self.a
}
pub fn b(&self) -> &'a B {
self.b
}
pub fn widening(&self) -> Widening {
self.widening
}
pub fn forward<const R: usize>(
&self,
x: Tensor<R>,
gate: Tensor<R>,
) -> Tensor<R>
where
A: Forward1Module<Tensor<R>, Output = Tensor<R>>,
B: Forward1Module<Tensor<R>, Output = Tensor<R>>,
{
let y = self.a.forward(x);
assert_eq!(gate.dims(), y.dims(), "a gate has its gated output's shape");
self.widening
.around(y.dtype(), |w| w.widen(y) * self.b.forward(w.widen(gate)))
}
}
impl<A: InputDTypes, B> InputDTypes for Gate<'_, A, B> {
fn preferred_dtype(&self) -> FloatDType {
self.a.preferred_dtype()
}
fn accepts_dtype(
&self,
dtype: FloatDType,
) -> bool {
self.a.accepts_dtype(dtype)
}
}
impl<A: ResultDType, B> ResultDType for Gate<'_, A, B> {
fn result_dtype(
&self,
input: FloatDType,
) -> FloatDType {
self.a.result_dtype(input)
}
}
#[cfg(test)]
mod tests {
use burn::{
module::Module,
nn::RmsNormConfig,
tensor::{
DType,
Distribution,
Tolerance,
activation::silu,
},
};
use super::*;
use crate::{
blocks::{
activation::{
BunsenActivation,
BunsenActivationConfig,
},
norm::{
BunsenNormalization,
BunsenNormalizationConfig,
},
},
burner::module::DTypeMapper,
support::testing::{
assert_tensors_close,
cpu_device,
seeded_tensor,
},
};
fn parts(device: &burn::prelude::Device) -> (BunsenNormalization, BunsenActivation) {
let norm =
BunsenNormalizationConfig::Rms(RmsNormConfig::new(6).with_epsilon(1e-6)).init(device);
(norm, BunsenActivationConfig::Silu.init(device))
}
#[test]
fn test_gate() {
let device = cpu_device();
let (norm, silu_act) = parts(&device);
let gate = Gate::new(&norm, &silu_act).with_widening(Widening::F32);
assert_eq!(gate.widening(), Widening::F32);
let x = seeded_tensor::<3>(1, [2, 4, 6], Distribution::Default, &device);
let z = seeded_tensor::<3>(2, [2, 4, 6], Distribution::Default, &device);
let expected = norm.forward(x.clone()) * silu(z.clone());
assert_tensors_close(&gate.forward(x, z), &expected, Tolerance::default());
}
#[test]
fn test_widening() {
let device = cpu_device();
let (norm, silu_act) = parts(&device);
let x = seeded_tensor::<2>(3, [3, 6], Distribution::Default, &device).cast(DType::BF16);
let z = seeded_tensor::<2>(4, [3, 6], Distribution::Default, &device).cast(DType::BF16);
let y = norm.forward(x.clone());
let widened = Gate::new(&norm, &silu_act)
.with_widening(Widening::F32)
.forward(x.clone(), z.clone());
let expected =
(y.clone().cast(DType::F32) * silu(z.clone().cast(DType::F32))).cast(DType::BF16);
assert_eq!(widened.dtype(), DType::BF16);
widened.into_data().assert_eq(&expected.into_data(), true);
let narrow = Gate::new(&norm, &silu_act).forward(x, z.clone());
narrow
.into_data()
.assert_eq(&(y * silu(z)).into_data(), true);
}
#[test]
fn test_dtypes() {
let device = cpu_device();
let (norm, silu_act) = parts(&device);
let norm = norm.map(&mut DTypeMapper::new(DType::BF16));
let gate = Gate::new(&norm, &silu_act);
assert_eq!(gate.preferred_dtype(), FloatDType::BF16);
assert!(gate.accepts_dtype(FloatDType::F32));
assert_eq!(gate.result_dtype(FloatDType::F16), FloatDType::F16);
assert!(core::ptr::eq(gate.a(), &norm));
assert!(core::ptr::eq(gate.b(), &silu_act));
}
#[test]
#[should_panic(expected = "a gate has its gated output's shape")]
fn test_gate_shape() {
let device = cpu_device();
let (norm, silu_act) = parts(&device);
let _ = Gate::new(&norm, &silu_act).forward(
Tensor::<2>::zeros([2, 6], &device),
Tensor::zeros([3, 6], &device),
);
}
}