use burn::{
Tensor,
config::Config,
module::Module,
nn::attention::{
MhaOutput,
MultiHeadAttention,
},
prelude::Bool,
tensor::Device,
};
use crate::{
blocks::norm::{
BunsenNormalization,
BunsenNormalizationConfig,
},
ops::transformers::attention::{
KVCache,
cross_attn,
cross_attn_w_kv_cache,
self_attn,
self_attn_w_kv_cache,
},
};
#[derive(Config, Debug)]
pub struct ResidualSublayerConfig {
#[config(default = "None")]
pub input: Option<BunsenNormalizationConfig>,
#[config(default = "None")]
pub pre: Option<BunsenNormalizationConfig>,
#[config(default = "None")]
pub post: Option<BunsenNormalizationConfig>,
#[config(default = "None")]
pub out: Option<BunsenNormalizationConfig>,
}
impl ResidualSublayerConfig {
pub fn pre_norm(pre: Option<BunsenNormalizationConfig>) -> Self {
Self::new().with_pre(pre)
}
pub fn post_norm(out: Option<BunsenNormalizationConfig>) -> Self {
Self::new().with_out(out)
}
pub fn init<M: Module>(
&self,
branch: M,
width: usize,
device: &Device,
) -> ResidualSublayer<M> {
let norm = |c: &Option<BunsenNormalizationConfig>| {
c.as_ref()
.map(|c| c.clone().with_num_features(width).init(device))
};
ResidualSublayer {
input: norm(&self.input),
pre: norm(&self.pre),
branch,
post: norm(&self.post),
out: norm(&self.out),
}
}
}
#[derive(Module, Debug)]
pub struct ResidualSublayer<M: Module> {
pub input: Option<BunsenNormalization>,
pub pre: Option<BunsenNormalization>,
pub branch: M,
pub post: Option<BunsenNormalization>,
pub out: Option<BunsenNormalization>,
}
impl<M: Module> ResidualSublayer<M> {
fn apply<const D: usize>(
norm: &Option<BunsenNormalization>,
x: Tensor<D>,
) -> Tensor<D> {
match norm {
Some(norm) => norm.forward(x),
None => x,
}
}
pub fn branch_input<const D: usize>(
&self,
x: Tensor<D>,
) -> Tensor<D> {
Self::apply(&self.pre, x)
}
pub fn residual<const D: usize>(
&self,
x: Tensor<D>,
f: impl FnOnce(&M, Tensor<D>) -> Tensor<D>,
) -> Tensor<D> {
Self::apply(&self.post, f(&self.branch, self.branch_input(x)))
}
pub fn residual_update_with<const D: usize, O: BranchOutput<D>>(
&self,
x: Tensor<D>,
f: impl FnOnce(&M, Tensor<D>) -> O,
) -> O::Updated {
let x = Self::apply(&self.input, x);
f(&self.branch, self.branch_input(x.clone()))
.into_updated(|residual| Self::apply(&self.out, x + Self::apply(&self.post, residual)))
}
pub fn residual_update<const D: usize, C>(
&self,
x: Tensor<D>,
ctx: C,
) -> <M::Output as BranchOutput<D>>::Updated
where
M: SublayerBranch<C, D>,
{
self.residual_update_with(x, |branch, z| branch.call_branch(z, ctx))
}
pub fn residual_update_with_ctx<const D: usize, C, O: BranchOutput<D>>(
&self,
x: Tensor<D>,
ctx: C,
f: impl FnOnce(&M, Tensor<D>, C) -> O,
) -> O::Updated {
self.residual_update_with(x, |branch, z| f(branch, z, ctx))
}
}
pub trait BranchOutput<const D: usize> {
type Updated;
fn into_updated(
self,
update: impl FnOnce(Tensor<D>) -> Tensor<D>,
) -> Self::Updated;
}
impl<const D: usize> BranchOutput<D> for Tensor<D> {
type Updated = Tensor<D>;
fn into_updated(
self,
update: impl FnOnce(Tensor<D>) -> Tensor<D>,
) -> Tensor<D> {
update(self)
}
}
impl BranchOutput<3> for MhaOutput {
type Updated = (Tensor<3>, Tensor<4>);
fn into_updated(
self,
update: impl FnOnce(Tensor<3>) -> Tensor<3>,
) -> (Tensor<3>, Tensor<4>) {
(update(self.context), self.weights)
}
}
pub trait SublayerBranch<C, const D: usize> {
type Output: BranchOutput<D>;
fn call_branch(
&self,
z: Tensor<D>,
ctx: C,
) -> Self::Output;
}
#[derive(Debug, Clone)]
pub struct SelfAttnCall {
pub mask: Option<Tensor<3, Bool>>,
}
impl SublayerBranch<SelfAttnCall, 3> for MultiHeadAttention {
type Output = Tensor<3>;
fn call_branch(
&self,
z: Tensor<3>,
ctx: SelfAttnCall,
) -> Tensor<3> {
self_attn(self, z, ctx.mask).context
}
}
#[derive(Debug, Clone)]
pub struct CrossAttnCall {
pub xa: Tensor<3>,
}
impl SublayerBranch<CrossAttnCall, 3> for MultiHeadAttention {
type Output = MhaOutput;
fn call_branch(
&self,
z: Tensor<3>,
ctx: CrossAttnCall,
) -> MhaOutput {
cross_attn(self, z, ctx.xa)
}
}
#[derive(Debug)]
pub struct SelfAttnKvCall<'a> {
pub mask: Option<Tensor<3, Bool>>,
pub cache: &'a mut KVCache,
pub layer: usize,
}
impl SublayerBranch<SelfAttnKvCall<'_>, 3> for MultiHeadAttention {
type Output = Tensor<3>;
fn call_branch(
&self,
z: Tensor<3>,
ctx: SelfAttnKvCall<'_>,
) -> Tensor<3> {
self_attn_w_kv_cache(self, z, ctx.mask, ctx.cache, ctx.layer)
}
}
#[derive(Debug, Clone, Copy)]
pub struct CrossAttnKvCall<'a> {
pub cache: &'a KVCache,
pub layer: usize,
}
impl SublayerBranch<CrossAttnKvCall<'_>, 3> for MultiHeadAttention {
type Output = Tensor<3>;
fn call_branch(
&self,
z: Tensor<3>,
ctx: CrossAttnKvCall<'_>,
) -> Tensor<3> {
cross_attn_w_kv_cache(self, z, ctx.cache, ctx.layer)
}
}
pub trait PlainSublayer<const D: usize> {
fn forward(
&self,
x: Tensor<D>,
) -> Tensor<D>;
}
impl<const D: usize, M> PlainSublayer<D> for ResidualSublayer<M>
where
M: Module + SublayerBranch<(), D, Output = Tensor<D>>,
{
fn forward(
&self,
x: Tensor<D>,
) -> Tensor<D> {
self.residual_update(x, ())
}
}
pub trait SelfAttnSublayer {
fn self_attn(
&self,
x: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
) -> Tensor<3>;
}
impl<M> SelfAttnSublayer for ResidualSublayer<M>
where
M: Module + SublayerBranch<SelfAttnCall, 3, Output = Tensor<3>>,
{
fn self_attn(
&self,
x: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
) -> Tensor<3> {
self.residual_update(x, SelfAttnCall { mask })
}
}
pub trait CrossAttnSublayer {
fn cross_attn(
&self,
x: Tensor<3>,
xa: Tensor<3>,
) -> (Tensor<3>, Tensor<4>);
}
impl<M> CrossAttnSublayer for ResidualSublayer<M>
where
M: Module + SublayerBranch<CrossAttnCall, 3, Output = MhaOutput>,
{
fn cross_attn(
&self,
x: Tensor<3>,
xa: Tensor<3>,
) -> (Tensor<3>, Tensor<4>) {
self.residual_update(x, CrossAttnCall { xa })
}
}
pub trait SelfAttnKvSublayer {
fn self_attn_w_kv_cache(
&self,
x: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
cache: &mut KVCache,
layer: usize,
) -> Tensor<3>;
}
impl<M> SelfAttnKvSublayer for ResidualSublayer<M>
where
M: Module + for<'a> SublayerBranch<SelfAttnKvCall<'a>, 3, Output = Tensor<3>>,
{
fn self_attn_w_kv_cache(
&self,
x: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
cache: &mut KVCache,
layer: usize,
) -> Tensor<3> {
self.residual_update(x, SelfAttnKvCall { mask, cache, layer })
}
}
pub trait CrossAttnKvSublayer {
fn cross_attn_w_kv_cache(
&self,
x: Tensor<3>,
cache: &KVCache,
layer: usize,
) -> Tensor<3>;
}
impl<M> CrossAttnKvSublayer for ResidualSublayer<M>
where
M: Module + for<'a> SublayerBranch<CrossAttnKvCall<'a>, 3, Output = Tensor<3>>,
{
fn cross_attn_w_kv_cache(
&self,
x: Tensor<3>,
cache: &KVCache,
layer: usize,
) -> Tensor<3> {
self.residual_update(x, CrossAttnKvCall { cache, layer })
}
}
#[cfg(test)]
mod tests {
use burn::{
module::Param,
nn::{
Linear,
LinearConfig,
RmsNormConfig,
},
tensor::Tolerance,
};
use super::*;
use crate::support::testing::{
assert_tensors_close,
cpu_device,
};
fn rms() -> Option<BunsenNormalizationConfig> {
Some(BunsenNormalizationConfig::Rms(RmsNormConfig::new(0)))
}
fn sublayer(config: ResidualSublayerConfig) -> ResidualSublayer<Linear> {
let device = cpu_device();
let mut s = config.init(LinearConfig::new(3, 3).init(&device), 3, &device);
for (g, norm) in [
(1.5, &mut s.input),
(0.5, &mut s.pre),
(2.0, &mut s.post),
(3.0, &mut s.out),
] {
if let Some(BunsenNormalization::Rms(r)) = norm {
r.gamma = Param::from_tensor(Tensor::from_floats([g, g, g], &device));
}
}
s
}
fn x() -> Tensor<2> {
Tensor::from_floats([[1.0, -2.0, 4.0], [0.5, 0.25, -1.0]], &cpu_device())
}
fn n(
norm: &Option<BunsenNormalization>,
x: Tensor<2>,
) -> Tensor<2> {
norm.as_ref().unwrap().forward(x)
}
#[test]
fn test_slots() {
let tol = Tolerance::default();
let x = x();
let f = |l: &Linear, z: Tensor<2>| l.forward(z);
let s = sublayer(ResidualSublayerConfig::new());
assert_tensors_close(
&s.residual_update_with(x.clone(), f),
&(x.clone() + s.branch.forward(x.clone())),
tol,
);
#[cfg(feature = "store")]
{
use burn::store::ModuleSnapshot;
let names: Vec<String> = s
.collect(None, None, false)
.into_iter()
.map(|t| t.name)
.collect();
assert_eq!(names, ["branch.weight", "branch.bias"]);
}
let s = sublayer(ResidualSublayerConfig::pre_norm(rms()).with_post(rms()));
assert_tensors_close(
&s.residual_update_with(x.clone(), f),
&(x.clone() + n(&s.post, s.branch.forward(n(&s.pre, x.clone())))),
tol,
);
let s = sublayer(ResidualSublayerConfig::post_norm(rms()));
assert_tensors_close(
&s.residual_update_with(x.clone(), f),
&n(&s.out, x.clone() + s.branch.forward(x.clone())),
tol,
);
let s = sublayer(
ResidualSublayerConfig::new()
.with_input(rms())
.with_pre(rms())
.with_post(rms())
.with_out(rms()),
);
let xi = n(&s.input, x.clone());
assert_tensors_close(
&s.residual_update_with(x.clone(), f),
&n(
&s.out,
xi.clone() + n(&s.post, s.branch.forward(n(&s.pre, xi.clone()))),
),
tol,
);
}
struct WithAux {
residual: Tensor<2>,
aux: f32,
}
impl BranchOutput<2> for WithAux {
type Updated = (Tensor<2>, f32);
fn into_updated(
self,
update: impl FnOnce(Tensor<2>) -> Tensor<2>,
) -> (Tensor<2>, f32) {
(update(self.residual), self.aux)
}
}
#[test]
fn test_branch_outputs() {
let tol = Tolerance::default();
let x = x();
let s = sublayer(
ResidualSublayerConfig::new()
.with_input(rms())
.with_pre(rms())
.with_post(rms())
.with_out(rms()),
);
let plain = s.residual_update_with(x.clone(), |l, z| l.forward(z));
let (stream, aux) = s.residual_update_with(x.clone(), |l, z| WithAux {
residual: l.forward(z),
aux: 7.0,
});
assert_tensors_close(&stream, &plain, tol);
assert_eq!(aux, 7.0);
let scaled =
s.residual_update_with_ctx(x.clone(), 2.0, |l, z, k| l.forward(z).mul_scalar(k));
let expected = s.residual_update_with(x, |l, z| l.forward(z).mul_scalar(2.0));
assert_tensors_close(&scaled, &expected, tol);
}
impl SublayerBranch<(), 2> for Linear {
type Output = Tensor<2>;
fn call_branch(
&self,
z: Tensor<2>,
(): (),
) -> Tensor<2> {
self.forward(z)
}
}
struct Scaled(f32);
impl SublayerBranch<Scaled, 2> for Linear {
type Output = WithAux;
fn call_branch(
&self,
z: Tensor<2>,
ctx: Scaled,
) -> WithAux {
WithAux {
residual: self.forward(z).mul_scalar(ctx.0),
aux: ctx.0,
}
}
}
#[test]
fn test_modes() {
let tol = Tolerance::default();
let x = x();
let s = sublayer(ResidualSublayerConfig::pre_norm(rms()).with_post(rms()));
assert_tensors_close(
&s.residual_update(x.clone(), ()),
&s.residual_update_with(x.clone(), |l, z| l.forward(z)),
tol,
);
let (stream, k) = s.residual_update(x.clone(), Scaled(3.0));
assert_tensors_close(
&stream,
&s.residual_update_with(x, |l, z| l.forward(z).mul_scalar(3.0)),
tol,
);
assert_eq!(k, 3.0);
}
#[test]
fn test_plain_sublayer() {
let x = x();
let s = sublayer(ResidualSublayerConfig::pre_norm(rms()).with_post(rms()));
assert_tensors_close(
&PlainSublayer::forward(&s, x.clone()),
&s.residual_update(x, ()),
Tolerance::default(),
);
}
}