use burn::{
config::Config,
module::Module,
tensor::{
Device,
FloatDType,
},
};
use crate::{
blocks::transformers::{
attention::csa::{
CausalSelfAttention,
CausalSelfAttentionConfig,
CausalSelfAttentionMeta,
},
mixers::{
GatedDeltaNet,
GatedDeltaNetConfig,
GatedDeltaNetMeta,
},
},
burner::module::{
InputDTypes,
ResultDType,
},
errors::BunsenResult,
};
#[derive(Config, Debug)]
pub enum TokenMixerConfig {
Attention(CausalSelfAttentionConfig),
GatedDeltaNet(GatedDeltaNetConfig),
}
impl From<CausalSelfAttentionConfig> for TokenMixerConfig {
fn from(config: CausalSelfAttentionConfig) -> Self {
Self::Attention(config)
}
}
impl From<GatedDeltaNetConfig> for TokenMixerConfig {
fn from(config: GatedDeltaNetConfig) -> Self {
Self::GatedDeltaNet(config)
}
}
impl TokenMixerConfig {
pub fn n_embed(&self) -> usize {
match self {
Self::Attention(c) => c.n_embed(),
Self::GatedDeltaNet(c) => c.n_embed(),
}
}
pub fn as_attention(&self) -> Option<&CausalSelfAttentionConfig> {
match self {
Self::Attention(c) => Some(c),
Self::GatedDeltaNet(_) => None,
}
}
pub fn as_attention_mut(&mut self) -> Option<&mut CausalSelfAttentionConfig> {
match self {
Self::Attention(c) => Some(c),
Self::GatedDeltaNet(_) => None,
}
}
pub fn attention(&self) -> &CausalSelfAttentionConfig {
self.as_attention().expect("a token mixer that attends")
}
pub fn attention_mut(&mut self) -> &mut CausalSelfAttentionConfig {
self.as_attention_mut().expect("a token mixer that attends")
}
pub fn try_init(
&self,
slot: usize,
device: &Device,
) -> BunsenResult<TokenMixer> {
Ok(match self {
Self::Attention(c) => TokenMixer::Attention(c.try_init(slot, device)?),
Self::GatedDeltaNet(c) => TokenMixer::GatedDeltaNet(c.try_init(slot, device)?),
})
}
}
#[derive(Module, Debug)]
#[allow(clippy::large_enum_variant)]
pub enum TokenMixer {
Attention(CausalSelfAttention),
GatedDeltaNet(GatedDeltaNet),
}
impl From<CausalSelfAttention> for TokenMixer {
fn from(attn: CausalSelfAttention) -> Self {
Self::Attention(attn)
}
}
impl From<GatedDeltaNet> for TokenMixer {
fn from(net: GatedDeltaNet) -> Self {
Self::GatedDeltaNet(net)
}
}
impl TokenMixer {
pub fn n_embed(&self) -> usize {
match self {
Self::Attention(m) => m.n_embed(),
Self::GatedDeltaNet(m) => m.n_embed(),
}
}
pub fn as_attention(&self) -> Option<&CausalSelfAttention> {
match self {
Self::Attention(m) => Some(m),
Self::GatedDeltaNet(_) => None,
}
}
pub fn as_attention_mut(&mut self) -> Option<&mut CausalSelfAttention> {
match self {
Self::Attention(m) => Some(m),
Self::GatedDeltaNet(_) => None,
}
}
pub fn attention(&self) -> &CausalSelfAttention {
self.as_attention().expect("a token mixer that attends")
}
pub fn attention_mut(&mut self) -> &mut CausalSelfAttention {
self.as_attention_mut().expect("a token mixer that attends")
}
pub fn as_gated_delta_net(&self) -> Option<&GatedDeltaNet> {
match self {
Self::Attention(_) => None,
Self::GatedDeltaNet(m) => Some(m),
}
}
}
impl InputDTypes for TokenMixer {
fn preferred_dtype(&self) -> FloatDType {
match self {
Self::Attention(m) => m.preferred_dtype(),
Self::GatedDeltaNet(m) => m.preferred_dtype(),
}
}
}
impl ResultDType for TokenMixer {
fn result_dtype(
&self,
input: FloatDType,
) -> FloatDType {
match self {
Self::Attention(m) => m.result_dtype(input),
Self::GatedDeltaNet(m) => m.result_dtype(input),
}
}
}