use burn::{
Tensor,
config::Config,
module::Module,
nn::norm::RmsNormConfig,
tensor::Device,
};
use crate::{
blocks::{
norm::BunsenNormalizationConfig,
transformers::{
attention::csa::CausalSelfAttentionConfig,
conditioning::AdaptiveScaleConfig,
decoders::{
causal::CausalDecoderBlockConfig,
stack::{
DecoderStackSides,
UniformStackConfig,
},
},
embedding::{
RotaryDirection,
RotaryEmbeddingConfig,
RotarySet,
TokenEmbeddingConfig,
},
head::LmHeadConfig,
mlp::TransformerGatedMlpConfig,
models::language_decoder::{
LmAttentionOnlyKind,
LmDecoderHolder,
LmDecoderMeta,
LmDecoderModel,
LmDecoderVariantConfig,
},
},
},
burner::module::ToStructureConfig,
errors::BunsenResult,
ops::{
embedding::timestep_embedding,
transformers::attention::{
KVCache,
KVCacheConfig,
KVCacheModel,
},
},
};
pub const VOXTRAL_TEXT_KV_CACHE_PRE_SIZE: usize = 1024;
pub const VOXTRAL_ADA_HIDDEN: usize = 32;
pub const VOXTRAL_TIME_THETA: f32 = 10_000.0;
impl KVCacheModel for VoxtralTextConfig {
fn kv_cache_config(&self) -> KVCacheConfig {
KVCacheConfig::for_decode(
self.n_kv_heads(),
VOXTRAL_TEXT_KV_CACHE_PRE_SIZE,
self.head_dim(),
self.n_layers(),
)
}
}
#[derive(Config, Debug)]
pub struct VoxtralTextConfig {
pub n_vocab: usize,
pub d_model: usize,
pub n_layers: usize,
pub n_heads: usize,
pub n_kv_heads: usize,
pub head_dim: usize,
pub d_ff: usize,
#[config(default = "1e-5")]
pub rms_eps: f64,
#[config(default = "1_000_000")]
pub rope_theta: usize,
#[config(default = "Some(8192)")]
pub sliding_window: Option<usize>,
}
impl VoxtralTextConfig {
pub fn voxtral_mini_4b_realtime() -> Self {
Self::new(131_072, 3072, 26, 32, 8, 128, 9216)
}
pub fn sliding_window(&self) -> Option<usize> {
self.sliding_window
}
pub fn stack(&self) -> UniformStackConfig {
let norm = BunsenNormalizationConfig::Rms(RmsNormConfig::new(0).with_epsilon(self.rms_eps));
UniformStackConfig::new(
self.n_layers,
CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(self.n_heads, self.n_kv_heads, self.d_model)
.with_head_dim(Some(self.head_dim))
.with_norm(None)
.with_sliding_window(self.sliding_window)
.with_softmax_dtype(Some(burn::tensor::FloatDType::F32))
.into(),
TransformerGatedMlpConfig::new(self.d_model, self.d_ff).into(),
)
.with_norm(norm.clone())
.with_mlp_scale(Some(AdaptiveScaleConfig::new(
self.d_model,
VOXTRAL_ADA_HIDDEN,
))),
norm,
RotaryEmbeddingConfig::new(0, self.head_dim)
.with_base(self.rope_theta)
.with_direction(RotaryDirection::Forward),
)
}
}
impl LmDecoderMeta for VoxtralTextConfig {
fn n_vocab(&self) -> usize {
self.n_vocab
}
fn d_model(&self) -> usize {
self.d_model
}
fn n_layers(&self) -> usize {
self.n_layers
}
fn n_kv_heads(&self) -> usize {
self.n_kv_heads
}
fn head_dim(&self) -> usize {
self.head_dim
}
}
impl ToStructureConfig for VoxtralTextConfig {
type Structure = LmDecoderVariantConfig;
fn try_to_structure(&self) -> BunsenResult<LmDecoderVariantConfig> {
Ok(LmDecoderVariantConfig::new(
TokenEmbeddingConfig::new(self.n_vocab, self.d_model),
LmHeadConfig::new(self.d_model, self.n_vocab),
self.stack().into(),
)
.with_kv_cache_pre_size(VOXTRAL_TEXT_KV_CACHE_PRE_SIZE))
}
}
#[derive(Module, Debug)]
pub struct VoxtralTextModel {
pub lm: LmDecoderModel,
}
impl LmDecoderHolder for VoxtralTextModel {
type Kind = LmAttentionOnlyKind;
fn lm(&self) -> &LmDecoderModel {
&self.lm
}
fn lm_mut(&mut self) -> &mut LmDecoderModel {
&mut self.lm
}
fn from_lm(lm: LmDecoderModel) -> Self {
Self { lm }
}
}
impl VoxtralTextModel {
pub fn sliding_window(&self) -> Option<usize> {
self.lm.stack.layers[0]
.attn
.branch
.attention()
.sliding_window
}
pub fn delay_condition(
&self,
delay_tokens: usize,
device: &Device,
) -> Tensor<3> {
timestep_embedding(
delay_tokens as f32,
self.d_model(),
VOXTRAL_TIME_THETA,
device,
)
.reshape([1, 1, self.d_model()])
}
pub fn forward_hidden(
&self,
embeds: Tensor<3>,
r_emb: &RotarySet,
cond: &Tensor<3>,
kv_cache: Option<&mut KVCache>,
) -> Tensor<3> {
self.lm.stack.forward_with(
embeds,
r_emb,
kv_cache,
DecoderStackSides {
cond: Some(cond.clone()),
..Default::default()
},
)
}
}
#[cfg(test)]
mod tests {
use burn::{
prelude::Int,
tensor::Tolerance,
};
use super::*;
use crate::{
blocks::transformers::models::language_decoder::LmDecoderOps,
burner::module::ModuleInit,
support::testing::{
assert_tensors_close,
cpu_device,
},
};
#[test]
fn test_meta_and_cache() {
let config = VoxtralTextConfig::new(40, 12, 2, 3, 1, 6, 20)
.with_rms_eps(1e-6)
.with_rope_theta(500)
.with_sliding_window(Some(3));
let device = cpu_device();
let model: VoxtralTextModel = config.init(&device);
assert_eq!(model.n_vocab(), config.n_vocab());
assert_eq!(model.d_model(), config.d_model());
assert_eq!(model.n_layers(), config.n_layers());
assert_eq!(model.n_kv_heads(), config.n_kv_heads());
assert_eq!(model.head_dim(), config.head_dim());
assert_eq!(model.sliding_window(), config.sliding_window());
assert_eq!(model.lm.stack.rotary.base, config.rope_theta);
assert_eq!(model.lm.stack.norm.epsilon(), Some(1e-6));
assert!(model.lm.stack.layers[0].mlp_scale.is_some());
let tokens = Tensor::<2, Int>::from_data([[1, 5, 7, 2, 9, 3]], &device);
let embeds = model.embed(tokens);
let r_emb = model.rotary(8, &device);
let cond = model.delay_condition(6, &device);
let whole = model.forward_hidden(embeds.clone(), &r_emb, &cond, None);
let mut cache = model.kv_cache_config().with_seq_len(8).init();
let head = model.forward_hidden(
embeds.clone().slice_dim(1, 0..4),
&r_emb,
&cond,
Some(&mut cache),
);
let tail = model.forward_hidden(embeds.slice_dim(1, 4..6), &r_emb, &cond, Some(&mut cache));
assert_tensors_close(
&Tensor::cat(vec![head, tail], 1),
&whole,
Tolerance::rel_abs(1e-4, 1e-5),
);
assert_eq!(model.logits(whole).dims(), [1, 6, 40]);
assert_eq!(cond.dims(), [1, 1, 12]);
}
}