use burn::{
Tensor,
config::Config,
module::Module,
nn::{
PaddingConfig1d,
conv::Conv1dConfig,
norm::RmsNormConfig,
},
tensor::Device,
};
use crate::{
blocks::{
activation::BunsenActivationConfig,
conv::{
ConvBlock1dConfig,
ConvSeq1d,
ConvSeq1dConfig,
},
norm::BunsenNormalizationConfig,
transformers::{
attention::csa::CausalSelfAttentionConfig,
decoders::{
causal::CausalDecoderBlockConfig,
stack::{
DecoderStack,
DecoderStackMeta,
DecoderStackStructureConfig,
UniformStackConfig,
},
},
embedding::{
RotaryDirection,
RotaryEmbeddingConfig,
RotarySet,
},
mlp::TransformerGatedMlpConfig,
},
},
burner::module::{
CanonicalKeys,
ModuleInit,
Rekey,
ToStructureConfig,
},
errors::BunsenResult,
ops::transformers::attention::{
KVCache,
KVCacheConfig,
KVCacheModel,
},
};
pub const VOXTRAL_STEM_STRIDE: usize = 2;
pub trait VoxtralAudioEncoderMeta {
fn n_mels(&self) -> usize;
fn d_model(&self) -> usize;
fn n_layers(&self) -> usize;
fn n_heads(&self) -> usize;
fn head_dim(&self) -> usize;
fn sliding_window(&self) -> Option<usize>;
fn output_len(
&self,
frames: usize,
) -> usize {
frames.div_ceil(VOXTRAL_STEM_STRIDE)
}
}
pub const VOXTRAL_AUDIO_KV_CACHE_PRE_SIZE: usize = 1536;
fn kv_cache_config(meta: &impl VoxtralAudioEncoderMeta) -> KVCacheConfig {
KVCacheConfig::for_decode(
meta.n_heads(),
VOXTRAL_AUDIO_KV_CACHE_PRE_SIZE,
meta.head_dim(),
meta.n_layers(),
)
}
impl KVCacheModel for VoxtralAudioEncoderConfig {
fn kv_cache_config(&self) -> KVCacheConfig {
kv_cache_config(self)
}
}
impl KVCacheModel for VoxtralAudioEncoder {
fn kv_cache_config(&self) -> KVCacheConfig {
self.stack.kv_cache_config(VOXTRAL_AUDIO_KV_CACHE_PRE_SIZE)
}
}
#[derive(Config, Debug)]
pub struct VoxtralAudioEncoderConfig {
pub n_mels: usize,
pub d_model: usize,
pub n_heads: usize,
pub head_dim: usize,
pub d_ff: usize,
pub n_layers: usize,
#[config(default = "1e-5")]
pub rms_eps: f64,
#[config(default = "1_000_000")]
pub rope_theta: usize,
#[config(default = "Some(750)")]
pub sliding_window: Option<usize>,
}
impl VoxtralAudioEncoderConfig {
pub fn voxtral_mini_4b_realtime() -> Self {
Self::new(128, 1280, 32, 64, 5120, 32)
}
pub fn stack(&self) -> UniformStackConfig {
let norm = BunsenNormalizationConfig::Rms(RmsNormConfig::new(0).with_epsilon(self.rms_eps));
UniformStackConfig::new(
self.n_layers,
self.layer().with_norm(norm.clone()),
norm,
RotaryEmbeddingConfig::new(0, self.head_dim)
.with_base(self.rope_theta)
.with_direction(RotaryDirection::Forward),
)
}
fn layer(&self) -> CausalDecoderBlockConfig {
CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(self.n_heads, self.n_heads, self.d_model)
.with_head_dim(Some(self.head_dim))
.with_norm(None)
.with_qkv_bias(true)
.with_key_bias(Some(false))
.with_output_bias(true)
.with_sliding_window(self.sliding_window)
.with_softmax_dtype(Some(burn::tensor::FloatDType::F32))
.into(),
TransformerGatedMlpConfig::new(self.d_model, self.d_ff)
.with_down_bias(Some(true))
.into(),
)
}
}
impl VoxtralAudioEncoderMeta for VoxtralAudioEncoderConfig {
fn n_mels(&self) -> usize {
self.n_mels
}
fn d_model(&self) -> usize {
self.d_model
}
fn n_layers(&self) -> usize {
self.n_layers
}
fn n_heads(&self) -> usize {
self.n_heads
}
fn head_dim(&self) -> usize {
self.head_dim
}
fn sliding_window(&self) -> Option<usize> {
self.sliding_window
}
}
impl ToStructureConfig for VoxtralAudioEncoderConfig {
type Structure = VoxtralAudioEncoderStructureConfig;
fn try_to_structure(&self) -> BunsenResult<VoxtralAudioEncoderStructureConfig> {
let conv = |c_in, stride: usize| {
ConvBlock1dConfig::new(
Conv1dConfig::new(c_in, self.d_model, 3)
.with_stride(stride)
.with_padding(PaddingConfig1d::Explicit(3 - stride, 0)),
)
.with_act(Some(BunsenActivationConfig::Gelu))
};
Ok(VoxtralAudioEncoderStructureConfig::new(
ConvSeq1dConfig::new(vec![
conv(self.n_mels, 1),
conv(self.d_model, VOXTRAL_STEM_STRIDE),
]),
self.stack().try_to_structure()?,
))
}
}
#[derive(Config, Debug)]
pub struct VoxtralAudioEncoderStructureConfig {
pub stem: ConvSeq1dConfig,
pub stack: DecoderStackStructureConfig,
}
impl ModuleInit<VoxtralAudioEncoder> for VoxtralAudioEncoderStructureConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<VoxtralAudioEncoder> {
Ok(VoxtralAudioEncoder {
stem: self.stem.try_init(device)?,
stack: self.stack.try_init(device)?,
})
}
}
#[derive(Module, Debug)]
pub struct VoxtralAudioEncoder {
pub stem: ConvSeq1d,
pub stack: DecoderStack,
}
impl VoxtralAudioEncoderMeta for VoxtralAudioEncoder {
fn n_mels(&self) -> usize {
self.stem.blocks[0].conv.weight.dims()[1]
}
fn d_model(&self) -> usize {
self.stack.d_model()
}
fn n_layers(&self) -> usize {
self.stack.n_layers()
}
fn n_heads(&self) -> usize {
use crate::blocks::transformers::attention::csa::CausalSelfAttentionMeta;
self.stack.layers[0].attn.branch.attention().n_head()
}
fn head_dim(&self) -> usize {
self.stack.head_dim()
}
fn sliding_window(&self) -> Option<usize> {
self.stack.layers[0].attn.branch.attention().sliding_window
}
}
impl crate::burner::module::InputDTypes for VoxtralAudioEncoder {
fn preferred_dtype(&self) -> burn::tensor::FloatDType {
crate::burner::module::module_float_dtype(&self.stem)
.unwrap_or(burn::tensor::FloatDType::F32)
}
}
impl crate::burner::module::CoercesInput for VoxtralAudioEncoder {}
impl crate::burner::module::ResultDType for VoxtralAudioEncoder {
fn result_dtype(
&self,
_input: burn::tensor::FloatDType,
) -> burn::tensor::FloatDType {
crate::burner::module::InputDTypes::preferred_dtype(self)
}
}
impl VoxtralAudioEncoder {
pub fn rotary(
&self,
seq_len: usize,
device: &Device,
) -> RotarySet {
self.stack.rotary(seq_len, device)
}
pub fn stem(
&self,
features: Tensor<3>,
) -> Tensor<3> {
let [batch, frames] = crate::contracts::unpack_shape_contract!(
["batch", "n_mels", "frames"],
&features.dims(),
&["batch", "frames"],
&[("n_mels", self.n_mels())]
);
let features = crate::burner::module::InputDTypes::coerce_input(self, features);
let y = self.stem.forward(features).swap_dims(1, 2);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "seq", "d_model"],
&y.dims(),
&[
("batch", batch),
("seq", self.output_len(frames)),
("d_model", self.d_model())
]
);
#[cfg(not(any(debug_assertions, test)))]
let _ = (batch, frames);
y
}
pub fn forward_frames(
&self,
x: Tensor<3>,
r_emb: &RotarySet,
kv_cache: Option<&mut KVCache>,
) -> Tensor<3> {
self.stack.forward_hidden(x, r_emb, kv_cache)
}
pub fn forward(
&self,
features: Tensor<3>,
) -> Tensor<3> {
let x = self.stem(features);
let r_emb = self.rotary(x.dims()[1], &x.device());
self.forward_frames(x, &r_emb, None)
}
}
impl CanonicalKeys for VoxtralAudioEncoder {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest().mount_canonical("stack.", &self.stack)
}
}
#[cfg(test)]
mod tests {
use burn::tensor::{
Distribution,
Tolerance,
};
use super::*;
use crate::support::testing::{
assert_tensors_close,
cpu_device,
seeded_tensor,
};
fn toy() -> VoxtralAudioEncoderConfig {
VoxtralAudioEncoderConfig::new(6, 16, 2, 6, 24, 2)
.with_rms_eps(1e-6)
.with_rope_theta(10_000)
.with_sliding_window(Some(4))
}
#[test]
fn test_meta() {
let config = toy();
let encoder: VoxtralAudioEncoder = config.init(&cpu_device());
assert_eq!(encoder.n_mels(), config.n_mels());
assert_eq!(encoder.d_model(), config.d_model());
assert_eq!(encoder.n_layers(), config.n_layers());
assert_eq!(encoder.n_heads(), config.n_heads());
assert_eq!(encoder.head_dim(), config.head_dim());
assert_eq!(encoder.sliding_window(), config.sliding_window());
assert_eq!(encoder.stack.rotary.base, config.rope_theta);
assert_eq!(encoder.stack.norm.epsilon(), Some(1e-6));
assert_eq!(config.output_len(7), 4);
assert_eq!(encoder.kv_cache_config().num_layers, 2);
let preset = VoxtralAudioEncoderConfig::voxtral_mini_4b_realtime();
assert_eq!(preset.sliding_window(), Some(750));
assert_eq!(preset.rope_theta, 1_000_000);
}
#[test]
fn test_stem_is_causal() {
let device = cpu_device();
let encoder: VoxtralAudioEncoder = toy().init(&device);
let a = seeded_tensor::<3>(1, [1, 6, 12], Distribution::Default, &device);
let b = Tensor::cat(
vec![
a.clone().slice_dim(2, 0..8),
seeded_tensor::<3>(2, [1, 6, 4], Distribution::Default, &device),
],
2,
);
let ya = encoder.stem(a);
let yb = encoder.stem(b);
assert_eq!(ya.dims(), [1, 6, 16]);
assert_tensors_close(
&ya.slice_dim(1, 0..4),
&yb.slice_dim(1, 0..4),
Tolerance::default(),
);
}
#[test]
fn test_chunked_through_the_cache() {
let device = cpu_device();
let encoder: VoxtralAudioEncoder = toy().init(&device);
let features = seeded_tensor::<3>(3, [1, 6, 20], Distribution::Default, &device);
let whole = encoder.forward(features.clone());
let x = encoder.stem(features);
let r_emb = encoder.rotary(10, &device);
let mut cache = encoder.kv_cache_config().with_seq_len(10).init();
let parts: Vec<Tensor<3>> = [0..3, 3..4, 4..10]
.into_iter()
.map(|r| encoder.forward_frames(x.clone().slice_dim(1, r), &r_emb, Some(&mut cache)))
.collect();
assert_tensors_close(&Tensor::cat(parts, 1), &whole, Tolerance::default());
}
}