use burn::{
Tensor,
config::Config,
module::Module,
prelude::Int,
tensor::Device,
};
use super::{
VOXTRAL_STEM_STRIDE,
VoxtralAudioEncoder,
VoxtralAudioEncoderConfig,
VoxtralAudioEncoderMeta,
VoxtralProjector,
VoxtralProjectorConfig,
VoxtralProjectorMeta,
VoxtralTextConfig,
VoxtralTextModel,
};
use crate::{
blocks::transformers::models::language_decoder::{
LmDecoderMeta,
LmDecoderOps,
},
burner::module::{
CanonicalKeys,
ModuleInit,
Rekey,
},
errors::{
BunsenResult,
ConstraintError,
},
};
pub trait VoxtralRealtimeMeta {
fn n_mels(&self) -> usize;
fn d_audio(&self) -> usize;
fn d_text(&self) -> usize;
fn n_vocab(&self) -> usize;
fn downsample(&self) -> usize;
fn mel_frames_per_position(&self) -> usize {
VOXTRAL_STEM_STRIDE * self.downsample()
}
}
#[derive(Config, Debug)]
pub struct VoxtralRealtimeConfig {
pub audio: VoxtralAudioEncoderConfig,
pub text: VoxtralTextConfig,
#[config(default = "4")]
pub downsample: usize,
}
impl VoxtralRealtimeConfig {
pub fn voxtral_mini_4b_realtime() -> Self {
Self::new(
VoxtralAudioEncoderConfig::voxtral_mini_4b_realtime(),
VoxtralTextConfig::voxtral_mini_4b_realtime(),
)
}
pub fn projector(&self) -> VoxtralProjectorConfig {
VoxtralProjectorConfig::new(self.audio.d_model(), self.text.d_model())
.with_downsample(self.downsample)
}
}
impl VoxtralRealtimeMeta for VoxtralRealtimeConfig {
fn n_mels(&self) -> usize {
self.audio.n_mels()
}
fn d_audio(&self) -> usize {
self.audio.d_model()
}
fn d_text(&self) -> usize {
self.text.d_model()
}
fn n_vocab(&self) -> usize {
self.text.n_vocab()
}
fn downsample(&self) -> usize {
self.downsample
}
}
impl ModuleInit<VoxtralRealtime> for VoxtralRealtimeConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<VoxtralRealtime> {
if self.downsample == 0 {
return Err(ConstraintError::out_of_range(
"VoxtralRealtimeConfig",
"downsample",
"0",
"at least 1",
)
.into());
}
Ok(VoxtralRealtime {
audio_tower: self.audio.try_init(device)?,
multi_modal_projector: self.projector().try_init(device)?,
language_model: self.text.try_init(device)?,
})
}
}
#[derive(Module, Debug)]
pub struct VoxtralRealtime {
pub audio_tower: VoxtralAudioEncoder,
pub multi_modal_projector: VoxtralProjector,
pub language_model: VoxtralTextModel,
}
impl VoxtralRealtimeMeta for VoxtralRealtime {
fn n_mels(&self) -> usize {
self.audio_tower.n_mels()
}
fn d_audio(&self) -> usize {
self.audio_tower.d_model()
}
fn d_text(&self) -> usize {
self.language_model.d_model()
}
fn n_vocab(&self) -> usize {
self.language_model.n_vocab()
}
fn downsample(&self) -> usize {
self.multi_modal_projector.downsample()
}
}
impl VoxtralRealtime {
pub fn audio_embeds(
&self,
features: Tensor<3>,
) -> Tensor<3> {
let [batch, frames] = crate::contracts::unpack_shape_contract!(
["batch", "n_mels", "frames"],
&features.dims(),
&["batch", "frames"],
&[("n_mels", self.n_mels())]
);
let y = self
.multi_modal_projector
.forward(self.audio_tower.forward(features));
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "positions", "d_text"],
&y.dims(),
&[
("batch", batch),
("positions", frames / self.mel_frames_per_position()),
("d_text", self.d_text())
]
);
#[cfg(not(any(debug_assertions, test)))]
let _ = (batch, frames);
y
}
pub fn logits(
&self,
ids: Tensor<2, Int>,
audio: Tensor<3>,
delay_tokens: usize,
) -> Tensor<3> {
let [batch, positions] = crate::contracts::unpack_shape_contract!(
["batch", "positions"],
&ids.dims(),
&["batch", "positions"]
);
crate::contracts::assert_shape_contract_periodically!(
["batch", "positions", "d_text"],
&audio.dims(),
&[
("batch", batch),
("positions", positions),
("d_text", self.d_text())
]
);
let device = audio.device();
let lm = &self.language_model;
let embeds = lm.embed(ids) + audio;
let hidden = lm.forward_hidden(
embeds,
&lm.rotary(positions, &device),
&lm.delay_condition(delay_tokens, &device),
None,
);
lm.logits(hidden)
}
}
impl CanonicalKeys for VoxtralRealtime {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest()
.mount_canonical("audio_tower.", &self.audio_tower)
.mount_canonical("language_model.lm.", &self.language_model.lm)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
fn toy() -> VoxtralRealtimeConfig {
VoxtralRealtimeConfig::new(
VoxtralAudioEncoderConfig::new(6, 8, 2, 4, 16, 1).with_sliding_window(Some(3)),
VoxtralTextConfig::new(30, 12, 1, 3, 1, 4, 20),
)
.with_downsample(2)
}
#[test]
fn test_meta_and_shapes() {
let device = cpu_device();
let config = toy();
let model: VoxtralRealtime = config.init(&device);
assert_eq!(model.n_mels(), config.n_mels());
assert_eq!(model.d_audio(), config.d_audio());
assert_eq!(model.d_text(), config.d_text());
assert_eq!(model.n_vocab(), config.n_vocab());
assert_eq!(model.downsample(), config.downsample());
assert_eq!(config.mel_frames_per_position(), 4);
assert_eq!(
VoxtralRealtimeConfig::voxtral_mini_4b_realtime().mel_frames_per_position(),
8
);
let audio = model.audio_embeds(Tensor::zeros([1, 6, 12], &device));
assert_eq!(audio.dims(), [1, 3, 12]);
let ids = Tensor::<2, Int>::from_data([[1, 2, 3]], &device);
assert_eq!(model.logits(ids, audio, 6).dims(), [1, 3, 30]);
}
}