use burn::{
Tensor,
module::Module,
prelude::{
Int,
TensorData,
},
tensor::Device,
};
use super::{
VoxtralTokenLayout,
blocks::VoxtralRealtime,
};
use crate::{
blocks::transformers::{
decoders::stack::DecoderStackCache,
models::language_decoder::{
LmDecoderMeta,
LmDecoderOps,
},
},
kits::support::decoding::{
CachedStep,
GreedySearch,
decode_cached,
},
ops::transformers::attention::KVCacheStorage,
};
#[derive(Module, Debug)]
pub struct VoxtralDecodeState {
pub cache: DecoderStackCache,
pub cond: Tensor<3>,
pub audio: Tensor<3>,
}
#[derive(Clone, Copy, Debug)]
pub struct VoxtralDecodeStep<'a> {
pub model: &'a VoxtralRealtime,
}
impl CachedStep for VoxtralDecodeStep<'_> {
type State = VoxtralDecodeState;
fn step(
&self,
ids: Tensor<2, Int>,
state: &mut VoxtralDecodeState,
) -> Tensor<2> {
let lm = &self.model.language_model;
let pos = state.cache.cache.pos();
let audio = state.audio.clone().slice_dim(1, pos..pos + 1);
let hidden = lm.forward_hidden(
lm.embed(ids) + audio,
&state.cache.rotary,
&state.cond,
Some(&mut state.cache.cache),
);
last_logits(lm.logits(hidden))
}
fn reorder(
&self,
state: &mut VoxtralDecodeState,
sources: &[usize],
) {
state.cache.reorder(sources);
let device = state.audio.device();
let index = Tensor::<1, Int>::from_data(
TensorData::new(
sources.iter().map(|&s| s as i64).collect::<Vec<_>>(),
[sources.len()],
),
&device,
);
state.audio = state.audio.clone().select(0, index);
}
}
fn last_logits(logits: Tensor<3>) -> Tensor<2> {
let [rows, seq, vocab] = logits.dims();
logits.slice_dim(1, seq - 1..seq).reshape([rows, vocab])
}
#[derive(Clone, Debug)]
pub struct VoxtralGreedyDecoder {
pub storage: KVCacheStorage,
}
impl Default for VoxtralGreedyDecoder {
fn default() -> Self {
Self {
storage: KVCacheStorage::Auto,
}
}
}
impl VoxtralGreedyDecoder {
pub fn state(
&self,
model: &VoxtralRealtime,
layout: &VoxtralTokenLayout,
audio: Tensor<3>,
device: &Device,
) -> VoxtralDecodeState {
let lm = &model.language_model;
let positions = audio.dims()[1];
VoxtralDecodeState {
cache: lm.lm.stack.decode_cache(1, positions, self.storage, device),
cond: lm.delay_condition(layout.delay_positions(), device),
audio,
}
}
pub fn decode(
&self,
model: &VoxtralRealtime,
layout: &VoxtralTokenLayout,
audio: Tensor<3>,
) -> Vec<i64> {
let [positions] = crate::contracts::unpack_shape_contract!(
[1, "positions", "d_text"],
&audio.dims(),
&["positions"],
&[("d_text", model.language_model.d_model())]
);
let prompt = layout.prompt();
assert!(
positions >= prompt.len(),
"{positions} audio positions under a {}-position prompt",
prompt.len()
);
let device = audio.device();
let mut state = self.state(model, layout, audio.clone(), &device);
let lm = &model.language_model;
let ids = Tensor::<2, Int>::from_data(
TensorData::new(prompt.clone(), [1, prompt.len()]),
&device,
);
let embeds = lm.embed(ids) + audio.slice_dim(1, 0..prompt.len());
let first = last_logits(lm.logits(lm.forward_hidden(
embeds,
&state.cache.rotary,
&state.cond,
Some(&mut state.cache.cache),
)));
let eos = layout.config().eos as i64;
let mut search = GreedySearch::new([eos], eos);
let budget = positions - prompt.len();
let out = decode_cached(
&VoxtralDecodeStep { model },
&mut state,
vec![prompt.clone()],
first,
budget,
&mut search,
&[],
);
let mut ids = prompt;
ids.extend(out[0][0].tokens_with_stop());
ids
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
burner::module::ModuleInit,
kits::speech::asr::voxtral::{
VoxtralTokenLayoutConfig,
blocks::{
VoxtralAudioEncoderConfig,
VoxtralRealtimeConfig,
VoxtralTextConfig,
},
},
support::testing::cpu_device,
};
#[test]
fn test_cached_decode_matches_uncached() {
let device = cpu_device();
let model: VoxtralRealtime = VoxtralRealtimeConfig::new(
VoxtralAudioEncoderConfig::new(6, 8, 2, 4, 16, 1).with_sliding_window(Some(3)),
VoxtralTextConfig::new(40, 12, 2, 3, 1, 4, 20).with_sliding_window(Some(5)),
)
.with_downsample(2)
.init(&device);
let layout = VoxtralTokenLayoutConfig::new()
.with_bos(1)
.with_eos(39)
.with_streaming_pad(3)
.with_streaming_word(4)
.with_left_pad_positions(2)
.with_delay_ms(160)
.init();
let audio = crate::support::testing::seeded_tensor::<3>(
1,
[1, 12, 12],
burn::tensor::Distribution::Default,
&device,
);
let ids = VoxtralGreedyDecoder::default().decode(&model, &layout, audio.clone());
let prompt = layout.prompt();
assert_eq!(&ids[..prompt.len()], &prompt[..]);
if ids.last() != Some(&39) {
assert_eq!(ids.len(), 12);
}
for n in prompt.len()..ids.len() {
let tokens =
Tensor::<2, Int>::from_data(TensorData::new(ids[..n].to_vec(), [1, n]), &device);
let logits = model.logits(tokens, audio.clone().slice_dim(1, 0..n), 2);
let expected = last_logits(logits).argmax(1).into_scalar::<i64>();
assert_eq!(ids[n], expected, "position {n}");
}
}
}