use alloc::sync::Arc;
use core::fmt::Debug;
use burn::{
Tensor,
config::Config,
prelude::{
Int,
TensorData,
},
tensor::{
Device,
activation::softmax,
},
};
use crate::{
kits::{
speech::asr::whisper::{
Whisper,
WhisperMeta,
driver::WhisperSpecialIds,
logit_filters::RestrictToLanguages,
},
support::decoding::{
LogitFilter,
MaximumLikelihoodRanker,
SearchConfig,
SequenceRanker,
TokenSearch,
},
},
ops::{
repeat::repeat_interleave,
split::split_padded,
},
};
pub fn split_mel_windows(
mels: Tensor<3>,
window: usize,
) -> Vec<Tensor<3>> {
split_padded(mels, window, 2)
}
#[derive(Config, Debug)]
pub struct GreedyDecodeConfig {
pub prompt: Vec<i64>,
pub eot_token: i64,
#[config(default = "224")]
pub max_tokens: usize,
}
#[derive(Config, Debug)]
pub struct DecodeConfig {
pub prompt: Vec<i64>,
pub eot_token: i64,
#[config(default = "224")]
pub max_tokens: usize,
#[config(default = "SearchConfig::new()")]
pub search: SearchConfig,
#[config(default = "None")]
pub sot_token: Option<i64>,
#[config(default = "None")]
pub no_speech_token: Option<i64>,
#[config(default = "true")]
pub shared_cross_kv: bool,
}
#[derive(Debug, Clone, PartialEq)]
pub struct DecodedTokens {
pub tokens: Vec<i64>,
pub sum_logprob: f32,
pub no_speech_prob: Option<f32>,
pub temperature: f64,
}
impl DecodedTokens {
pub fn avg_logprob(&self) -> f64 {
f64::from(self.sum_logprob) / (self.tokens.len() as f64 + 1.0)
}
}
impl DecodeConfig {
pub fn init_decoder(&self) -> Box<dyn TokenSearch> {
self.search.init_search(&[self.eot_token], self.prompt[0])
}
pub fn init_ranker(&self) -> MaximumLikelihoodRanker {
self.search.init_ranker()
}
}
impl From<&GreedyDecodeConfig> for DecodeConfig {
fn from(config: &GreedyDecodeConfig) -> Self {
Self::new(config.prompt.clone(), config.eot_token).with_max_tokens(config.max_tokens)
}
}
fn feed_tensor(
feed: &[i64],
step_len: usize,
device: &Device,
) -> Tensor<2, Int> {
let rows = feed.len() / step_len;
Tensor::from_data(TensorData::new(feed.to_vec(), [rows, step_len]), device)
}
impl Whisper {
pub fn decode_windows(
&self,
mels: Tensor<3>,
config: &DecodeConfig,
filters: &[Arc<dyn LogitFilter>],
) -> Vec<Vec<i64>> {
self.decode_windows_full(mels, config, filters)
.into_iter()
.map(|d| d.tokens)
.collect()
}
pub fn decode_windows_full(
&self,
mels: Tensor<3>,
config: &DecodeConfig,
filters: &[Arc<dyn LogitFilter>],
) -> Vec<DecodedTokens> {
let [n_audio, _, frames] = mels.dims();
assert!(n_audio > 0, "decode needs at least one row");
assert_eq!(
frames,
self.max_audio_ctx(),
"window must be the model's audio context width",
);
let xa = self.forward_encoder(mels);
self.decode_features_full(xa, config, filters)
}
pub fn decode_windows_with(
&self,
mels: Tensor<3>,
config: &DecodeConfig,
decoder: &mut dyn TokenSearch,
filters: &[Arc<dyn LogitFilter>],
) -> Vec<Vec<i64>> {
let [n_audio, _, frames] = mels.dims();
assert!(n_audio > 0, "decode needs at least one row");
assert_eq!(
frames,
self.max_audio_ctx(),
"window must be the model's audio context width",
);
let xa = self.forward_encoder(mels);
self.decode_features_with(xa, config, decoder, filters)
}
pub fn decode_features(
&self,
xa: Tensor<3>,
config: &DecodeConfig,
filters: &[Arc<dyn LogitFilter>],
) -> Vec<Vec<i64>> {
self.decode_features_full(xa, config, filters)
.into_iter()
.map(|d| d.tokens)
.collect()
}
pub fn decode_features_full(
&self,
xa: Tensor<3>,
config: &DecodeConfig,
filters: &[Arc<dyn LogitFilter>],
) -> Vec<DecodedTokens> {
self.search(xa, config, config.init_decoder().as_mut(), filters)
}
pub fn decode_features_with(
&self,
xa: Tensor<3>,
config: &DecodeConfig,
decoder: &mut dyn TokenSearch,
filters: &[Arc<dyn LogitFilter>],
) -> Vec<Vec<i64>> {
self.search(xa, config, decoder, filters)
.into_iter()
.map(|d| d.tokens)
.collect()
}
pub fn search(
&self,
xa: Tensor<3>,
config: &DecodeConfig,
decoder: &mut dyn TokenSearch,
filters: &[Arc<dyn LogitFilter>],
) -> Vec<DecodedTokens> {
let n_audio = xa.dims()[0];
assert!(n_audio > 0, "decode needs at least one row");
assert!(!config.prompt.is_empty(), "prompt must not be empty");
decoder.reset();
let k = decoder.group_size();
let rows = n_audio * k;
let prompt_len = config.prompt.len();
let device = xa.device();
let mut cache = if k > 1 && !config.shared_cross_kv {
self.decoder
.new_cache(repeat_interleave::<3, 4, _>(xa, k, 0))
} else {
self.decoder.new_cache_grouped(xa, k)
};
let mut tokens: Vec<Vec<i64>> = vec![config.prompt.clone(); rows];
let mut sum_logprobs = vec![0f32; rows];
let mut no_speech: Option<Vec<f32>> = None;
let probe = match (config.sot_token, config.no_speech_token) {
(Some(sot), Some(no_speech)) => config
.prompt
.iter()
.rposition(|&t| t == sot)
.map(|at| (at, no_speech)),
_ => None,
};
let mut feed: Vec<i64> = config.prompt.repeat(rows);
let mut step_len = prompt_len;
for step in 0..config.max_tokens {
let logits = self
.decoder
.forward_cached(feed_tensor(&feed, step_len, &device), &mut cache);
let [_, positions, vocab] = logits.dims();
if step == 0
&& let Some((at, no_speech_id)) = probe
&& let Ok(id) = usize::try_from(no_speech_id)
&& id < vocab
{
let at_sot: Tensor<2> = logits
.clone()
.slice_dim(1, at as isize..(at + 1) as isize)
.reshape([rows, vocab]);
let probs = softmax(at_sot, 1).slice_dim(1, id as isize..(id + 1) as isize);
no_speech = Some(probs.into_data().try_into_vec_as::<f32>().unwrap());
}
let last = positions - 1;
let mut logits: Tensor<2> = logits
.slice_dim(1, last as isize..(last + 1) as isize)
.reshape([rows, vocab]);
for filter in filters {
logits = filter.apply(logits, &tokens, prompt_len);
}
let (next, completed) =
decoder.update(&mut tokens, logits, &mut sum_logprobs, &mut |sources| {
cache.reorder(sources)
});
if completed {
break;
}
feed = next;
step_len = 1;
if cache.pos() >= self.max_text_ctx() {
break;
}
}
let ranker = config.init_ranker();
decoder
.finalize(tokens, sum_logprobs, prompt_len)
.into_iter()
.enumerate()
.map(|(audio, candidates)| {
let best = ranker.rank(&candidates);
let candidate = candidates
.into_iter()
.nth(best)
.expect("the ranker picked a candidate");
DecodedTokens {
tokens: candidate.tokens,
sum_logprob: candidate.sum_logprob,
no_speech_prob: no_speech.as_ref().map(|p| p[audio * k]),
temperature: config.search.temperature,
}
})
.collect()
}
pub fn detect_language(
&self,
xa: Tensor<3>,
ids: &WhisperSpecialIds,
) -> Vec<i64> {
let config = DecodeConfig::new(vec![ids.sot], ids.eot).with_max_tokens(1);
let filters: [Arc<dyn LogitFilter>; 1] = [Arc::new(RestrictToLanguages::new(ids))];
self.decode_features(xa, &config, &filters)
.into_iter()
.map(|row| row[0])
.collect()
}
pub fn decode_window_batched(
&self,
mels: Tensor<3>,
config: &GreedyDecodeConfig,
) -> Vec<Vec<i64>> {
self.decode_windows(mels, &DecodeConfig::from(config), &[])
}
pub fn decode_window(
&self,
mels: Tensor<3>,
config: &GreedyDecodeConfig,
) -> Vec<i64> {
assert_eq!(mels.dims()[0], 1, "decode_window handles a batch of one");
self.decode_window_batched(mels, config)
.pop()
.expect("one row in, one row out")
}
pub fn decode_chunked(
&self,
mels: Tensor<3>,
config: &GreedyDecodeConfig,
) -> Vec<Vec<i64>> {
split_mel_windows(mels, self.max_audio_ctx())
.into_iter()
.map(|window| self.decode_window(window, config))
.collect()
}
}
#[cfg(test)]
mod tests {
use burn::tensor::Distribution;
use serial_test::serial;
use super::*;
use crate::{
burner::module::ModuleInit,
kits::speech::asr::whisper::blocks::WhisperApiConfig,
support::testing::{
DeviceMemoryGuard,
performance_device,
},
};
fn tiny_model(device: &burn::prelude::Device) -> Whisper {
WhisperApiConfig::new(
8, 32, 64,
16, 1, 12,
1,
)
.init(device)
}
#[test]
#[serial]
fn test_mel_windows_splits_and_pads() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let (batch, n_mels, window) = (1, 4, 10);
let exact: Tensor<3> = Tensor::random([batch, n_mels, 20], Distribution::Default, &device);
let windows = split_mel_windows(exact, window);
assert_eq!(windows.len(), 2);
assert!(windows.iter().all(|w| w.dims() == [batch, n_mels, window]));
let ragged: Tensor<3> = Tensor::random([batch, n_mels, 25], Distribution::Default, &device);
let windows = split_mel_windows(ragged, window);
assert_eq!(windows.len(), 3);
assert!(windows.iter().all(|w| w.dims() == [batch, n_mels, window]));
let short: Tensor<3> = Tensor::random([batch, n_mels, 3], Distribution::Default, &device);
assert_eq!(split_mel_windows(short, window).len(), 1);
}
#[test]
#[serial]
fn test_mel_windows_preserves_content() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let frames = 7;
let mels: Tensor<3> = Tensor::from_data(
TensorData::new(
(0..frames).map(|f| (f + 1) as f64).collect::<Vec<_>>(),
[1, 1, frames],
),
&device,
);
let windows = split_mel_windows(mels, 10);
assert_eq!(windows.len(), 1);
let got: Vec<f64> = windows[0].to_data().try_into_vec_as::<f64>().unwrap();
assert_eq!(got, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 0.0, 0.0, 0.0]);
}
#[test]
#[serial]
fn test_decode_window_respects_the_token_cap() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let mels: Tensor<3> = Tensor::random(
[1, model.n_mels(), model.max_audio_ctx()],
Distribution::Default,
&device,
);
let config = GreedyDecodeConfig::new(vec![1, 2], -1).with_max_tokens(5);
let out = model.decode_window(mels, &config);
assert!(
out.len() <= 5,
"generated {} tokens against a cap of 5",
out.len(),
);
assert!(
out.iter()
.all(|&t| t >= 0 && (t as usize) < model.vocab_size()),
"generated an id outside the vocabulary",
);
}
#[test]
#[serial]
fn test_decode_window_stops_on_eot() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let mels: Tensor<3> = Tensor::random(
[1, model.n_mels(), model.max_audio_ctx()],
Distribution::Default,
&device,
);
let probe = GreedyDecodeConfig::new(vec![1, 2], -1).with_max_tokens(1);
let first = model.decode_window(mels.clone(), &probe);
assert_eq!(first.len(), 1);
let config = GreedyDecodeConfig::new(vec![1, 2], first[0]).with_max_tokens(5);
assert!(
model.decode_window(mels, &config).is_empty(),
"decoding did not stop on the first emitted token",
);
}
#[test]
#[serial]
fn test_decode_chunked_covers_every_window() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let window = model.max_audio_ctx();
let mels: Tensor<3> = Tensor::random(
[1, model.n_mels(), window * 2 + window / 2],
Distribution::Default,
&device,
);
let config = GreedyDecodeConfig::new(vec![1], -1).with_max_tokens(3);
let per_window = model.decode_chunked(mels, &config);
assert_eq!(
per_window.len(),
3,
"the ragged tail must get its own window"
);
}
#[test]
#[serial]
fn test_batched_decode_matches_individual() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let (batch, window) = (3, model.max_audio_ctx());
let mels: Tensor<3> = Tensor::random(
[batch, model.n_mels(), window],
Distribution::Default,
&device,
);
let config = GreedyDecodeConfig::new(vec![1, 2], -1).with_max_tokens(6);
let batched = model.decode_window_batched(mels.clone(), &config);
assert_eq!(batched.len(), batch);
for row in 0..batch {
let alone = model.decode_window(
mels.clone().slice_dim(0, row as isize..(row + 1) as isize),
&config,
);
assert_eq!(
batched[row], alone,
"row {row} differs from its solo decode"
);
}
}
#[test]
#[serial]
fn test_batched_rows_finish_independently() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let (batch, window) = (3, model.max_audio_ctx());
let mels: Tensor<3> = Tensor::random(
[batch, model.n_mels(), window],
Distribution::Default,
&device,
);
let probe = GreedyDecodeConfig::new(vec![1, 2], -1).with_max_tokens(1);
let first_of_row0 = model.decode_window(mels.clone().slice_dim(0, 0..1), &probe)[0];
let config = GreedyDecodeConfig::new(vec![1, 2], first_of_row0).with_max_tokens(6);
let batched = model.decode_window_batched(mels.clone(), &config);
assert!(
batched[0].is_empty(),
"row 0 should stop on its first token"
);
for row in 0..batch {
let alone = model.decode_window(
mels.clone().slice_dim(0, row as isize..(row + 1) as isize),
&config,
);
assert_eq!(
batched[row], alone,
"row {row} was affected by another row finishing",
);
}
}
}
#[cfg(test)]
mod search_tests {
use burn::tensor::{
Distribution,
Tolerance,
};
use serial_test::serial;
use super::*;
use crate::{
burner::module::ModuleInit,
kits::{
speech::asr::whisper::blocks::WhisperApiConfig,
support::decoding::{
BeamSearch,
SuppressTokens,
},
},
support::testing::{
DeviceMemoryGuard,
performance_device,
},
};
type F = f32;
const VOCAB: usize = 32;
const EOT: i64 = 31;
fn tiny_model(device: &burn::prelude::Device) -> Whisper {
device.seed(11);
WhisperApiConfig::new(
8, VOCAB, 64,
16, 1, 12,
1,
)
.init(device)
}
fn windows(
n: usize,
device: &burn::prelude::Device,
) -> Tensor<3> {
Tensor::random([n, 8, 16], Distribution::Default, device)
}
#[test]
#[serial]
fn test_beam_of_one_is_greedy() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let mels = windows(3, &device);
for max_tokens in [2, 224] {
let config = DecodeConfig::new(vec![1, 2], EOT).with_max_tokens(max_tokens);
let greedy = model.decode_windows(mels.clone(), &config, &[]);
let mut beam = BeamSearch::new(1, [EOT], None);
let wide = model.decode_windows_with(mels.clone(), &config, &mut beam, &[]);
assert_eq!(wide, greedy, "beam of one at a cap of {max_tokens}");
assert!(greedy.iter().all(|row| row.len() <= max_tokens));
}
}
#[test]
#[serial]
fn test_beam_search_runs_wider() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let mels = windows(2, &device);
let config = DecodeConfig::new(vec![1, 2], EOT)
.with_max_tokens(6)
.with_search(
SearchConfig::new()
.with_beam_size(3)
.with_patience(Some(2.0)),
);
let first = model.decode_windows(mels.clone(), &config, &[]);
assert_eq!(first.len(), 2, "one answer per audio, not per beam");
assert!(first.iter().all(|row| row.len() <= 6));
assert!(first.iter().all(|row| !row.contains(&EOT)));
let again = model.decode_windows(mels, &config, &[]);
assert_eq!(again, first);
}
#[test]
#[serial]
fn test_reorder_permutes_the_self_attention_cache() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let mels = windows(1, &device).repeat_dim(0, 2);
let prompts = |rows: Vec<i64>| feed_tensor(&rows, 2, &device);
let next = feed_tensor(&[5, 5], 1, &device);
let mut swapped = model.decoder.new_cache(model.forward_encoder(mels.clone()));
model
.decoder
.forward_cached(prompts(vec![1, 2, 3, 4]), &mut swapped);
swapped.reorder(&[1, 0]);
let after_swap = model.decoder.forward_cached(next.clone(), &mut swapped);
let mut built = model.decoder.new_cache(model.forward_encoder(mels.clone()));
model
.decoder
.forward_cached(prompts(vec![3, 4, 1, 2]), &mut built);
let as_built = model.decoder.forward_cached(next.clone(), &mut built);
after_swap
.to_data_as::<F>()
.assert_approx_eq::<F>(&as_built.to_data_as::<F>(), Tolerance::permissive());
let mut branched = model.decoder.new_cache(model.forward_encoder(mels));
model
.decoder
.forward_cached(prompts(vec![1, 2, 3, 4]), &mut branched);
branched.reorder(&[0, 0]);
let both = model.decoder.forward_cached(next, &mut branched);
let [_, positions, vocab] = both.dims();
let row0 = both.clone().slice_dim(0, 0..1).reshape([positions, vocab]);
let row1 = both.slice_dim(0, 1..2).reshape([positions, vocab]);
row0.to_data_as::<F>()
.assert_approx_eq::<F>(&row1.to_data_as::<F>(), Tolerance::permissive());
}
#[test]
#[serial]
fn test_filters_reach_the_search() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let mels = windows(1, &device);
let only = |allowed: &[i64]| -> Vec<Arc<dyn LogitFilter>> {
let ids = (0..VOCAB as i64).filter(|id| !allowed.contains(id));
vec![Arc::new(SuppressTokens::new(ids))]
};
let config = DecodeConfig::new(vec![1, 2], EOT).with_max_tokens(4);
let out = model.decode_windows(mels.clone(), &config, &only(&[7]));
assert_eq!(out, vec![vec![7, 7, 7, 7]]);
let config = config.with_search(SearchConfig::new().with_beam_size(2));
let out = model.decode_windows(mels, &config, &only(&[7, 9]));
assert_eq!(out[0].len(), 4);
assert!(out[0].iter().all(|t| *t == 7 || *t == 9), "{out:?}");
}
#[test]
#[serial]
fn test_sampling_best_of_and_the_probe() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let mels = windows(2, &device);
let config = DecodeConfig::new(vec![1, 2], EOT)
.with_max_tokens(4)
.with_search(
SearchConfig::new()
.with_temperature(0.7)
.with_best_of(Some(3)),
)
.with_sot_token(Some(1))
.with_no_speech_token(Some(7));
device.seed(3);
let first = model.decode_windows_full(mels.clone(), &config, &[]);
assert_eq!(first.len(), 2, "one answer per audio, not per trajectory");
for d in &first {
assert_eq!(d.temperature, 0.7);
assert!(d.tokens.len() <= 4);
let p = d.no_speech_prob.expect("probed");
assert!((0.0..=1.0).contains(&p), "{p}");
assert!(d.sum_logprob <= 0.0);
assert!(d.avg_logprob() <= 0.0);
}
device.seed(3);
let again = model.decode_windows_full(mels.clone(), &config, &[]);
assert_eq!(again, first, "repeatable under the seed");
let plain = model.decode_windows_full(
mels,
&DecodeConfig::new(vec![1, 2], EOT).with_max_tokens(4),
&[],
);
assert!(
plain
.iter()
.all(|d| d.no_speech_prob.is_none() && d.temperature == 0.0)
);
}
#[test]
#[serial]
fn test_shared_cross_kv_matches_materialized() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let xa = model.forward_encoder(windows(2, &device));
let group = 3;
let mut shared = model.decoder.new_cache_grouped(xa.clone(), group);
assert_eq!(shared.group(), group);
let mut materialized = model
.decoder
.new_cache(repeat_interleave::<3, 4, _>(xa, group, 0));
assert_eq!(materialized.group(), 1);
let prompts = feed_tensor(&[1, 2, 1, 3, 1, 4, 2, 1, 3, 1, 4, 1], 2, &device);
let a = model.decoder.forward_cached(prompts.clone(), &mut shared);
let b = model.decoder.forward_cached(prompts, &mut materialized);
a.to_data_as::<F>()
.assert_approx_eq::<F>(&b.to_data_as::<F>(), Tolerance::permissive());
let sources = [1, 0, 1, 5, 5, 3];
shared.reorder(&sources);
materialized.reorder(&sources);
let next = feed_tensor(&[7, 8, 9, 7, 8, 9], 1, &device);
let a = model.decoder.forward_cached(next.clone(), &mut shared);
let b = model.decoder.forward_cached(next, &mut materialized);
a.to_data_as::<F>()
.assert_approx_eq::<F>(&b.to_data_as::<F>(), Tolerance::permissive());
}
#[test]
#[serial]
fn test_beam_search_on_shared_cross_kv_is_the_ground() {
#[derive(Debug)]
struct Ramp;
impl LogitFilter for Ramp {
fn apply(
&self,
logits: Tensor<2>,
_tokens: &[Vec<i64>],
_prompt_len: usize,
) -> Tensor<2> {
let [rows, vocab] = logits.dims();
let ramp: Tensor<1> =
Tensor::arange(0..vocab as i64, &logits.device()).float() * 0.05;
logits + ramp.unsqueeze::<2>().expand([rows, vocab])
}
}
let filters: Vec<Arc<dyn LogitFilter>> = vec![Arc::new(Ramp)];
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let model = tiny_model(&device);
let mels = windows(2, &device);
let base = DecodeConfig::new(vec![1, 2], EOT)
.with_max_tokens(6)
.with_search(SearchConfig::new().with_beam_size(3));
let shared = model.decode_windows_full(mels.clone(), &base, &filters);
let ground =
model.decode_windows_full(mels, &base.clone().with_shared_cross_kv(false), &filters);
assert_eq!(shared.len(), 2);
for (s, g) in shared.iter().zip(&ground) {
assert_eq!(s.tokens, g.tokens);
assert!(
(s.sum_logprob - g.sum_logprob).abs() < 1e-3,
"{} vs {}",
s.sum_logprob,
g.sum_logprob
);
}
}
}