use burn::{
Tensor,
prelude::{
Backend,
Int,
},
tensor::{
Distribution,
activation::log_softmax,
},
};
use crate::kits::speech::whisper::decode::TokenDecoder;
#[derive(Debug, Clone)]
pub struct WhisperGreedyDecoder {
eot: i64,
filler: i64,
temperature: f64,
group: usize,
finished: Vec<bool>,
}
impl WhisperGreedyDecoder {
pub fn new(
eot: i64,
filler: i64,
) -> Self {
Self {
eot,
filler,
temperature: 0.0,
group: 1,
finished: Vec::new(),
}
}
pub fn with_temperature(
mut self,
temperature: f64,
) -> Self {
assert!(temperature >= 0.0, "a temperature is not negative");
self.temperature = temperature;
self
}
pub fn with_group(
mut self,
group: usize,
) -> Self {
assert!(group >= 1, "at least one trajectory");
self.group = group;
self
}
pub fn temperature(&self) -> f64 {
self.temperature
}
}
impl<B: Backend> TokenDecoder<B> for WhisperGreedyDecoder {
fn group_size(&self) -> usize {
self.group
}
fn reset(&mut self) {
self.finished.clear();
}
fn update(
&mut self,
tokens: &mut Vec<Vec<i64>>,
logits: Tensor<B, 2>,
sum_logprobs: &mut [f32],
_reorder: &mut dyn FnMut(&[usize]),
) -> (Vec<i64>, bool) {
let [rows, _] = logits.dims();
if self.finished.len() != rows {
self.finished = vec![false; rows];
}
let picked: Tensor<B, 2, Int> = if self.temperature > 0.0 {
let uniform: Tensor<B, 2> = Tensor::random(
logits.dims(),
Distribution::Uniform(0.0, 1.0),
&logits.device(),
)
.clamp(1e-20, 1.0);
let gumbel = uniform.log().neg().log().neg();
(logits.clone() / self.temperature + gumbel).argmax(1)
} else {
logits.clone().argmax(1)
};
let chosen: Vec<i64> = picked
.clone()
.into_data()
.convert::<i64>()
.to_vec()
.unwrap();
let logprobs: Vec<f32> = log_softmax(logits, 1)
.gather(1, picked)
.into_data()
.convert::<f32>()
.to_vec()
.unwrap();
let mut feed = Vec::with_capacity(rows);
for row in 0..rows {
let token = chosen[row];
if self.finished[row] {
feed.push(self.filler);
} else if token == self.eot {
self.finished[row] = true;
feed.push(self.filler);
} else {
tokens[row].push(token);
sum_logprobs[row] += logprobs[row];
feed.push(token);
}
}
let completed = self.finished.iter().all(|&done| done);
(feed, completed)
}
fn finalize(
&mut self,
tokens: Vec<Vec<i64>>,
sum_logprobs: Vec<f32>,
prompt_len: usize,
) -> Vec<Vec<(Vec<i64>, f32)>> {
let group = self.group.max(1);
tokens
.into_iter()
.zip(sum_logprobs)
.map(|(sequence, logprob)| (sequence[prompt_len..].to_vec(), logprob))
.collect::<Vec<_>>()
.chunks(group)
.map(|audio| audio.to_vec())
.collect()
}
}
#[cfg(test)]
mod tests {
use burn::prelude::TensorData;
use super::*;
use crate::support::testing::CpuBackend;
type B = CpuBackend;
fn logits(rows: &[&[f32]]) -> Tensor<B, 2> {
let vocab = rows[0].len();
let flat: Vec<f32> = rows.iter().flat_map(|r| r.iter().copied()).collect();
Tensor::from_data(
TensorData::new(flat, [rows.len(), vocab]),
&Default::default(),
)
}
#[test]
fn test_rows_finish_independently() {
let eot = 3;
let mut decoder = WhisperGreedyDecoder::new(eot, 9);
let mut tokens = vec![vec![9, 9], vec![9, 9]];
let mut sums = vec![0.0, 0.0];
let mut reorders = 0;
let mut reorder = |_: &[usize]| reorders += 1;
let (feed, done) = TokenDecoder::<B>::update(
&mut decoder,
&mut tokens,
logits(&[&[0.0, 5.0, 0.0, 0.0], &[0.0, 0.0, 0.0, 5.0]]),
&mut sums,
&mut reorder,
);
assert_eq!(feed, vec![1, 9], "row 1 stopped and is fed the filler");
assert!(!done);
assert_eq!(tokens, vec![vec![9, 9, 1], vec![9, 9]]);
assert!(sums[0] < 0.0 && sums[0] > -0.1, "log p of a confident pick");
assert_eq!(sums[1], 0.0, "a stop token adds nothing");
let (feed, done) = TokenDecoder::<B>::update(
&mut decoder,
&mut tokens,
logits(&[&[0.0, 0.0, 0.0, 5.0], &[5.0, 0.0, 0.0, 0.0]]),
&mut sums,
&mut reorder,
);
assert_eq!(feed, vec![9, 9]);
assert!(done);
assert_eq!(
tokens,
vec![vec![9, 9, 1], vec![9, 9]],
"a finished row ignores its logits"
);
assert_eq!(reorders, 0, "greedy never permutes the cache");
let out = TokenDecoder::<B>::finalize(&mut decoder, tokens, sums, 2);
assert_eq!(out[0].len(), 1);
assert_eq!(out[0][0].0, vec![1]);
assert_eq!(out[1][0].0, Vec::<i64>::new());
TokenDecoder::<B>::reset(&mut decoder);
assert!(decoder.finished.is_empty());
}
#[test]
fn test_sampling() {
let mut decoder = WhisperGreedyDecoder::new(3, 9).with_temperature(0.5);
assert_eq!(decoder.temperature(), 0.5);
let mut reorder = |_: &[usize]| {};
for _ in 0..8 {
let mut tokens = vec![vec![9]];
let mut sums = vec![0.0];
let (feed, _) = TokenDecoder::<B>::update(
&mut decoder,
&mut tokens,
logits(&[&[0.0, 30.0, 0.0, 0.0]]),
&mut sums,
&mut reorder,
);
assert_eq!(feed, vec![1]);
assert!(
sums[0] > -1e-6 && sums[0] <= 0.0,
"log p of a certain pick: {}",
sums[0]
);
TokenDecoder::<B>::reset(&mut decoder);
}
let mut seen = [0usize; 4];
for _ in 0..64 {
let mut tokens = vec![vec![9]];
let mut sums = vec![0.0];
let (feed, _) = TokenDecoder::<B>::update(
&mut decoder,
&mut tokens,
logits(&[&[5.0, -30.0, 5.0, -30.0]]),
&mut sums,
&mut reorder,
);
seen[feed[0] as usize] += 1;
TokenDecoder::<B>::reset(&mut decoder);
}
assert!(seen[0] > 0 && seen[2] > 0, "both sides of a tie: {seen:?}");
assert!(
seen[1] == 0 && seen[3] == 0,
"never the improbable: {seen:?}"
);
}
#[test]
fn test_group_finalize() {
let mut decoder = WhisperGreedyDecoder::new(3, 9)
.with_temperature(0.5)
.with_group(2);
assert_eq!(TokenDecoder::<B>::group_size(&decoder), 2);
let out = TokenDecoder::<B>::finalize(
&mut decoder,
vec![vec![9, 1], vec![9, 2], vec![9, 1, 1], vec![9]],
vec![-1.0, -2.0, -3.0, -4.0],
1,
);
assert_eq!(out.len(), 2);
assert_eq!(out[0], vec![(vec![1], -1.0), (vec![2], -2.0)]);
assert_eq!(out[1], vec![(vec![1, 1], -3.0), (vec![], -4.0)]);
}
}