use burn::{
Tensor,
config::Config,
prelude::{
Bool,
Int,
TensorData,
},
};
use super::blocks::{
ParakeetTdt,
ParakeetTdtMeta,
};
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct TdtHypothesis {
pub tokens: Vec<usize>,
pub frames: Vec<usize>,
pub durations: Vec<usize>,
}
#[derive(Config, Debug)]
pub struct TdtGreedyDecoderConfig {
#[config(default = "Some(10)")]
pub max_symbols_per_frame: Option<usize>,
}
impl TdtGreedyDecoderConfig {
pub fn init(&self) -> TdtGreedyDecoder {
TdtGreedyDecoder {
config: self.clone(),
}
}
}
#[derive(Clone, Debug)]
pub struct TdtGreedyDecoder {
config: TdtGreedyDecoderConfig,
}
impl TdtGreedyDecoder {
pub fn config(&self) -> &TdtGreedyDecoderConfig {
&self.config
}
pub fn decode(
&self,
model: &ParakeetTdt,
encoded: Tensor<3>,
lengths: &[usize],
) -> Vec<TdtHypothesis> {
let [batch, frames, d] = encoded.dims();
assert_eq!(lengths.len(), batch, "one length per batch item");
let device = encoded.device();
let blank = model.blank_id();
let n_vocab = model.n_vocab();
let durations = model.durations().to_vec();
let mut hypotheses = vec![TdtHypothesis::default(); batch];
let mut t = vec![0usize; batch];
let mut at_frame = vec![0usize; batch];
let ids = |v: &[usize]| {
let v: Vec<i64> = v.iter().map(|&x| x as i64).collect();
Tensor::<1, Int>::from_data(TensorData::new(v, [batch]), &device)
};
let (mut prediction, mut state) = model.decoder.step(
ids(&vec![blank; batch]),
model.decoder.new_state(batch, &device),
);
while (0..batch).any(|b| t[b] < lengths[b]) {
let index: Vec<usize> = (0..batch)
.map(|b| t[b].min(lengths[b].max(1) - 1).min(frames - 1))
.collect();
let frame = encoded
.clone()
.gather(1, ids(&index).reshape([batch, 1, 1]).expand([batch, 1, d]))
.reshape([batch, d]);
let logits = model.joint.forward(frame, prediction.clone());
let tokens = logits
.clone()
.slice_dim(1, 0..n_vocab)
.argmax(1)
.into_data()
.try_into_vec_as::<i64>()
.expect("token argmax");
let choices = logits
.slice_dim(1, n_vocab..)
.argmax(1)
.into_data()
.try_into_vec_as::<i64>()
.expect("duration argmax");
let mut emitted = vec![false; batch];
for b in 0..batch {
if t[b] >= lengths[b] {
continue;
}
let token = tokens[b] as usize;
let mut duration = durations[choices[b] as usize];
if token != blank {
emitted[b] = true;
let hypothesis = &mut hypotheses[b];
hypothesis.tokens.push(token);
hypothesis.frames.push(t[b]);
hypothesis.durations.push(duration);
at_frame[b] += 1;
}
if duration == 0
&& (token == blank
|| self
.config
.max_symbols_per_frame
.is_some_and(|max| at_frame[b] >= max))
{
duration = 1;
}
if duration > 0 {
at_frame[b] = 0;
}
t[b] += duration;
}
if emitted.iter().any(|&e| e) {
let previous: Vec<usize> = (0..batch)
.map(|b| {
if emitted[b] {
tokens[b] as usize
} else {
blank
}
})
.collect();
let (next_prediction, next_state) =
model.decoder.step(ids(&previous), state.clone());
let mask = Tensor::<1, Bool>::from_data(TensorData::new(emitted, [batch]), &device);
let width = prediction.dims()[1];
prediction = prediction.mask_where(
mask.clone().reshape([batch, 1]).expand([batch, width]),
next_prediction,
);
state = state.select(mask, next_state);
}
}
hypotheses
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
blocks::{
signals::DwStridingSubsamplingConfig,
transformers::encoders::conformer::{
ConformerBlockConfig,
ConformerEncoderConfig,
},
},
burner::module::ModuleInit,
kits::speech::parakeet::blocks::ParakeetTdtConfig,
support::testing::cpu_device,
};
#[test]
fn test_decode_terminates_in_bounds() {
let device = cpu_device();
let config = ParakeetTdtConfig::new(
ConformerEncoderConfig::new(
DwStridingSubsamplingConfig::new(24, 4, 16),
ConformerBlockConfig::new(16, 2, 32, 5),
1,
),
11,
12,
);
let model: ParakeetTdt = config.init(&device);
let encoded = Tensor::<3>::random(
[2, 9, 12],
burn::tensor::Distribution::Normal(0.0, 1.0),
&device,
);
let decoder = TdtGreedyDecoderConfig::new().init();
let out = decoder.decode(&model, encoded, &[9, 4]);
assert_eq!(out.len(), 2);
for (h, len) in out.iter().zip([9, 4]) {
assert_eq!(h.tokens.len(), h.frames.len());
assert!(h.tokens.iter().all(|&t| t < 10));
assert!(h.frames.iter().all(|&f| f < len));
assert!(h.frames.windows(2).all(|w| w[0] <= w[1]));
for f in 0..len {
assert!(h.frames.iter().filter(|&&x| x == f).count() <= 10);
}
}
}
}