use burn::{
Tensor,
tensor::Device,
};
use serde::{
Deserialize,
Serialize,
};
use super::{
ParakeetFrontEnd,
ParakeetFrontEndConfig,
ParakeetTdt,
ParakeetVocabulary,
TdtGreedyDecoderConfig,
TdtHypothesis,
};
use crate::{
burner::module::ModuleInit,
data::catalog::presets::PresetSelection,
errors::BunsenResult,
};
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ParakeetPolicy {
pub window_seconds: f64,
}
impl Default for ParakeetPolicy {
fn default() -> Self {
Self {
window_seconds: 60.0,
}
}
}
#[derive(Debug)]
pub struct ParakeetConstruct {
pub model: ParakeetTdt,
pub front_end: ParakeetFrontEnd,
pub vocab: ParakeetVocabulary,
pub policy: ParakeetPolicy,
pub presets: PresetSelection,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ParakeetTranscript {
pub hypothesis: TdtHypothesis,
pub text: String,
}
#[derive(Clone, Debug, PartialEq)]
pub struct TimedWord {
pub text: String,
pub start: f64,
pub end: f64,
}
impl ParakeetTranscript {
pub fn words(
&self,
vocab: &ParakeetVocabulary,
frame_seconds: f64,
) -> BunsenResult<Vec<TimedWord>> {
let h = &self.hypothesis;
let mut words: Vec<TimedWord> = Vec::new();
for (k, &token) in h.tokens.iter().enumerate() {
if vocab.is_special(token) {
continue;
}
let piece = vocab.transcript(&[token])?;
let start = h.frames[k] as f64 * frame_seconds;
let end = (h.frames[k] + h.durations[k].max(1)) as f64 * frame_seconds;
let begins = vocab
.piece(token)
.is_some_and(|p| p.starts_with(super::WORD_MARK));
match words.last_mut() {
Some(word) if !begins => {
word.text.push_str(&piece);
word.end = end;
}
_ => words.push(TimedWord {
text: piece,
start,
end,
}),
}
}
Ok(words)
}
}
impl ParakeetConstruct {
pub fn new(
model: ParakeetTdt,
vocab: ParakeetVocabulary,
device: &Device,
) -> BunsenResult<Self> {
Ok(Self {
model,
front_end: ParakeetFrontEndConfig::new().try_init(device)?,
vocab,
policy: ParakeetPolicy::default(),
presets: PresetSelection::default(),
})
}
pub fn with_policy(
mut self,
policy: ParakeetPolicy,
presets: PresetSelection,
) -> Self {
self.policy = policy;
self.presets = presets;
self
}
pub fn frame_seconds(&self) -> f64 {
use super::{
ParakeetFrontEndMeta,
ParakeetTdtMeta,
};
(self.front_end.hop() * self.model.subsampling_factor()) as f64
/ self.front_end.sample_rate() as f64
}
pub fn frame_samples(&self) -> usize {
use super::{
ParakeetFrontEndMeta,
ParakeetTdtMeta,
};
self.front_end.hop() * self.model.subsampling_factor()
}
pub fn transcribe_long(
&self,
samples: &[f32],
max_window: usize,
batch: usize,
device: &Device,
) -> BunsenResult<ParakeetTranscript> {
let frame = self.frame_samples();
assert!(
max_window >= 6 * frame,
"max_window shorter than six frames"
);
assert!(batch > 0, "batch is 0");
let windows = crate::ops::signal::quiet_cuts(samples, max_window, frame);
let mut joined = TdtHypothesis::default();
let mut texts = Vec::new();
for group in windows.chunks(batch) {
let longest = group.iter().map(|(a, b)| b - a).max().unwrap_or(0);
let mut data = vec![0.0f32; group.len() * longest];
for (row, &(a, b)) in group.iter().enumerate() {
data[row * longest..row * longest + (b - a)].copy_from_slice(&samples[a..b]);
}
let lengths: Vec<usize> = group.iter().map(|(a, b)| b - a).collect();
let waveforms = Tensor::<2>::from_data(
burn::prelude::TensorData::new(data, [group.len(), longest]),
(device, burn::tensor::DType::F32),
);
for (transcript, &(a, _)) in self
.transcribe(waveforms, Some(&lengths))?
.into_iter()
.zip(group)
{
let offset = a / frame;
let h = transcript.hypothesis;
joined.tokens.extend(h.tokens);
joined.frames.extend(h.frames.iter().map(|f| f + offset));
joined.durations.extend(h.durations);
if !transcript.text.is_empty() {
texts.push(transcript.text);
}
}
}
Ok(ParakeetTranscript {
hypothesis: joined,
text: texts.join(" "),
})
}
pub fn transcribe(
&self,
waveforms: Tensor<2>,
lengths: Option<&[usize]>,
) -> BunsenResult<Vec<ParakeetTranscript>> {
let (features, frames) = self.front_end.forward(waveforms, lengths);
let (encoded, frames) = self.model.encode(features, Some(&frames));
TdtGreedyDecoderConfig::new()
.init()
.decode(&self.model, encoded, &frames)
.into_iter()
.map(|hypothesis| {
let text = self.vocab.transcript(&hypothesis.tokens)?;
Ok(ParakeetTranscript { hypothesis, text })
})
.collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_words() {
let vocab = ParakeetVocabulary::new(
[
"<unk>",
"\u{2581}the",
"\u{2581}mo",
"on",
".",
"\u{2581}why",
]
.map(String::from)
.to_vec(),
);
let transcript = ParakeetTranscript {
hypothesis: TdtHypothesis {
tokens: vec![1, 0, 2, 3, 4, 5],
frames: vec![3, 4, 5, 6, 6, 10],
durations: vec![2, 1, 1, 0, 2, 4],
},
text: "the moon. why".into(),
};
let words = transcript.words(&vocab, 0.08).unwrap();
let summary: Vec<(String, f64, f64)> = words
.iter()
.map(|w| {
(
w.text.clone(),
(w.start * 100.0).round(),
(w.end * 100.0).round(),
)
})
.collect();
assert_eq!(
summary,
[
("the".to_string(), 24.0, 40.0),
("moon.".to_string(), 40.0, 64.0),
("why".to_string(), 80.0, 112.0),
]
);
}
}