use std::sync::Arc;
use burn::{
Tensor,
module::Module,
prelude::TensorData,
tensor::Device,
};
use super::{
ParakeetBundle,
ParakeetFrontEnd,
ParakeetFrontEndMeta,
ParakeetTdtMeta,
TdtGreedyDecoderConfig,
TdtHypothesis,
WORD_MARK,
};
use crate::{
errors::{
BunsenError,
BunsenResult,
},
kits::speech::asr_driver::{
AsrEngine,
AsrStreamContext,
AsrStreamDriver,
DecodeRequest,
MomentsLevel,
StreamingFeatures,
TimedTokens,
WindowDecode,
WordSpan,
pause_cut,
word_segments,
},
ops::signal::perceptive_audio::PerceptiveAudioConverterMeta,
};
pub type ParakeetStreamDriver = AsrStreamDriver<ParakeetEngine>;
pub type ParakeetStreamContext = AsrStreamContext<ParakeetEngine>;
#[derive(Clone, Debug)]
pub struct ParakeetFeatures {
front_end: ParakeetFrontEnd,
last: Option<f32>,
pending: Vec<f32>,
finished: bool,
device: Device,
}
impl ParakeetFeatures {
pub fn new(
front_end: ParakeetFrontEnd,
device: Device,
) -> Self {
let half = front_end.config.win_length / 2;
Self {
front_end,
last: None,
pending: vec![0.0; half],
finished: false,
device,
}
}
fn frames(&mut self) -> Option<Tensor<3>> {
let converter = &self.front_end.converter;
let (n_fft, hop) = (converter.n_fft(), converter.hop());
if self.pending.len() < n_fft {
return None;
}
let count = (self.pending.len() - n_fft) / hop + 1;
let span = (count - 1) * hop + n_fft;
let x = Tensor::<2>::from_data(
TensorData::new(self.pending[..span].to_vec(), [1, span]),
&self.device,
);
self.pending.drain(..count * hop);
let mel = converter.mel(converter.spectrum(converter.frame(x)));
Some(mel.add_scalar(self.front_end.config.log_guard).log())
}
}
impl StreamingFeatures for ParakeetFeatures {
fn push(
&mut self,
samples: &[f32],
) -> BunsenResult<Option<Tensor<3>>> {
let a = self.front_end.config.preemphasis as f32;
for &x in samples {
self.pending.push(match self.last {
Some(last) => x - a * last,
None => x,
});
self.last = Some(x);
}
Ok(self.frames())
}
fn finish(&mut self) -> BunsenResult<Option<Tensor<3>>> {
if self.finished || self.last.is_none() {
self.finished = true;
return Ok(None);
}
self.finished = true;
let half = self.front_end.config.win_length / 2;
self.pending.extend(std::iter::repeat_n(0.0, half));
Ok(self.frames())
}
}
fn normalize(
front_end: &ParakeetFrontEnd,
window: Tensor<3>,
level: &MomentsLevel,
) -> Tensor<2> {
let [_, frames, n_mels] = window.dims();
let std = level
.sample_variance()
.sqrt()
.add_scalar(front_end.config.norm_epsilon);
((window - level.mean.clone().unsqueeze_dim(1)) / std.unsqueeze_dim(1))
.reshape([frames, n_mels])
.swap_dims(0, 1)
}
#[derive(Clone, Debug)]
pub struct ParakeetEngine {
bundle: Arc<ParakeetBundle>,
window_frames: usize,
device: Device,
}
impl ParakeetEngine {
pub const DEFAULT_WINDOW_SECONDS: f64 = 60.0;
pub fn new(bundle: Arc<ParakeetBundle>) -> Self {
let device = bundle.model.devices()[0].clone();
let engine = Self {
bundle,
window_frames: 0,
device,
};
engine.with_window_seconds(Self::DEFAULT_WINDOW_SECONDS)
}
pub fn with_window_seconds(
mut self,
seconds: f64,
) -> Self {
let front_end = &self.bundle.front_end;
let subsampling = self.bundle.model.subsampling_factor();
let frames = (seconds * front_end.sample_rate() as f64 / front_end.hop() as f64) as usize;
let frames = frames / subsampling * subsampling;
assert!(
frames > 0,
"a window of {seconds} s is under one encoder frame"
);
self.window_frames = frames;
self
}
pub fn bundle(&self) -> &Arc<ParakeetBundle> {
&self.bundle
}
fn spans(
&self,
hypothesis: &TdtHypothesis,
) -> Vec<WordSpan> {
let subsampling = self.bundle.model.subsampling_factor();
let vocab = &self.bundle.vocab;
let h = hypothesis;
let opens: Vec<usize> = (0..h.tokens.len())
.filter(|&k| {
k == 0
|| vocab
.piece(h.tokens[k])
.is_some_and(|p| p.starts_with(WORD_MARK))
})
.collect();
opens
.iter()
.enumerate()
.map(|(i, &first)| {
let last = opens.get(i + 1).map_or(h.tokens.len(), |&next| next) - 1;
WordSpan {
first_token: first,
start: h.frames[first] * subsampling,
end: (h.frames[last] + h.durations[last].max(1)) * subsampling,
}
})
.collect()
}
fn lay_out(
&self,
hypothesis: TdtHypothesis,
frames: usize,
) -> WindowDecode<()> {
let tokens: Vec<i64> = hypothesis.tokens.iter().map(|&t| t as i64).collect();
let words = self.spans(&hypothesis);
let grid = self.bundle.model.subsampling_factor();
let cut = (frames >= self.window_frames)
.then(|| pause_cut(&words, frames, grid))
.flatten();
let (advance, count) = cut.unwrap_or((frames, words.len()));
let committed = words.get(count).map_or(tokens.len(), |w| w.first_token);
WindowDecode {
segments: word_segments(&words[..count], &tokens[..committed], advance),
tail: cut.map(|(cut, _)| TimedTokens {
start: cut,
end: frames,
tokens: tokens[committed..].to_vec(),
}),
skip: tokens.is_empty(),
tokens,
advance,
carry: (),
}
}
}
impl AsrEngine for ParakeetEngine {
type Carry = ();
type Features = ParakeetFeatures;
type Level = MomentsLevel;
type StreamState = ();
fn sample_rate(&self) -> usize {
self.bundle.front_end.sample_rate()
}
fn hop(&self) -> usize {
self.bundle.front_end.hop()
}
fn window_frames(&self) -> usize {
self.window_frames
}
fn grid_frames(&self) -> usize {
self.bundle.model.subsampling_factor()
}
fn end_frames_dropped(&self) -> usize {
1
}
fn new_features(&self) -> BunsenResult<ParakeetFeatures> {
Ok(ParakeetFeatures::new(
self.bundle.front_end.clone(),
self.device.clone(),
))
}
fn new_stream_state(&self) {}
fn decode(
&self,
requests: Vec<DecodeRequest<'_, Self>>,
) -> BunsenResult<Vec<WindowDecode<()>>> {
if requests.is_empty() {
return Ok(Vec::new());
}
let lengths: Vec<usize> = requests.iter().map(|r| r.window.dims()[1]).collect();
let longest = lengths.iter().copied().max().unwrap_or(0);
let rows: Vec<Tensor<3>> = requests
.into_iter()
.map(|r| {
let frames = r.window.dims()[1];
let features = normalize(&self.bundle.front_end, r.window, &r.level);
let [n_mels, _] = features.dims();
let features = if frames < longest {
Tensor::cat(
vec![
features,
Tensor::zeros([n_mels, longest - frames], &self.device),
],
1,
)
} else {
features
};
features.unsqueeze_dim(0)
})
.collect();
let model = &self.bundle.model;
let (encoded, encoded_lengths) = model.encode(Tensor::cat(rows, 0), Some(&lengths));
let hypotheses =
TdtGreedyDecoderConfig::new()
.init()
.decode(model, encoded, &encoded_lengths);
Ok(hypotheses
.into_iter()
.zip(lengths)
.map(|(hypothesis, frames)| self.lay_out(hypothesis, frames))
.collect())
}
fn commit(
&self,
_state: &mut (),
_transcript: &[i64],
_carry: (),
) {
}
fn text(
&self,
tokens: &[i64],
) -> BunsenResult<Option<String>> {
let ids = tokens
.iter()
.map(|&t| {
usize::try_from(t).map_err(|_| {
BunsenError::invalid_resource(format!("{t} is not a Parakeet token id"))
})
})
.collect::<BunsenResult<Vec<usize>>>()?;
Ok(Some(self.bundle.vocab.transcript(&ids)?))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
blocks::conformer::{
ConformerBlockConfig,
ConformerEncoderConfig,
DwStridingSubsamplingConfig,
},
burner::module::ModuleInit,
kits::speech::{
asr_driver::{
EmissionPolicy,
LevelPolicy,
LevelStatistic,
StreamClock,
TranscriptEvent,
},
parakeet::{
ParakeetFrontEndConfig,
ParakeetTdt,
ParakeetTdtConfig,
ParakeetVocabulary,
},
},
support::testing::performance_device,
};
fn toy_bundle(device: &Device) -> ParakeetBundle {
let config = ParakeetTdtConfig::new(
ConformerEncoderConfig::new(
DwStridingSubsamplingConfig::new(24, 4, 16),
ConformerBlockConfig::new(16, 2, 32, 5)
.with_bias(false)
.with_conv_bias(false),
2,
),
11,
12,
);
let model: ParakeetTdt = config.init(device);
let pieces = [
"<unk>",
"\u{2581}the",
"\u{2581}mo",
"on",
".",
"\u{2581}why",
"\u{2581}we",
"\u{2581}go",
"s",
"t",
"<blk>",
];
ParakeetBundle {
model,
front_end: ParakeetFrontEndConfig::new().with_n_mels(24).init(device),
vocab: ParakeetVocabulary::new(pieces.map(String::from).to_vec()),
}
}
fn tone(samples: usize) -> Vec<f32> {
(0..samples)
.map(|i| {
let t = i as f32 / samples as f32;
let s = 2.0 * std::f32::consts::PI * i as f32 / 16_000.0;
(std::f32::consts::PI * t).sin()
* ((440.0 * s).sin() + 0.3 * (1_730.0 * s).sin())
* 0.4
})
.collect()
}
#[test]
fn test_streamed_features_are_the_front_ends() {
let device = performance_device();
let front_end: ParakeetFrontEnd = ParakeetFrontEndConfig::new().init(&device);
let samples = tone(20_850);
let wave = Tensor::<2>::from_data(
TensorData::new(samples.clone(), [1, samples.len()]),
&device,
);
let (one_shot, valid) = front_end.forward(wave, None);
let valid = valid[0];
assert_eq!(valid, 20_850 / 160);
let one_shot = one_shot.slice_dim(2, 0..valid).reshape([128, valid]);
for piece in [1, 160, 999, samples.len()] {
let mut features = ParakeetFeatures::new(front_end.clone(), device.clone());
let mut frames = Vec::new();
for chunk in samples.chunks(piece) {
frames.extend(features.push(chunk).unwrap());
}
frames.extend(features.finish().unwrap());
assert!(features.finish().unwrap().is_none());
let frames = Tensor::cat(frames, 1);
assert_eq!(frames.dims(), [1, valid + 1, 128], "piece {piece}");
let window = frames.slice_dim(1, 0..valid);
let level = MomentsLevel::of(window.clone());
normalize(&front_end, window, &level)
.to_data()
.assert_approx_eq::<f32>(
&one_shot.to_data(),
burn::tensor::Tolerance::absolute(2e-4),
);
}
let mut empty = ParakeetFeatures::new(front_end, device);
assert!(empty.push(&[]).unwrap().is_none());
assert!(empty.finish().unwrap().is_none());
}
#[test]
fn test_lay_out() {
let device = performance_device();
let engine = ParakeetEngine::new(Arc::new(toy_bundle(&device))).with_window_seconds(2.0);
assert_eq!(engine.window_frames(), 200);
assert_eq!(engine.grid_frames(), 8);
let full = engine.window_frames();
let hypothesis = |tokens: Vec<usize>, frames: Vec<usize>| TdtHypothesis {
durations: vec![1; tokens.len()],
tokens,
frames,
};
let spread = hypothesis(vec![1, 2, 3, 5, 6, 7], vec![2, 5, 6, 18, 20, 23]);
let decode = engine.lay_out(spread, full);
assert_eq!(decode.tokens, [1, 2, 3, 5, 6, 7]);
assert_eq!(
decode.segments,
[
TimedTokens {
start: 16,
end: 24,
tokens: vec![1],
},
TimedTokens {
start: 40,
end: 56,
tokens: vec![2, 3],
},
]
);
assert_eq!(
decode.tail,
Some(TimedTokens {
start: 96,
end: full,
tokens: vec![5, 6, 7],
})
);
assert_eq!(decode.advance, 96);
let words = hypothesis(vec![1, 2, 3, 4, 5, 6, 8], vec![0, 2, 3, 3, 5, 7, 8]);
let decode = engine.lay_out(words.clone(), full);
let spans = |d: &WindowDecode<()>| {
d.segments
.iter()
.map(|s| (s.start, s.end, s.tokens.clone()))
.collect::<Vec<_>>()
};
assert_eq!(
spans(&decode),
[(0, 8, vec![1]), (16, 32, vec![2, 3, 4]), (40, 48, vec![5]),]
);
assert_eq!(decode.tail.as_ref().unwrap().tokens, [6, 8]);
assert_eq!(decode.advance, 48);
let short = engine.lay_out(words, full - 1);
assert_eq!(short.segments.len(), 4);
assert_eq!(short.segments[3].tokens, [6, 8]);
assert_eq!((short.segments[3].start, short.segments[3].end), (56, 72));
assert_eq!((short.advance, short.tail), (full - 1, None));
let mut overlap = hypothesis(vec![1, 5], vec![2, 3]);
overlap.durations = vec![4, 1];
let decode = engine.lay_out(overlap, full - 1);
assert_eq!(spans(&decode)[0], (16, 24, vec![1]));
let one_word = engine.lay_out(hypothesis(vec![2, 3], vec![4, 5]), full);
assert_eq!(spans(&one_word), [(32, 48, vec![2, 3])]);
assert_eq!((one_word.advance, one_word.tail), (full, None));
let silent = engine.lay_out(hypothesis(vec![], vec![]), full - 1);
assert!(silent.skip && silent.segments.is_empty());
assert_eq!(engine.text(&[5, 6]).unwrap().as_deref(), Some("why we"));
assert!(engine.text(&[-1]).is_err());
}
#[test]
fn test_streams_through_the_core() {
let device = performance_device();
let engine = ParakeetEngine::new(Arc::new(toy_bundle(&device))).with_window_seconds(1.0);
let driver = ParakeetStreamDriver::new(engine, EmissionPolicy::offline()).unwrap();
let mut stream = driver
.new_context(StreamClock::uniform(16_000), LevelPolicy::PerWindow)
.unwrap();
let mut events = Vec::new();
for piece in tone(40_000).chunks(1_600) {
events.extend(stream.write_read(piece).unwrap());
}
events.extend(stream.end_read().unwrap());
assert!(stream.is_finished());
let mut end = 0.0;
for event in &events {
let TranscriptEvent::Committed(segment) = event else {
panic!("the offline policy only commits: {event:?}");
};
assert!(segment.start >= end && segment.end >= segment.start);
assert!(segment.end <= 2.5 + 1e-9);
assert_eq!(segment.text, driver.engine().text(&segment.tokens).unwrap());
end = segment.end;
}
}
}