use alloc::sync::Arc;
use burn::{
module::Module,
tensor::Device,
};
use crate::{
errors::BunsenResult,
kits::speech::asr::{
driver::{
AsrEngine,
AsrStreamContext,
AsrStreamDriver,
DecodeRequest,
LogMelFeatures,
WindowDecode,
WordSpan,
token_ids,
window_frames_in,
},
voxtral::{
VoxtralConstruct,
VoxtralFrontEndMeta,
VoxtralTranscript,
blocks::VoxtralRealtimeMeta,
},
},
};
pub type VoxtralStreamDriver = AsrStreamDriver<VoxtralEngine>;
pub type VoxtralStreamContext = AsrStreamContext<VoxtralEngine>;
#[derive(Clone, Debug)]
pub struct VoxtralEngine {
bundle: Arc<VoxtralConstruct>,
window_frames: usize,
device: Device,
}
impl VoxtralEngine {
pub fn new(bundle: Arc<VoxtralConstruct>) -> BunsenResult<Self> {
let device = bundle.model.devices()[0].clone();
let seconds = bundle.policy.window_seconds;
let engine = Self {
bundle,
window_frames: 0,
device,
};
engine.with_window_seconds(seconds)
}
pub fn with_window_seconds(
mut self,
seconds: f64,
) -> BunsenResult<Self> {
let front_end = &self.bundle.front_end;
self.window_frames = window_frames_in(
"VoxtralEngine::with_window_seconds",
seconds,
front_end.sample_rate(),
front_end.hop(),
self.bundle.model.mel_frames_per_position(),
None,
)?;
Ok(self)
}
pub fn bundle(&self) -> &Arc<VoxtralConstruct> {
&self.bundle
}
fn spans(
&self,
transcript: &VoxtralTranscript,
) -> (Vec<WordSpan>, Vec<i64>) {
let per = self.bundle.model.mel_frames_per_position();
let config = self.bundle.layout.config();
let lead = config.left_pad_positions + self.bundle.layout.delay_positions();
let frame = |position: usize| position.saturating_sub(lead) * per;
let mut tokens = Vec::new();
let mut words = Vec::new();
for word in &transcript.words {
let first = tokens.len();
tokens.extend_from_slice(&word.ids);
let last = word.position + word.ids.len();
words.push(WordSpan {
first_token: first,
start: frame(word.position),
end: frame(last + 1),
});
}
(words, tokens)
}
fn lay_out(
&self,
transcript: &VoxtralTranscript,
frames: usize,
) -> WindowDecode<()> {
let (words, tokens) = self.spans(transcript);
WindowDecode::from_words(
&words,
tokens,
frames,
self.window_frames,
self.grid_frames(),
(),
)
}
}
impl AsrEngine for VoxtralEngine {
type Carry = ();
type Features = LogMelFeatures;
type Level = ();
type StreamState = ();
fn sample_rate(&self) -> usize {
self.bundle.front_end.sample_rate()
}
fn hop(&self) -> usize {
self.bundle.front_end.hop()
}
fn window_frames(&self) -> usize {
self.window_frames
}
fn grid_frames(&self) -> usize {
self.bundle.model.mel_frames_per_position()
}
fn end_frames_dropped(&self) -> usize {
1
}
fn new_features(&self) -> BunsenResult<LogMelFeatures> {
Ok(LogMelFeatures::new(
self.bundle.front_end.converter.clone(),
self.device.clone(),
))
}
fn new_stream_state(&self) {}
fn decode(
&self,
requests: Vec<DecodeRequest<'_, Self>>,
) -> BunsenResult<Vec<WindowDecode<()>>> {
requests
.into_iter()
.map(|request| {
let frames = request.window.dims()[1];
let features = self.bundle.front_end.package(request.window);
let transcript = self.bundle.transcribe_features(features)?;
Ok(self.lay_out(&transcript, frames))
})
.collect()
}
fn commit(
&self,
_state: &mut (),
_transcript: &[i64],
_carry: (),
) {
}
fn text(
&self,
tokens: &[i64],
) -> BunsenResult<Option<String>> {
let text = tokens
.iter()
.copied()
.filter(|&t| !self.bundle.layout.is_control(t));
let ids = token_ids(text, "Voxtral")?;
Ok(Some(self.bundle.vocab.text(&ids)?))
}
}