use burn::{
Tensor,
module::Module,
};
use serde::{
Deserialize,
Serialize,
};
use super::{
VoxtralFrontEnd,
VoxtralFrontEndConfig,
VoxtralFrontEndMeta,
VoxtralGreedyDecoder,
VoxtralTokenLayout,
VoxtralTokenLayoutConfig,
VoxtralTranscript,
blocks::{
VoxtralRealtime,
VoxtralRealtimeMeta,
},
};
use crate::{
burner::module::ModuleInit,
data::catalog::presets::PresetSelection,
errors::BunsenResult,
kits::support::tokens::TekkenVocabulary,
};
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct VoxtralPolicy {
pub window_seconds: f64,
pub delay_ms: Option<usize>,
}
impl Default for VoxtralPolicy {
fn default() -> Self {
Self {
window_seconds: 30.0,
delay_ms: None,
}
}
}
#[derive(Clone, Debug)]
pub struct VoxtralConstruct {
pub model: VoxtralRealtime,
pub front_end: VoxtralFrontEnd,
pub vocab: TekkenVocabulary,
pub layout: VoxtralTokenLayout,
pub decoder: VoxtralGreedyDecoder,
pub policy: VoxtralPolicy,
pub presets: PresetSelection,
}
impl VoxtralConstruct {
pub fn new(
model: VoxtralRealtime,
vocab: TekkenVocabulary,
layout: VoxtralTokenLayout,
) -> BunsenResult<Self> {
let device = model.devices()[0].clone();
let front_end = VoxtralFrontEndConfig::new()
.with_n_mels(model.n_mels())
.try_init(&device)?;
Ok(Self {
model,
front_end,
vocab,
layout,
decoder: VoxtralGreedyDecoder::default(),
policy: VoxtralPolicy::default(),
presets: PresetSelection::default(),
})
}
pub fn with_policy(
self,
policy: VoxtralPolicy,
presets: PresetSelection,
) -> BunsenResult<Self> {
let mut construct = match policy.delay_ms {
Some(ms) => self.with_delay_ms(ms)?,
None => self,
};
construct.policy = policy;
construct.presets = presets;
Ok(construct)
}
pub fn with_delay_ms(
mut self,
delay_ms: usize,
) -> BunsenResult<Self> {
let config: VoxtralTokenLayoutConfig = self.layout.config().clone().with_delay_ms(delay_ms);
self.layout = config.try_init()?;
Ok(self)
}
pub fn samples_per_position(&self) -> usize {
self.front_end.hop() * self.model.mel_frames_per_position()
}
pub fn transcribe_features(
&self,
features: Tensor<3>,
) -> BunsenResult<VoxtralTranscript> {
let [frames] = crate::contracts::unpack_shape_contract!(
[1, "n_mels", "frames"],
&features.dims(),
&["frames"],
&[("n_mels", self.front_end.n_mels())]
);
let device = features.device();
let per = self.model.mel_frames_per_position();
let left = self.layout.config().left_pad_positions * per;
let right = frames.next_multiple_of(per) - frames + self.layout.right_pad_positions() * per;
let padded = Tensor::cat(
vec![
self.front_end.silence(left, &device),
features,
self.front_end.silence(right, &device),
],
2,
);
let audio = self.model.audio_embeds(padded);
let ids = self.decoder.decode(&self.model, &self.layout, audio);
VoxtralTranscript::parse(ids, &self.layout, &self.vocab)
}
pub fn transcribe(
&self,
samples: &[f32],
) -> BunsenResult<VoxtralTranscript> {
let device = self.model.devices()[0].clone();
let padded = self.layout.pad(samples, self.samples_per_position());
let features = self.front_end.forward(&padded, &device)?;
let audio = self.model.audio_embeds(features);
let ids = self.decoder.decode(&self.model, &self.layout, audio);
VoxtralTranscript::parse(ids, &self.layout, &self.vocab)
}
}