use std::sync::Arc;
use burn::{
config::Config,
module::Module,
prelude::Backend,
};
use crate::{
errors::{
BunsenError,
BunsenResult,
ConstraintError,
Rule,
},
kits::{
speech::{
silero_vad::{
SileroVad,
SileroVadMeta,
},
whisper::{
DecodeConfig,
Whisper,
WhisperFallbackConfig,
WhisperMeta,
blocks::{
AUDIO_ENCODER_STRIDE,
WhisperFrontEndConfig,
},
driver::{
CommitRule,
EmissionPolicy,
StreamClampPolicy,
StreamClock,
VoiceActivityFilterConfig,
WhisperBundle,
WhisperStreamContext,
WhisperTask,
WhisperTokenLayout,
},
logit_filters::{
ApplyTimestampRules,
LogitFilter,
},
},
},
tokens::Detokenizer,
},
ops::signal::perceptive_audio::{
PerceptiveAudioConverter,
PerceptiveAudioConverterMeta,
},
};
#[derive(Config, Debug)]
pub struct WhisperStreamDriverConfig {
#[config(default = "None")]
pub language: Option<String>,
#[config(default = "WhisperTask::Transcribe")]
pub task: WhisperTask,
#[config(default = "false")]
pub timestamps: bool,
#[config(default = "Some(1.0)")]
pub max_initial_timestamp: Option<f64>,
#[config(default = "224")]
pub max_tokens: usize,
#[config(default = "1")]
pub beam_size: usize,
#[config(default = "None")]
pub patience: Option<f64>,
#[config(default = "None")]
pub length_penalty: Option<f64>,
#[config(default = "true")]
pub condition_on_previous_text: bool,
#[config(default = "EmissionPolicy::offline()")]
pub emission: EmissionPolicy,
#[config(default = "WhisperFallbackConfig::new()")]
pub fallback: WhisperFallbackConfig,
}
impl WhisperStreamDriverConfig {
pub fn init<B: Backend>(
&self,
model: Whisper<B>,
device: &B::Device,
) -> BunsenResult<WhisperStreamDriver<B>> {
self.init_from_bundle(Arc::new(WhisperBundle::from_model(model)?), device)
}
pub fn init_with_layout<B: Backend>(
&self,
whisper_model: Whisper<B>,
token_layout: WhisperTokenLayout,
device: &B::Device,
) -> BunsenResult<WhisperStreamDriver<B>> {
self.init_from_bundle(
Arc::new(WhisperBundle::new(whisper_model, token_layout)),
device,
)
}
pub fn init_from_bundle<B: Backend>(
&self,
bundle: Arc<WhisperBundle<B>>,
device: &B::Device,
) -> BunsenResult<WhisperStreamDriver<B>> {
bundle.validate()?;
let special_ids = *bundle.layout.ids();
if !special_ids.is_multilingual() && self.language.is_some() {
return Err(BunsenError::illegal(
"WhisperStreamDriverConfig.language: an English-only checkpoint takes no language",
));
}
let task = special_ids.is_multilingual().then_some(self.task);
let prompt = match (special_ids.is_multilingual(), self.language.as_deref()) {
(true, None) => Vec::new(),
(_, language) => bundle
.layout
.sot_sequence(language, task, self.timestamps)?,
};
let max_initial_timestamp_index = self.max_initial_timestamp.map(|seconds| {
(seconds / bundle.layout.layout().timestamp_step_seconds).round() as usize
});
let mut filters: Vec<Arc<dyn LogitFilter<B>>> = bundle.default_filters();
if self.timestamps {
filters.push(Arc::new(ApplyTimestampRules::new(
&special_ids,
max_initial_timestamp_index,
)));
}
const OWNER: &str = "WhisperStreamDriverConfig";
let triggers = &self.emission.triggers;
if !triggers.window_full && !triggers.endpoint {
return Err(BunsenError::illegal(
"WhisperStreamDriverConfig.emission.triggers: with neither the window_full nor \
the endpoint trigger nothing would ever decode",
));
}
if triggers.interval.is_some_and(|i| i.is_zero()) {
return Err(BunsenError::from(ConstraintError::zero_or_empty(
OWNER,
"emission.triggers.interval",
))
.with_details("an interval of zero would draft on every push"));
}
if triggers.interval.is_some() && !triggers.endpoint {
return Err(BunsenError::illegal(
"WhisperStreamDriverConfig.emission.triggers: the interval trigger drafts only \
while speech is in progress, which only the endpoint trigger's voice-activity \
gate tracks; turn endpoint on as well",
));
}
if let CommitRule::Agreement { .. } = self.emission.commit {
return Err(BunsenError::unsupported(
"WhisperStreamDriverConfig.emission.commit: the Agreement commit rule is not \
implemented; use Complete or LastTimestamp",
));
}
if self.beam_size == 0 {
return Err(ConstraintError::zero_or_empty(OWNER, "beam_size").into());
}
if self.fallback.temperatures.is_empty() {
return Err(ConstraintError::zero_or_empty(OWNER, "fallback.temperatures").into());
}
if self.fallback.best_of == Some(0) {
return Err(ConstraintError::zero_or_empty(OWNER, "fallback.best_of").into());
}
if (self.beam_size as f64 * self.patience.unwrap_or(1.0)).round() < 1.0 {
return Err(ConstraintError::out_of_range(
OWNER,
"patience",
self.patience.unwrap_or(1.0),
format!(
"[{}, ..) with {} beams, to collect a candidate",
0.5 / self.beam_size as f64,
self.beam_size,
),
)
.into());
}
let audio_converter = bundle
.model
.front_end()
.try_init_audio_converter(bundle.model.n_mels(), device)?;
#[cfg(feature = "tokenizer")]
let detokenizer: Option<Arc<dyn Detokenizer>> = bundle
.detokenizer()?
.map(|detokenizer| Arc::new(detokenizer) as Arc<dyn Detokenizer>);
#[cfg(not(feature = "tokenizer"))]
let detokenizer: Option<Arc<dyn Detokenizer>> = None;
Ok(WhisperStreamDriver {
config: self.clone(),
bundle,
audio_converter,
vad_model: None,
prompt,
task,
max_initial_timestamp_index,
filters,
va_filter: None,
detokenizer,
})
}
}
#[derive(Clone, Debug)]
pub struct WhisperStreamDriver<B: Backend> {
config: WhisperStreamDriverConfig,
audio_converter: PerceptiveAudioConverter<B>,
bundle: Arc<WhisperBundle<B>>,
vad_model: Option<SileroVad<B>>,
va_filter: Option<VoiceActivityFilterConfig>,
prompt: Vec<i64>,
task: Option<WhisperTask>,
max_initial_timestamp_index: Option<usize>,
filters: Vec<Arc<dyn LogitFilter<B>>>,
detokenizer: Option<Arc<dyn Detokenizer>>,
}
impl<B: Backend> WhisperStreamDriver<B> {
pub fn with_detokenizer(
mut self,
detokenizer: Arc<dyn Detokenizer>,
) -> Self {
self.detokenizer = Some(detokenizer);
self
}
pub fn with_logit_filters(
mut self,
filters: Vec<Arc<dyn LogitFilter<B>>>,
) -> Self {
self.filters = filters;
if self.config.timestamps {
self.filters.push(Arc::new(ApplyTimestampRules::new(
self.bundle.layout.ids(),
self.max_initial_timestamp_index,
)));
}
self
}
pub fn with_vad(
mut self,
vad: SileroVad<B>,
filter: VoiceActivityFilterConfig,
) -> BunsenResult<Self> {
self.check_vad(&vad, &filter)?;
self.vad_model = Some(vad);
self.va_filter = Some(filter);
Ok(self)
}
fn check_vad(
&self,
vad: &SileroVad<B>,
filter: &VoiceActivityFilterConfig,
) -> BunsenResult<()> {
const OWNER: &str = "WhisperStreamDriver::with_vad";
let must_equal = |lhs: (&str, usize), rhs: (&str, usize)| -> BunsenError {
ConstraintError::new(
OWNER,
"",
Rule::Relation {
lhs: (lhs.0.into(), lhs.1.to_string()),
op: "==",
rhs: (rhs.0.into(), rhs.1.to_string()),
},
)
.into()
};
let rate = self.sample_rate();
if vad.sample_rate() != rate {
return Err(must_equal(
("vad.sample_rate()", vad.sample_rate()),
("the driver's sample_rate()", rate),
));
}
if filter.sample_rate != rate {
return Err(must_equal(
("filter.sample_rate", filter.sample_rate),
("the driver's sample_rate()", rate),
));
}
if filter.samples_per_chunk != vad.chunk_size() {
return Err(must_equal(
("filter.samples_per_chunk", filter.samples_per_chunk),
("vad.chunk_size()", vad.chunk_size()),
));
}
Ok(())
}
pub fn config(&self) -> &WhisperStreamDriverConfig {
&self.config
}
pub fn bundle(&self) -> &Arc<WhisperBundle<B>> {
&self.bundle
}
pub fn whisper_model(&self) -> &Whisper<B> {
&self.bundle.model
}
pub fn audio_converter(&self) -> &PerceptiveAudioConverter<B> {
&self.audio_converter
}
pub fn silero_vad_model(&self) -> Option<&SileroVad<B>> {
self.vad_model.as_ref()
}
pub fn va_filter_config(&self) -> Option<VoiceActivityFilterConfig> {
self.va_filter.clone()
}
pub fn token_layout(&self) -> &WhisperTokenLayout {
&self.bundle.layout
}
pub fn prompt(&self) -> &[i64] {
&self.prompt
}
pub fn filters(&self) -> &[Arc<dyn LogitFilter<B>>] {
&self.filters
}
pub fn task(&self) -> Option<WhisperTask> {
self.task
}
pub fn detects_language(&self) -> bool {
self.prompt.is_empty()
}
pub fn sot_sequence(
&self,
language: Option<&str>,
) -> BunsenResult<Vec<i64>> {
self.bundle
.layout
.sot_sequence(language, self.task, self.config.timestamps)
}
pub fn frames_per_timestamp(&self) -> usize {
AUDIO_ENCODER_STRIDE
}
pub fn decode_config(
&self,
prompt: Vec<i64>,
) -> DecodeConfig {
let ids = self.bundle.layout.ids();
DecodeConfig::new(prompt, ids.eot)
.with_max_tokens(self.config.max_tokens)
.with_beam_size(self.config.beam_size)
.with_patience(self.config.patience)
.with_length_penalty(self.config.length_penalty)
.with_sot_token(Some(ids.sot))
.with_no_speech_token(Some(ids.no_speech))
}
pub fn interval_samples(&self) -> Option<usize> {
self.config
.emission
.triggers
.interval
.map(|i| (i.as_secs_f64() * self.sample_rate() as f64).round() as usize)
}
pub fn detokenizer(&self) -> Option<&Arc<dyn Detokenizer>> {
self.detokenizer.as_ref()
}
pub fn window_frames(&self) -> usize {
self.bundle.model.max_audio_ctx()
}
pub fn sample_rate(&self) -> usize {
self.bundle.model.sample_rate()
}
pub fn front_end(&self) -> &WhisperFrontEndConfig {
self.bundle.model.front_end()
}
pub fn encoder_grid(&self) -> usize {
self.frames_per_timestamp() * self.audio_converter.hop()
}
pub fn devices(&self) -> Vec<B::Device> {
self.bundle.model.devices()
}
pub fn new_context<C: StreamClampPolicy<B> + 'static>(
&self,
clock: StreamClock,
clamp: C,
) -> BunsenResult<WhisperStreamContext<B>> {
if clock.rate() != self.sample_rate() {
return Err(ConstraintError::new(
"WhisperStreamDriver::new_context",
"",
Rule::Relation {
lhs: ("clock.rate()".into(), clock.rate().to_string()),
op: "==",
rhs: (
"the model's sample_rate()".into(),
self.sample_rate().to_string(),
),
},
)
.into());
}
if self.config.emission.triggers.endpoint && self.vad_model.is_none() {
return Err(BunsenError::illegal(
"the endpoint trigger needs a voice-activity model; attach one with with_vad",
));
}
Ok(WhisperStreamContext::init(
self.clone(),
clock,
Box::new(clamp),
))
}
}