use std::sync::Arc;
use burn::{
config::Config,
module::Module,
tensor::Device,
};
use crate::{
errors::{
BunsenError,
BunsenResult,
ConstraintError,
},
kits::{
speech::{
asr_driver::{
AsrStreamDriver,
EmissionPolicy,
},
whisper::{
DecodeConfig,
Whisper,
WhisperFallbackConfig,
WhisperMeta,
blocks::{
AUDIO_ENCODER_STRIDE,
WhisperFrontEndConfig,
},
driver::{
WhisperBundle,
WhisperTask,
WhisperTokenLayout,
},
logit_filters::{
ApplyTimestampRules,
LogitFilter,
},
},
},
tokens::Detokenizer,
},
ops::signal::perceptive_audio::{
PerceptiveAudioConverter,
PerceptiveAudioConverterMeta,
},
};
#[derive(Config, Debug)]
pub struct WhisperStreamDriverConfig {
#[config(default = "None")]
pub language: Option<String>,
#[config(default = "WhisperTask::Transcribe")]
pub task: WhisperTask,
#[config(default = "false")]
pub timestamps: bool,
#[config(default = "Some(1.0)")]
pub max_initial_timestamp: Option<f64>,
#[config(default = "224")]
pub max_tokens: usize,
#[config(default = "1")]
pub beam_size: usize,
#[config(default = "None")]
pub patience: Option<f64>,
#[config(default = "None")]
pub length_penalty: Option<f64>,
#[config(default = "true")]
pub condition_on_previous_text: bool,
#[config(default = "EmissionPolicy::offline()")]
pub emission: EmissionPolicy,
#[config(default = "WhisperFallbackConfig::new()")]
pub fallback: WhisperFallbackConfig,
}
impl WhisperStreamDriverConfig {
pub fn init(
&self,
model: Whisper,
device: &Device,
) -> BunsenResult<WhisperStreamDriver> {
self.init_from_bundle(Arc::new(WhisperBundle::from_model(model)?), device)
}
pub fn init_with_layout(
&self,
whisper_model: Whisper,
token_layout: WhisperTokenLayout,
device: &Device,
) -> BunsenResult<WhisperStreamDriver> {
self.init_from_bundle(
Arc::new(WhisperBundle::new(whisper_model, token_layout)),
device,
)
}
pub fn init_from_bundle(
&self,
bundle: Arc<WhisperBundle>,
device: &Device,
) -> BunsenResult<WhisperStreamDriver> {
bundle.validate()?;
let special_ids = *bundle.layout.ids();
if !special_ids.is_multilingual() && self.language.is_some() {
return Err(BunsenError::illegal(
"WhisperStreamDriverConfig.language: an English-only checkpoint takes no language",
));
}
let task = special_ids.is_multilingual().then_some(self.task);
let prompt = match (special_ids.is_multilingual(), self.language.as_deref()) {
(true, None) => Vec::new(),
(_, language) => bundle
.layout
.sot_sequence(language, task, self.timestamps)?,
};
let max_initial_timestamp_index = self.max_initial_timestamp.map(|seconds| {
(seconds / bundle.layout.layout().timestamp_step_seconds).round() as usize
});
let mut filters: Vec<Arc<dyn LogitFilter>> = bundle.default_filters();
if self.timestamps {
filters.push(Arc::new(ApplyTimestampRules::new(
&special_ids,
max_initial_timestamp_index,
)));
}
const OWNER: &str = "WhisperStreamDriverConfig";
if self.beam_size == 0 {
return Err(ConstraintError::zero_or_empty(OWNER, "beam_size").into());
}
if self.fallback.temperatures.is_empty() {
return Err(ConstraintError::zero_or_empty(OWNER, "fallback.temperatures").into());
}
if self.fallback.best_of == Some(0) {
return Err(ConstraintError::zero_or_empty(OWNER, "fallback.best_of").into());
}
if (self.beam_size as f64 * self.patience.unwrap_or(1.0)).round() < 1.0 {
return Err(ConstraintError::out_of_range(
OWNER,
"patience",
self.patience.unwrap_or(1.0),
format!(
"[{}, ..) with {} beams, to collect a candidate",
0.5 / self.beam_size as f64,
self.beam_size,
),
)
.into());
}
let audio_converter = bundle
.model
.front_end()
.try_init_audio_converter(bundle.model.n_mels(), device)?;
#[cfg(feature = "tokenizer")]
let detokenizer: Option<Arc<dyn Detokenizer>> = bundle
.detokenizer()?
.map(|detokenizer| Arc::new(detokenizer) as Arc<dyn Detokenizer>);
#[cfg(not(feature = "tokenizer"))]
let detokenizer: Option<Arc<dyn Detokenizer>> = None;
let engine = WhisperEngine {
config: self.clone(),
bundle,
audio_converter,
prompt,
task,
max_initial_timestamp_index,
filters,
detokenizer,
};
AsrStreamDriver::new(engine, self.emission.clone())
}
}
pub type WhisperStreamDriver = AsrStreamDriver<WhisperEngine>;
#[derive(Clone, Debug)]
pub struct WhisperEngine {
pub(super) config: WhisperStreamDriverConfig,
pub(super) audio_converter: PerceptiveAudioConverter,
pub(super) bundle: Arc<WhisperBundle>,
pub(super) prompt: Vec<i64>,
pub(super) task: Option<WhisperTask>,
pub(super) max_initial_timestamp_index: Option<usize>,
pub(super) filters: Vec<Arc<dyn LogitFilter>>,
pub(super) detokenizer: Option<Arc<dyn Detokenizer>>,
}
impl WhisperEngine {
pub fn config(&self) -> &WhisperStreamDriverConfig {
&self.config
}
pub fn bundle(&self) -> &Arc<WhisperBundle> {
&self.bundle
}
pub fn whisper_model(&self) -> &Whisper {
&self.bundle.model
}
pub fn audio_converter(&self) -> &PerceptiveAudioConverter {
&self.audio_converter
}
pub fn token_layout(&self) -> &WhisperTokenLayout {
&self.bundle.layout
}
pub fn prompt(&self) -> &[i64] {
&self.prompt
}
pub fn filters(&self) -> &[Arc<dyn LogitFilter>] {
&self.filters
}
pub fn task(&self) -> Option<WhisperTask> {
self.task
}
pub fn detects_language(&self) -> bool {
self.prompt.is_empty()
}
pub fn sot_sequence(
&self,
language: Option<&str>,
) -> BunsenResult<Vec<i64>> {
self.bundle
.layout
.sot_sequence(language, self.task, self.config.timestamps)
}
pub fn frames_per_timestamp(&self) -> usize {
AUDIO_ENCODER_STRIDE
}
pub fn decode_config(
&self,
prompt: Vec<i64>,
) -> DecodeConfig {
let ids = self.bundle.layout.ids();
DecodeConfig::new(prompt, ids.eot)
.with_max_tokens(self.config.max_tokens)
.with_beam_size(self.config.beam_size)
.with_patience(self.config.patience)
.with_length_penalty(self.config.length_penalty)
.with_sot_token(Some(ids.sot))
.with_no_speech_token(Some(ids.no_speech))
}
pub fn detokenizer(&self) -> Option<&Arc<dyn Detokenizer>> {
self.detokenizer.as_ref()
}
pub fn window_frames(&self) -> usize {
self.bundle.model.max_audio_ctx()
}
pub fn front_end(&self) -> &WhisperFrontEndConfig {
self.bundle.model.front_end()
}
pub fn encoder_grid(&self) -> usize {
self.frames_per_timestamp() * self.audio_converter.hop()
}
pub fn devices(&self) -> Vec<Device> {
self.bundle.model.devices()
}
}
impl AsrStreamDriver<WhisperEngine> {
pub fn config(&self) -> &WhisperStreamDriverConfig {
self.engine().config()
}
pub fn bundle(&self) -> &Arc<WhisperBundle> {
self.engine().bundle()
}
pub fn whisper_model(&self) -> &Whisper {
self.engine().whisper_model()
}
pub fn audio_converter(&self) -> &PerceptiveAudioConverter {
self.engine().audio_converter()
}
pub fn token_layout(&self) -> &WhisperTokenLayout {
self.engine().token_layout()
}
pub fn prompt(&self) -> &[i64] {
self.engine().prompt()
}
pub fn filters(&self) -> &[Arc<dyn LogitFilter>] {
self.engine().filters()
}
pub fn detects_language(&self) -> bool {
self.engine().detects_language()
}
pub fn frames_per_timestamp(&self) -> usize {
self.engine().frames_per_timestamp()
}
pub fn decode_config(
&self,
prompt: Vec<i64>,
) -> DecodeConfig {
self.engine().decode_config(prompt)
}
pub fn window_frames(&self) -> usize {
self.engine().window_frames()
}
pub fn front_end(&self) -> &WhisperFrontEndConfig {
self.engine().front_end()
}
pub fn devices(&self) -> Vec<Device> {
self.engine().devices()
}
pub fn encoder_grid(&self) -> usize {
self.engine().encoder_grid()
}
pub fn with_detokenizer(
self,
detokenizer: Arc<dyn Detokenizer>,
) -> Self {
self.map_engine(|mut engine| {
engine.detokenizer = Some(detokenizer);
engine
})
}
pub fn with_logit_filters(
self,
filters: Vec<Arc<dyn LogitFilter>>,
) -> Self {
self.map_engine(|mut engine| {
engine.filters = filters;
if engine.config.timestamps {
engine.filters.push(Arc::new(ApplyTimestampRules::new(
engine.bundle.layout.ids(),
engine.max_initial_timestamp_index,
)));
}
engine
})
}
}