use std::{
fmt,
sync::Arc,
};
use crate::{
errors::{
BunsenResult,
ConstraintError,
Rule,
},
kits::{
speech::whisper::{
Whisper,
WhisperMeta,
blocks::{
AudioEncoderMeta,
TextDecoderMeta,
},
driver::WhisperTokenLayout,
logit_filters::{
LogitFilter,
default_filters,
},
},
tokens::TiktokenRanks,
},
};
#[derive(Debug)]
pub struct WhisperBundle {
pub model: Whisper,
pub layout: WhisperTokenLayout,
pub ranks: Option<TiktokenRanks>,
}
impl WhisperBundle {
pub fn new(
model: Whisper,
layout: WhisperTokenLayout,
) -> Self {
Self {
model,
layout,
ranks: None,
}
}
pub fn from_model(model: Whisper) -> BunsenResult<Self> {
let layout = model.token_layout().policy_for_vocab(model.vocab_size())?;
Ok(Self::new(model, layout))
}
pub fn with_ranks(
mut self,
ranks: TiktokenRanks,
) -> Self {
self.ranks = Some(ranks);
self
}
pub fn validate(&self) -> BunsenResult<()> {
let ids = self.layout.ids();
if ids.n_vocab() > self.model.vocab_size() {
return Err(ConstraintError::new(
"WhisperBundle",
"",
Rule::Relation {
lhs: ("layout.ids().n_vocab()".into(), ids.n_vocab().to_string()),
op: "<=",
rhs: (
"model.vocab_size()".into(),
self.model.vocab_size().to_string(),
),
},
)
.into());
}
if let Some(ranks) = &self.ranks {
self.layout.token_spans(ranks)?;
}
Ok(())
}
pub fn default_filters(&self) -> Vec<Arc<dyn LogitFilter>> {
match &self.ranks {
Some(ranks) => default_filters(ranks, self.layout.ids()),
None => Vec::new(),
}
}
#[cfg(feature = "tokenizer")]
pub fn detokenizer(
&self
) -> BunsenResult<Option<crate::kits::tokens::WordchipperDetokenizer<u16>>> {
self.ranks
.as_ref()
.map(|ranks| self.layout.detokenizer(ranks))
.transpose()
}
}
impl fmt::Display for WhisperBundle {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
write!(
f,
"{} mels, vocabulary {}, d_model {}, {} + {} layers",
self.model.n_mels(),
self.model.vocab_size(),
self.model.d_model(),
self.model.encoder().n_layers(),
self.model.decoder().n_layers(),
)?;
match &self.ranks {
Some(ranks) => write!(f, ", {} ranks", ranks.len()),
None => f.write_str(", ids only"),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
burner::module::ModuleInit,
errors::{
BunsenErrorKind,
testing::ErrorMatcher,
},
kits::speech::whisper::{
WhisperGeometry,
blocks::WhisperTokenLayoutConfig,
},
support::testing::cpu_device,
};
fn tiny_model() -> Whisper {
WhisperGeometry::openai(8, 64, 64, 1, 1)
.to_api_config()
.try_init(&cpu_device())
.unwrap()
}
fn tiny_layout() -> WhisperTokenLayout {
WhisperTokenLayoutConfig::new()
.with_languages(vec!["en".to_string()])
.with_timestamp_tokens(1)
.policy(2, 1)
.unwrap()
}
#[test]
fn test_ids_only() {
let bundle = WhisperBundle::new(tiny_model(), tiny_layout());
let shown = bundle.to_string();
assert!(shown.contains(" mels, vocabulary "), "{shown}");
assert!(shown.ends_with(" layers, ids only"), "{shown}");
bundle.validate().unwrap();
assert!(bundle.ranks.is_none());
assert!(bundle.default_filters().is_empty());
#[cfg(feature = "tokenizer")]
assert!(bundle.detokenizer().unwrap().is_none());
}
#[test]
fn test_with_a_vocabulary() {
let ranks = TiktokenRanks::parse("IA== 0\nb2s= 1\n").unwrap();
let bundle = WhisperBundle::new(tiny_model(), tiny_layout()).with_ranks(ranks);
assert!(bundle.to_string().ends_with(" ranks"), "{bundle}");
bundle.validate().unwrap();
assert!(!bundle.default_filters().is_empty());
#[cfg(feature = "tokenizer")]
{
use crate::kits::tokens::Detokenizer;
assert_eq!(
bundle
.detokenizer()
.unwrap()
.unwrap()
.detokenize(&[1])
.unwrap(),
"ok"
);
}
let short = WhisperBundle::new(tiny_model(), tiny_layout())
.with_ranks(TiktokenRanks::parse("IA== 0\n").unwrap());
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.has_cause::<ConstraintError>()
.assert_err(&short.validate());
}
#[test]
fn test_layout_must_fit_the_model() {
let too_big = WhisperBundle::new(
tiny_model(),
WhisperTokenLayout::from_vocab_size(51865).unwrap(),
);
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.has_cause::<ConstraintError>()
.message_contains("51865")
.assert_err(&too_big.validate());
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.has_cause::<ConstraintError>()
.assert_err(&WhisperBundle::from_model(tiny_model()));
}
}