use burn::config::Config;
use crate::errors::{
BunsenError,
BunsenResult,
};
pub const LANGUAGES: &[&str] = &[
"en", "zh", "de", "es", "ru", "ko", "fr", "ja", "pt", "tr", "pl", "ca", "nl", "ar", "sv", "it",
"id", "hi", "fi", "vi", "he", "uk", "el", "ms", "cs", "ro", "da", "hu", "ta", "no", "th", "ur",
"hr", "bg", "lt", "la", "mi", "ml", "cy", "sk", "te", "fa", "lv", "bn", "sr", "az", "sl", "kn",
"et", "mk", "br", "eu", "is", "hy", "ne", "mn", "bs", "kk", "sq", "sw", "gl", "mr", "pa", "si",
"km", "sn", "yo", "so", "af", "oc", "ka", "be", "tg", "sd", "gu", "am", "yi", "lo", "uz", "fo",
"ht", "ps", "tk", "nn", "mt", "sa", "lb", "my", "bo", "tl", "mg", "as", "tt", "haw", "ln",
"ha", "ba", "jw", "su", "yue",
];
pub const LEADING_SPECIALS: [&str; 2] = ["<|endoftext|>", "<|startoftranscript|>"];
pub const CONTROL_TOKENS: [&str; 6] = [
"<|translate|>",
"<|transcribe|>",
"<|startoflm|>",
"<|startofprev|>",
"<|nospeech|>",
"<|notimestamps|>",
];
#[derive(Config, Debug, PartialEq)]
pub struct WhisperTokenLayoutConfig {
#[config(default = "LANGUAGES.iter().map(|&code| code.to_string()).collect()")]
pub languages: Vec<String>,
#[config(default = "50256")]
pub english_base_ranks: usize,
#[config(default = "50257")]
pub multilingual_base_ranks: usize,
#[config(default = "LEADING_SPECIALS.iter().map(|&name| name.to_string()).collect()")]
pub leading_specials: Vec<String>,
#[config(default = "CONTROL_TOKENS.iter().map(|&name| name.to_string()).collect()")]
pub control_tokens: Vec<String>,
#[config(default = "1501")]
pub timestamp_tokens: usize,
#[config(default = "0.02")]
pub timestamp_step_seconds: f64,
}
impl WhisperTokenLayoutConfig {
pub fn validate(&self) -> BunsenResult<()> {
if self.leading_specials.len() != LEADING_SPECIALS.len() {
return Err(BunsenError::Invalid(format!(
"a layout has {} leading specials, not {}",
LEADING_SPECIALS.len(),
self.leading_specials.len(),
)));
}
if self.control_tokens.len() != CONTROL_TOKENS.len() {
return Err(BunsenError::Invalid(format!(
"a layout has {} control tokens, not {}",
CONTROL_TOKENS.len(),
self.control_tokens.len(),
)));
}
if self.languages.is_empty() {
return Err(BunsenError::Invalid(
"a layout needs at least one language".to_string(),
));
}
if self.timestamp_tokens == 0 {
return Err(BunsenError::Invalid(
"a layout needs at least one timestamp token".to_string(),
));
}
if self.timestamp_step_seconds.is_nan() || self.timestamp_step_seconds <= 0.0 {
return Err(BunsenError::Invalid(format!(
"the timestamp step must be positive, got {}",
self.timestamp_step_seconds,
)));
}
Ok(())
}
pub fn size_without_languages(
&self,
n_base: usize,
) -> usize {
n_base + self.leading_specials.len() + self.control_tokens.len() + self.timestamp_tokens
}
pub fn timestamp_seconds(
&self,
index: usize,
) -> f64 {
index as f64 * self.timestamp_step_seconds
}
pub fn timestamp_name(
&self,
index: usize,
) -> String {
let hundredths = (self.timestamp_seconds(index) * 100.0).round() as usize;
format!("<|{}.{:02}|>", hundredths / 100, hundredths % 100)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_languages_table() {
assert_eq!(LANGUAGES.len(), 100);
assert_eq!(LANGUAGES[0], "en");
assert_eq!(LANGUAGES[98], "su");
assert_eq!(LANGUAGES[99], "yue");
}
#[test]
fn test_defaults_are_upstreams() {
let layout = WhisperTokenLayoutConfig::new();
assert!(layout.validate().is_ok());
assert_eq!(layout.languages.len(), 100);
assert_eq!(layout.leading_specials[0], "<|endoftext|>");
assert_eq!(layout.control_tokens[5], "<|notimestamps|>");
assert_eq!(layout.size_without_languages(50257), 51_766);
assert_eq!(layout.timestamp_name(0), "<|0.00|>");
assert_eq!(layout.timestamp_name(1), "<|0.02|>");
assert_eq!(layout.timestamp_name(50), "<|1.00|>");
assert_eq!(layout.timestamp_name(103), "<|2.06|>");
assert_eq!(layout.timestamp_name(1500), "<|30.00|>");
assert_eq!(layout.timestamp_seconds(100), 2.0);
}
#[test]
fn test_validate_rejects_missing_roles() {
let layout = WhisperTokenLayoutConfig::new();
assert!(
layout
.clone()
.with_leading_specials(vec!["<|eot|>".to_string()])
.validate()
.is_err()
);
assert!(
layout
.clone()
.with_control_tokens(Vec::new())
.validate()
.is_err()
);
assert!(
layout
.clone()
.with_languages(Vec::new())
.validate()
.is_err()
);
assert!(layout.clone().with_timestamp_tokens(0).validate().is_err());
assert!(layout.with_timestamp_step_seconds(0.0).validate().is_err());
}
}