use std::{
path::PathBuf,
sync::Arc,
};
use crate::{
data::pretrained::{
HfProvider,
PretrainedProvider,
StaticPretrained,
StaticPretrainedGroup,
StaticPretrainedTable,
StaticResourceMap,
WELL_KNOWN,
},
kits::speech::whisper::pretrained::{
BASE_CHECKPOINT,
BASE_EN_CHECKPOINT,
CHECKPOINT,
GPT2_VOCABULARY,
LARGE_V1_CHECKPOINT,
LARGE_V2_CHECKPOINT,
LARGE_V3_CHECKPOINT,
LARGE_V3_TURBO_CHECKPOINT,
MEDIUM_CHECKPOINT,
MEDIUM_EN_CHECKPOINT,
MULTILINGUAL_VOCABULARY,
SMALL_CHECKPOINT,
SMALL_EN_CHECKPOINT,
TINY_CHECKPOINT,
TINY_EN_CHECKPOINT,
},
};
pub const WHISPER_KIT: &str = "whisper";
pub const OPENAI_LOCAL_DIR: &str = "openai-whisper";
pub fn openai_download_root() -> Option<PathBuf> {
let cache_home = std::env::var_os("XDG_CACHE_HOME")
.map(PathBuf::from)
.or_else(|| std::env::home_dir().map(|h| h.join(".cache")))?;
Some(cache_home.join("whisper"))
}
const fn openai(
name: &'static str,
aliases: &'static [&'static str],
prefab: &'static str,
description: &'static str,
maps: &'static [&'static StaticResourceMap<'static>],
) -> StaticPretrained<'static> {
StaticPretrained {
name,
aliases,
description,
license: Some("MIT"),
origin: Some("https://github.com/openai/whisper/blob/main/whisper/__init__.py"),
prefab: Some(prefab),
maps,
}
}
static TINY_EN: StaticPretrained<'static> = openai(
"tiny.en",
&[],
"tiny.en",
"39 M parameters, English-only",
&[&TINY_EN_CHECKPOINT, &GPT2_VOCABULARY],
);
static TINY: StaticPretrained<'static> = openai(
"tiny",
&[],
"tiny",
"39 M parameters, multilingual",
&[&TINY_CHECKPOINT, &MULTILINGUAL_VOCABULARY],
);
static BASE_EN: StaticPretrained<'static> = openai(
"base.en",
&[],
"base.en",
"74 M parameters, English-only",
&[&BASE_EN_CHECKPOINT, &GPT2_VOCABULARY],
);
static BASE: StaticPretrained<'static> = openai(
"base",
&[],
"base",
"74 M parameters, multilingual",
&[&BASE_CHECKPOINT, &MULTILINGUAL_VOCABULARY],
);
static SMALL_EN: StaticPretrained<'static> = openai(
"small.en",
&[],
"small.en",
"244 M parameters, English-only",
&[&SMALL_EN_CHECKPOINT, &GPT2_VOCABULARY],
);
static SMALL: StaticPretrained<'static> = openai(
"small",
&[],
"small",
"244 M parameters, multilingual",
&[&SMALL_CHECKPOINT, &MULTILINGUAL_VOCABULARY],
);
static MEDIUM_EN: StaticPretrained<'static> = openai(
"medium.en",
&[],
"medium.en",
"769 M parameters, English-only",
&[&MEDIUM_EN_CHECKPOINT, &GPT2_VOCABULARY],
);
static MEDIUM: StaticPretrained<'static> = openai(
"medium",
&[],
"medium",
"769 M parameters, multilingual",
&[&MEDIUM_CHECKPOINT, &MULTILINGUAL_VOCABULARY],
);
static LARGE_V1: StaticPretrained<'static> = openai(
"large-v1",
&[],
"large",
"1550 M parameters, multilingual",
&[&LARGE_V1_CHECKPOINT, &MULTILINGUAL_VOCABULARY],
);
static LARGE_V2: StaticPretrained<'static> = openai(
"large-v2",
&[],
"large",
"1550 M parameters, multilingual",
&[&LARGE_V2_CHECKPOINT, &MULTILINGUAL_VOCABULARY],
);
static LARGE_V3: StaticPretrained<'static> = openai(
"large-v3",
&["large"],
"large-v3",
"1550 M parameters, multilingual, 128 mels",
&[&LARGE_V3_CHECKPOINT, &MULTILINGUAL_VOCABULARY],
);
static LARGE_V3_TURBO: StaticPretrained<'static> = openai(
"large-v3-turbo",
&["turbo"],
"large-v3-turbo",
"809 M parameters, multilingual, 128 mels, four-layer decoder",
&[&LARGE_V3_TURBO_CHECKPOINT, &MULTILINGUAL_VOCABULARY],
);
pub static OPENAI: StaticPretrainedGroup<'static> = StaticPretrainedGroup {
name: "openai",
description: "OpenAI's Whisper checkpoints, as `openai-whisper` names and pins them",
license: Some("MIT"),
origin: Some("https://github.com/openai/whisper/blob/main/whisper/__init__.py"),
items: &[
&TINY_EN,
&TINY,
&BASE_EN,
&BASE,
&SMALL_EN,
&SMALL,
&MEDIUM_EN,
&MEDIUM,
&LARGE_V1,
&LARGE_V2,
&LARGE_V3,
&LARGE_V3_TURBO,
],
};
pub static WELL_KNOWN_TABLE: StaticPretrainedTable<'static> = StaticPretrainedTable {
name: WELL_KNOWN,
description: "the Whisper checkpoints bunsen knows by name",
groups: &[&OPENAI],
};
#[cfg(feature = "whisper-weights")]
pub fn bundled_whisper_table() -> crate::data::pretrained::PretrainedTable {
use crate::{
data::pretrained::{
BUNDLED,
PretrainedGroup,
PretrainedTable,
},
kits::speech::whisper::pretrained::VOCABULARY,
};
let mut base = BASE.to_pretrained();
base.resources = base
.resources
.with_local_files(
BUNDLED,
&[
(CHECKPOINT, bunsen_bundled_whisper::base_pt()),
(VOCABULARY, bunsen_bundled_whisper::multilingual_tiktoken()),
],
)
.unwrap_or_else(|e| panic!("{e}"));
PretrainedTable {
name: BUNDLED.to_string(),
description: "the Whisper assets bunsen-bundled-whisper ships, used in place".to_string(),
groups: vec![PretrainedGroup {
name: OPENAI.name.to_string(),
description: OPENAI.description.to_string(),
license: OPENAI.license.map(str::to_string),
origin: OPENAI.origin.map(str::to_string),
items: vec![base],
}],
}
}
pub fn default_whisper_providers() -> Vec<Arc<dyn PretrainedProvider>> {
let mut providers: Vec<Arc<dyn PretrainedProvider>> =
vec![Arc::new(WELL_KNOWN_TABLE.to_table())];
#[cfg(feature = "whisper-weights")]
providers.push(Arc::new(bundled_whisper_table()));
providers.push(Arc::new(HfProvider::new().with_checkpoint_key(CHECKPOINT)));
providers
}
#[cfg(test)]
mod tests {
use super::*;
use crate::kits::speech::whisper::{
driver::WhisperSpecialIds,
pretrained::{
CHECKPOINT,
VOCABULARY,
WHISPER_PREFABS,
WhisperVocabulary,
},
};
fn layout_of(prefab: &str) -> WhisperSpecialIds {
let cfg = WHISPER_PREFABS.expect_lookup_prefab(prefab).to_config();
*cfg.token_layout
.policy_for_vocab(cfg.vocab_size)
.unwrap()
.ids()
}
#[test]
fn test_every_row_names_a_prefab_and_fuses() {
for group in WELL_KNOWN_TABLE.groups {
for row in group.items {
let prefab = row.prefab.expect("every whisper row names a prefab");
assert!(
WHISPER_PREFABS.lookup_prefab(prefab).is_some(),
"{}: prefab {prefab:?} is not in WHISPER_PREFABS",
group.id(row),
);
let map = row.try_to_map().unwrap();
map.validate().unwrap();
assert_eq!(map.keys(), [CHECKPOINT, VOCABULARY], "{}", group.id(row));
let checkpoint = map.get(CHECKPOINT).unwrap();
assert_eq!(checkpoint.file, format!("{}.pt", row.name));
assert!(checkpoint.is_pinned());
assert!(map.get(VOCABULARY).unwrap().is_pinned());
}
}
assert_eq!(OPENAI.items.len(), 12);
}
#[test]
fn test_the_declared_vocabulary_is_the_rules() {
for row in OPENAI.items {
let declared = row.to_map();
let declared = declared.get(VOCABULARY).unwrap();
let rule = WhisperVocabulary::for_layout(&layout_of(row.prefab.unwrap()))
.map()
.to_map();
let rule = rule.get(VOCABULARY).unwrap();
assert_eq!(declared, rule, "{}", OPENAI.id(row));
}
}
#[test]
fn test_the_well_known_table_lists_the_openai_rows() {
let table = WELL_KNOWN_TABLE.to_table();
assert_eq!(table.name(), "well-known");
let ids = table.ids();
assert_eq!(ids.len(), 12);
assert_eq!(ids[0], "well-known:openai/tiny.en");
assert_eq!(ids[11], "well-known:openai/large-v3-turbo");
let name = |spec: &str| table.lookup(spec).unwrap().map(|p| p.name);
assert_eq!(name("openai/tiny.en").as_deref(), Some("openai/tiny.en"));
assert_eq!(
name("openai/turbo").as_deref(),
Some("openai/large-v3-turbo")
);
assert_eq!(name("turbo").as_deref(), Some("openai/large-v3-turbo"));
assert_eq!(name("large").as_deref(), Some("openai/large-v3"));
assert_eq!(
name("large-v3-turbo").as_deref(),
Some("openai/large-v3-turbo")
);
assert_eq!(name("openai/gigantic"), None);
assert_eq!(name("gigantic"), None);
}
#[test]
fn test_names_and_aliases_are_unique_within_a_group() {
for group in WELL_KNOWN_TABLE.groups {
let mut names: Vec<&str> = group
.items
.iter()
.flat_map(|p| std::iter::once(p.name).chain(p.aliases.iter().copied()))
.collect();
let n = names.len();
names.sort_unstable();
names.dedup();
assert_eq!(names.len(), n, "{}: a name is repeated", group.name);
}
}
}