use std::sync::Arc;
use burn::tensor::Device;
use crate::{
data::pretrained::{
Deferred,
LoadedResources,
PretrainedCache,
PretrainedFactory,
},
errors::BunsenResult,
kits::speech::whisper::{
WhisperApiConfig,
driver::WhisperBundle,
pretrained::{
WhisperConstruct,
default_whisper_providers,
},
},
};
pub fn default_whisper_factory() -> BunsenResult<PretrainedFactory<WhisperConstruct>> {
PretrainedFactory::new().with_providers(default_whisper_providers())
}
impl PretrainedFactory<WhisperConstruct> {
pub fn load_bundle(
&self,
spec: &str,
cache: &PretrainedCache,
device: &Device,
) -> BunsenResult<Arc<WhisperBundle>> {
Ok(self.load(spec, cache, device)?.handle)
}
}
impl Deferred<WhisperConstruct> {
pub fn scan(
&self,
loaded: &LoadedResources,
) -> BunsenResult<WhisperApiConfig> {
self.hook.scan(&self.model, loaded)
}
pub fn load_bundle(
&self,
cache: &PretrainedCache,
device: &Device,
) -> BunsenResult<Arc<WhisperBundle>> {
Ok(self.load(cache, device)?.handle)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "store_safetensors")]
use crate::kits::speech::whisper::pretrained::WhisperReader;
use crate::{
data::{
cache::BunsenDiskCacheOptions,
pretrained::{
PretrainedCacheOptions,
PretrainedRef,
ResourceMap,
WELL_KNOWN,
testing::ListsNothing,
},
},
errors::{
BunsenErrorKind,
testing::ErrorMatcher,
},
kits::speech::whisper::pretrained::{
CHECKPOINT,
OPENAI,
WHISPER_KIT,
openai_download_root,
},
};
fn offline_cache_in(dir: &std::path::Path) -> PretrainedCache {
PretrainedCache::new(
PretrainedCacheOptions::default()
.with_disk(
BunsenDiskCacheOptions::default()
.with_cache_dir(Some(dir.join("cache")))
.without_transfer_observers(),
)
.with_offline(true),
)
.unwrap()
}
#[test]
fn test_the_defaults_register() {
let factory = default_whisper_factory().unwrap();
assert_eq!(factory.kit(), WHISPER_KIT);
assert_eq!(factory.providers()[0].name(), "well-known");
assert_eq!(factory.provider(WELL_KNOWN).unwrap().name(), "well-known");
assert_eq!(factory.provider(WELL_KNOWN).unwrap().ids().len(), 12);
assert_eq!(factory.providers().last().unwrap().name(), "hf");
assert!(factory.provider("hf").unwrap().ids().is_empty());
if cfg!(feature = "whisper-weights") {
assert_eq!(factory.providers().len(), 3);
assert_eq!(factory.providers()[1].name(), "bundled");
assert_eq!(factory.ids().len(), 13);
} else {
assert_eq!(factory.providers().len(), 2);
assert_eq!(factory.ids().len(), 12);
}
let (provider, large) = factory.lookup("large").unwrap();
assert_eq!(provider, "well-known");
assert_eq!(large.name, "openai/large-v3");
let (_, turbo) = factory.lookup("openai/turbo").unwrap();
assert_eq!(turbo.name, "openai/large-v3-turbo");
assert_eq!(
factory.lookup("well-known:openai/tiny").unwrap().1.name,
"openai/tiny"
);
ErrorMatcher::kind(BunsenErrorKind::Lookup).assert_err(&factory.lookup("nobody:base"));
ErrorMatcher::kind(BunsenErrorKind::Lookup).assert_err(&factory.lookup("gigantic"));
assert!(openai_download_root().is_some_and(|d| d.ends_with("whisper")));
}
#[test]
fn test_resolve_names_and_aliases() {
let factory = default_whisper_factory().unwrap();
let dir = tempfile::tempdir().unwrap();
let cache = offline_cache_in(dir.path());
let model = factory.resolve("openai/tiny.en", &cache).unwrap();
match &model.model {
PretrainedRef::Named {
provider,
pretrained,
} => {
assert_eq!(provider, "well-known");
assert_eq!(pretrained.name, "openai/tiny.en");
assert_eq!(pretrained.resources.keys(), [CHECKPOINT, "vocabulary"]);
}
other => panic!("{other:?}"),
}
assert!(format!("{:?}", model.hook).contains("WhisperConstruct"));
assert_eq!(
factory
.resolve("well-known:openai/tiny.en", &cache)
.unwrap()
.id(),
"well-known:openai/tiny.en"
);
assert_eq!(
factory.resolve("turbo", &cache).unwrap().id(),
"well-known:openai/large-v3-turbo"
);
let file = dir.path().join("ckpt.pt");
std::fs::write(&file, b"x").unwrap();
let spec = file.to_str().unwrap();
ErrorMatcher::kind(BunsenErrorKind::Lookup).assert_err(&factory.resolve(spec, &cache));
let given =
Deferred::<WhisperConstruct>::from_map(ResourceMap::given(spec, CHECKPOINT, &file))
.unwrap();
assert_eq!(given.id(), spec);
assert!(given.model.named().is_none());
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.assert_err(&factory.resolve("openai/gigantic", &cache));
ErrorMatcher::kind(BunsenErrorKind::Policy)
.display_contains("hf:openai/whisper-tiny")
.assert_err(&factory.resolve("hf:openai/whisper-tiny", &cache));
ErrorMatcher::kind(BunsenErrorKind::Policy)
.assert_err(&factory.lookup("hf:openai/whisper-tiny"));
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.assert_err(&factory.lookup("openai/whisper-tiny"));
ErrorMatcher::kind(BunsenErrorKind::Illegal).assert_err(&factory.lookup("hf:whisper-tiny"));
}
#[test]
fn test_one_prefab_many_pretrained() {
let large: Vec<&str> = OPENAI.for_prefab("large").iter().map(|p| p.name).collect();
assert_eq!(large, ["large-v1", "large-v2"]);
let derived: Vec<String> = default_whisper_factory()
.unwrap()
.for_prefab("large")
.iter()
.map(|(p, row)| format!("{p}:{}", row.name))
.collect();
assert_eq!(
derived,
["well-known:openai/large-v1", "well-known:openai/large-v2"]
);
}
#[test]
fn test_a_plugin_joins_the_defaults() {
let hub = Arc::new(ListsNothing::default());
let defaults = default_whisper_providers().len();
let factory = default_whisper_factory()
.unwrap()
.with_provider(hub.clone())
.unwrap();
assert_eq!(factory.providers().len(), defaults + 1);
assert_eq!(
factory.ids().len(),
factory.providers()[..defaults]
.iter()
.map(|p| p.ids().len())
.sum::<usize>()
);
let (provider, row) = factory.lookup("hub:openai/whisper-base").unwrap();
assert_eq!(provider, "hub");
assert_eq!(row.name, "openai/whisper-base");
assert_eq!(factory.lookup("base").unwrap().0, "well-known");
assert_eq!(hub.lookups(), 1, "a bare name never reached the hub");
let dir = tempfile::tempdir().unwrap();
let resolved = factory.resolve("hub:openai/whisper-base", &offline_cache_in(dir.path()));
#[cfg(feature = "store_safetensors")]
assert!(matches!(
resolved.unwrap().hook.reader,
WhisperReader::Safetensors(_)
));
#[cfg(not(feature = "store_safetensors"))]
ErrorMatcher::kind(BunsenErrorKind::Unsupported)
.message_contains("store_safetensors")
.assert_err(&resolved);
}
#[test]
fn test_the_defaults_have_no_duplicate() {
ErrorMatcher::kind(BunsenErrorKind::Illegal).assert_err(
&default_whisper_factory()
.unwrap()
.with_providers(default_whisper_providers()),
);
}
#[cfg(feature = "whisper-weights")]
#[test]
fn test_the_bundled_provider_serves_openai_base_in_place() {
use crate::{
data::pretrained::{
BUNDLED,
CacheStatus,
Provenance,
},
kits::speech::whisper::pretrained::{
BASE_CHECKPOINT,
VOCABULARY,
},
};
let dir = tempfile::tempdir().unwrap();
let cache = offline_cache_in(dir.path());
let factory = default_whisper_factory().unwrap();
let bundled = factory.provider(BUNDLED).unwrap();
assert_eq!(bundled.ids(), ["bundled:openai/base"]);
assert_eq!(bundled.license(), None);
let model = factory.resolve("bundled:openai/base", &cache).unwrap();
assert_eq!(model.id(), "bundled:openai/base");
let status = model.status(&cache);
assert_eq!(status[CHECKPOINT], CacheStatus::LocalDir);
assert_eq!(status[VOCABULARY], CacheStatus::LocalDir);
let map = model.to_map();
assert_eq!(
map.get(CHECKPOINT).unwrap().sha256,
BASE_CHECKPOINT.to_map().get(CHECKPOINT).unwrap().sha256,
"the same file, pinned the same"
);
let loaded = cache.load(WHISPER_KIT, &map).unwrap();
for (key, part) in loaded.iter() {
assert_eq!(part.provenance, Provenance::LocalDir, "{key}");
}
assert_eq!(
loaded.expect(CHECKPOINT).unwrap(),
bunsen_bundled_whisper::base_pt()
);
assert_eq!(
loaded.expect(VOCABULARY).unwrap(),
bunsen_bundled_whisper::multilingual_tiktoken()
);
assert!(!dir.path().join("cache").exists(), "nothing was written");
assert_eq!(
factory.resolve("openai/base", &cache).unwrap().id(),
"well-known:openai/base"
);
}
#[cfg(feature = "whisper-weights")]
#[test]
fn test_the_bundle_serves_openai_base() {
use crate::{
data::pretrained::{
CacheStatus,
Provenance,
},
kits::speech::whisper::pretrained::{
VOCABULARY,
testing::offline_cache,
},
};
let cache = offline_cache();
let factory = default_whisper_factory().unwrap();
let model = factory.resolve("openai/base", &cache).unwrap();
assert_eq!(model.id(), "well-known:openai/base");
let status = model.status(&cache);
assert_eq!(status[CHECKPOINT], CacheStatus::Cached);
assert_eq!(status[VOCABULARY], CacheStatus::Cached);
let loaded = cache.load(WHISPER_KIT, &model.to_map()).unwrap();
assert_eq!(loaded.map.name, "well-known:openai/base");
for (key, part) in loaded.iter() {
assert_eq!(part.provenance, Provenance::Cached, "{key}");
assert!(part.path.starts_with(bunsen_bundled_whisper::cache_dir()));
}
assert_eq!(
loaded.expect(CHECKPOINT).unwrap(),
bunsen_bundled_whisper::base_pt()
);
}
}
#[cfg(all(test, feature = "store_safetensors", feature = "fetch"))]
mod hub_tests {
use burn::tensor::{
Tensor,
Tolerance,
};
use crate::{
data::pretrained::{
PretrainedCache,
PretrainedCacheOptions,
Provenance,
},
kits::speech::whisper::{
WhisperMeta,
pretrained::{
CHECKPOINT,
WHISPER_PREFABS,
default_whisper_factory,
},
},
support::testing::{
assert_tensors_close,
cpu_device,
},
};
#[test]
fn test_hf_whisper_tiny_is_openai_tiny() {
let device = cpu_device();
let cache = PretrainedCache::new(PretrainedCacheOptions::default()).unwrap();
let factory = default_whisper_factory().unwrap();
let model = factory.resolve("hf:openai/whisper-tiny", &cache).unwrap();
let checkpoint = model.to_map();
let checkpoint = checkpoint.get(CHECKPOINT).unwrap();
assert_eq!(checkpoint.file, "model.safetensors");
assert!(checkpoint.is_pinned(), "pinned by the hub's listing");
let loaded = model.load(&cache, &device).unwrap();
assert!(matches!(
loaded.resources.get(CHECKPOINT).unwrap().provenance,
Provenance::Cached | Provenance::Downloaded
));
let hf = loaded.handle;
let tiny = WHISPER_PREFABS
.expect_lookup_prefab("tiny")
.to_config()
.geometry();
assert_eq!(hf.model.n_mels(), tiny.n_mels);
assert_eq!(hf.model.vocab_size(), tiny.vocab_size);
assert_eq!(hf.model.d_model(), tiny.d_model);
assert_eq!(hf.model.max_audio_ctx(), tiny.max_audio_ctx);
assert_eq!(hf.model.max_text_ctx(), tiny.max_text_ctx);
assert_eq!(hf.model.encoder.blocks.len(), tiny.n_encoder_layers);
assert_eq!(hf.model.decoder.blocks.len(), tiny.n_decoder_layers);
assert!(hf.layout.ids().is_multilingual());
assert_eq!(hf.ranks.as_ref().map(|r| r.len()), Some(50257));
let openai = factory.load_bundle("openai/tiny", &cache, &device).unwrap();
let close = |a: Tensor<2>, b: Tensor<2>| {
assert_tensors_close(&a, &b, Tolerance::<f32>::default());
};
close(
hf.model.encoder.blocks[0].attn.query.weight.val(),
openai.model.encoder.blocks[0].attn.query.weight.val(),
);
close(
hf.model.decoder.blocks[3].cross_attn.output.weight.val(),
openai.model.decoder.blocks[3]
.cross_attn
.output
.weight
.val(),
);
close(
hf.model.decoder.blocks[1].mlp.linear2.weight.val(),
openai.model.decoder.blocks[1].mlp.linear2.weight.val(),
);
close(
hf.model.encoder.positional_embedding.val(),
openai.model.encoder.positional_embedding.val(),
);
close(
hf.model.decoder.token_embedding.weight.val(),
openai.model.decoder.token_embedding.weight.val(),
);
close(
hf.model.decoder.ln.gamma.val().unsqueeze(),
openai.model.decoder.ln.gamma.val().unsqueeze(),
);
}
}