use alloc::{
format,
string::{
String,
ToString,
},
vec::Vec,
};
use serde::{
Deserialize,
Serialize,
};
use super::{
PretrainedWeightsDescriptor,
StaticPretrainedWeightsDescriptor,
};
#[derive(Debug)]
pub struct StaticPretrainedProvider<'a> {
pub name: &'a str,
pub description: &'a str,
pub license: Option<&'a str>,
pub origin: Option<&'a str>,
pub items: &'a [&'a StaticPretrainedWeightsDescriptor<'a>],
}
impl<'a> StaticPretrainedProvider<'a> {
pub fn lookup(
&self,
name: &str,
) -> Option<&'a StaticPretrainedWeightsDescriptor<'a>> {
self.items.iter().copied().find(|d| d.matches(name))
}
pub fn id(
&self,
pretrained: &StaticPretrainedWeightsDescriptor<'_>,
) -> String {
format!("{}/{}", self.name, pretrained.name)
}
pub fn ids(&self) -> Vec<String> {
self.items.iter().map(|d| self.id(d)).collect()
}
pub fn for_prefab(
&self,
prefab: &str,
) -> Vec<&'a StaticPretrainedWeightsDescriptor<'a>> {
self.items
.iter()
.copied()
.filter(|d| d.prefab == prefab)
.collect()
}
pub fn to_provider(&self) -> PretrainedProvider {
PretrainedProvider {
name: self.name.to_string(),
description: self.description.to_string(),
license: self.license.map(str::to_string),
origin: self.origin.map(str::to_string),
items: self.items.iter().map(|d| d.to_descriptor()).collect(),
}
}
}
impl From<&StaticPretrainedProvider<'_>> for PretrainedProvider {
fn from(provider: &StaticPretrainedProvider<'_>) -> Self {
provider.to_provider()
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PretrainedProvider {
pub name: String,
pub description: String,
pub license: Option<String>,
pub origin: Option<String>,
pub items: Vec<PretrainedWeightsDescriptor>,
}
impl PretrainedProvider {
pub fn lookup(
&self,
name: &str,
) -> Option<&PretrainedWeightsDescriptor> {
self.items.iter().find(|d| d.matches(name))
}
pub fn id(
&self,
pretrained: &PretrainedWeightsDescriptor,
) -> String {
format!("{}/{}", self.name, pretrained.name)
}
pub fn ids(&self) -> Vec<String> {
self.items.iter().map(|d| self.id(d)).collect()
}
pub fn for_prefab(
&self,
prefab: &str,
) -> Vec<&PretrainedWeightsDescriptor> {
self.items.iter().filter(|d| d.prefab == prefab).collect()
}
}
pub fn provider<'a>(
providers: &[&'a StaticPretrainedProvider<'a>],
name: &str,
) -> Option<&'a StaticPretrainedProvider<'a>> {
providers.iter().copied().find(|p| p.name == name)
}
pub fn lookup_pretrained<'a>(
providers: &[&'a StaticPretrainedProvider<'a>],
provider: Option<&str>,
name: &str,
) -> Option<(
&'a StaticPretrainedProvider<'a>,
&'a StaticPretrainedWeightsDescriptor<'a>,
)> {
match provider {
Some(provider) => {
let provider = self::provider(providers, provider)?;
provider.lookup(name).map(|d| (provider, d))
}
None => {
let mut hits = providers
.iter()
.copied()
.filter_map(|provider| provider.lookup(name).map(|d| (provider, d)));
let first = hits.next()?;
match hits.next() {
Some(_) => None,
None => Some(first),
}
}
}
}
pub fn available_ids(providers: &[&StaticPretrainedProvider<'_>]) -> Vec<String> {
providers.iter().flat_map(|p| p.ids()).collect()
}
pub fn pretrained_for_prefab<'a>(
providers: &[&'a StaticPretrainedProvider<'a>],
prefab: &str,
) -> Vec<(
&'a StaticPretrainedProvider<'a>,
&'a StaticPretrainedWeightsDescriptor<'a>,
)> {
providers
.iter()
.copied()
.flat_map(|p| p.for_prefab(prefab).into_iter().map(move |d| (p, d)))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::data::pretrained::{
StaticWeightsSource,
WeightsFormat,
};
const fn entry(
name: &'static str,
aliases: &'static [&'static str],
prefab: &'static str,
sources: &'static [StaticWeightsSource<'static>],
) -> StaticPretrainedWeightsDescriptor<'static> {
StaticPretrainedWeightsDescriptor {
name,
description: "an entry",
license: None,
origin: None,
prefab,
aliases,
file: "w.pt",
sha256: None,
format: WeightsFormat::PYTORCH_F32,
sources,
}
}
static A_SMALL: StaticPretrainedWeightsDescriptor<'static> = entry(
"small",
&["s"],
"small",
&[StaticWeightsSource::Url("https://a.example/small.pt")],
);
static A_LARGE_V1: StaticPretrainedWeightsDescriptor<'static> = entry(
"large-v1",
&[],
"large",
&[StaticWeightsSource::Url("https://a.example/large-v1.pt")],
);
static A_LARGE_V2: StaticPretrainedWeightsDescriptor<'static> = entry(
"large-v2",
&["large"],
"large",
&[StaticWeightsSource::Url("https://a.example/large-v2.pt")],
);
static B_SMALL: StaticPretrainedWeightsDescriptor<'static> = entry(
"small",
&[],
"small",
&[StaticWeightsSource::Url("https://b.example/small.pt")],
);
static B_TINY: StaticPretrainedWeightsDescriptor<'static> = entry(
"tiny",
&[],
"tiny",
&[StaticWeightsSource::Url("https://b.example/tiny.pt")],
);
static A: StaticPretrainedProvider<'static> = StaticPretrainedProvider {
name: "a",
description: "provider a",
license: Some("MIT"),
origin: Some("https://a.example"),
items: &[&A_SMALL, &A_LARGE_V1, &A_LARGE_V2],
};
static B: StaticPretrainedProvider<'static> = StaticPretrainedProvider {
name: "b",
description: "provider b",
license: None,
origin: None,
items: &[&B_SMALL, &B_TINY],
};
static PROVIDERS: &[&StaticPretrainedProvider<'static>] = &[&A, &B];
#[test]
fn test_provider_lookups_and_ids() {
assert_eq!(A.lookup("s").map(|d| d.name), Some("small"));
assert_eq!(A.lookup("large").map(|d| d.name), Some("large-v2"));
assert!(A.lookup("tiny").is_none());
assert_eq!(A.id(&A_SMALL), "a/small");
assert_eq!(A.ids(), vec!["a/small", "a/large-v1", "a/large-v2"]);
assert_eq!(
A.for_prefab("large")
.iter()
.map(|d| d.name)
.collect::<Vec<_>>(),
vec!["large-v1", "large-v2"]
);
assert!(A.for_prefab("tiny").is_empty());
}
#[test]
fn test_lookup_pretrained_across_providers() {
assert!(provider(PROVIDERS, "b").is_some());
assert!(provider(PROVIDERS, "c").is_none());
let (p, d) = lookup_pretrained(PROVIDERS, Some("b"), "small").unwrap();
assert_eq!((p.name, d.name), ("b", "small"));
let (p, d) = lookup_pretrained(PROVIDERS, None, "tiny").unwrap();
assert_eq!((p.name, d.name), ("b", "tiny"));
let (p, d) = lookup_pretrained(PROVIDERS, None, "large").unwrap();
assert_eq!((p.name, d.name), ("a", "large-v2"));
assert!(
lookup_pretrained(PROVIDERS, None, "small").is_none(),
"both providers have a `small`"
);
assert!(lookup_pretrained(PROVIDERS, Some("c"), "small").is_none());
assert!(lookup_pretrained(PROVIDERS, None, "gigantic").is_none());
assert_eq!(
available_ids(PROVIDERS),
vec!["a/small", "a/large-v1", "a/large-v2", "b/small", "b/tiny"]
);
assert_eq!(
pretrained_for_prefab(PROVIDERS, "small")
.iter()
.map(|(p, d)| format!("{}/{}", p.name, d.name))
.collect::<Vec<_>>(),
vec!["a/small", "b/small"]
);
}
#[test]
fn test_owned_provider_matches_the_static_one() {
let owned = A.to_provider();
assert_eq!(owned, PretrainedProvider::from(&A));
assert_eq!(owned.name, "a");
assert_eq!(owned.license.as_deref(), Some("MIT"));
assert_eq!(owned.items.len(), 3);
assert_eq!(
owned.lookup("large").map(|d| d.name.as_str()),
Some("large-v2")
);
assert_eq!(owned.ids(), A.ids());
assert_eq!(owned.for_prefab("large").len(), 2);
assert_eq!(owned.id(&owned.items[0]), "a/small");
let json = serde_json::to_string(&owned).unwrap();
assert_eq!(
serde_json::from_str::<PretrainedProvider>(&json).unwrap(),
owned
);
}
}