use alloc::{
format,
string::String,
};
use serde::{
Deserialize,
Serialize,
};
use crate::data::resources::is_sha256_hex;
pub const HUB_ORIGIN: &str = "https://huggingface.co";
pub const HUB_KIT: &str = "hub";
pub const HF_MIRROR_CONTRACT: &str = include_str!("hf_mirror.yml");
pub const HF_MIRROR_CATALOG_CONTRACT: &str = "hf-mirror";
pub const HF_MIRROR_ORIGIN: &str = "origin";
pub const HUB_MAIN: &str = "main";
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct HubRepo {
pub repo: String,
pub revision: String,
pub dataset: bool,
}
impl HubRepo {
pub fn new(
repo: impl Into<String>,
revision: impl Into<String>,
) -> Self {
Self {
repo: repo.into(),
revision: revision.into(),
dataset: false,
}
}
pub fn dataset(mut self) -> Self {
self.dataset = true;
self
}
fn kind_prefix(&self) -> &'static str {
if self.dataset { "datasets/" } else { "" }
}
pub fn file_url(
&self,
file: &str,
) -> String {
self.file_url_at(HUB_ORIGIN, file)
}
pub fn file_url_at(
&self,
origin: &str,
file: &str,
) -> String {
format!(
"{origin}/{}{}/resolve/{}/{file}",
self.kind_prefix(),
self.repo,
self.revision
)
}
pub fn listing_url(&self) -> String {
self.listing_url_at(HUB_ORIGIN)
}
pub fn listing_url_at(
&self,
origin: &str,
) -> String {
format!("{}?recursive=true", self.tree_url_at(origin))
}
pub fn tree_url_at(
&self,
origin: &str,
) -> String {
let api = if self.dataset { "datasets" } else { "models" };
format!("{origin}/api/{api}/{}/tree/{}", self.repo, self.revision)
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct HubEntry {
#[serde(rename = "type")]
pub kind: String,
pub path: String,
#[serde(default)]
pub size: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub lfs: Option<HubLfs>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct HubLfs {
pub oid: String,
#[serde(default)]
pub size: u64,
}
impl HubEntry {
pub fn is_file(&self) -> bool {
self.kind == "file"
}
pub fn sha256(&self) -> Option<String> {
let oid = self.lfs.as_ref()?.oid.to_ascii_lowercase();
is_sha256_hex(&oid).then_some(oid)
}
pub fn file_size(&self) -> u64 {
self.lfs.as_ref().map_or(self.size, |l| l.size)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_urls() {
let repo = HubRepo::new("org/name", "abc");
assert_eq!(
repo.file_url("a/b.json"),
"https://huggingface.co/org/name/resolve/abc/a/b.json"
);
assert_eq!(
repo.listing_url(),
"https://huggingface.co/api/models/org/name/tree/abc?recursive=true"
);
assert_eq!(
repo.tree_url_at("http://h"),
"http://h/api/models/org/name/tree/abc"
);
let data = HubRepo::new("org/set", "abc").dataset();
assert_eq!(
data.file_url("shard_00000.parquet"),
"https://huggingface.co/datasets/org/set/resolve/abc/shard_00000.parquet"
);
assert_eq!(
data.listing_url(),
"https://huggingface.co/api/datasets/org/set/tree/abc?recursive=true"
);
}
#[test]
fn test_an_entry_pins_by_its_lfs_oid() {
let entry: HubEntry = serde_json::from_str(&format!(
r#"{{"type": "file", "path": "m.safetensors", "size": 134,
"lfs": {{"oid": "{}", "size": 7}}}}"#,
"AB".repeat(32)
))
.unwrap();
assert!(entry.is_file());
assert_eq!(entry.sha256(), Some("ab".repeat(32)));
assert_eq!(entry.file_size(), 7);
}
}