#![forbid(unsafe_code)]
use std::collections::{HashMap, HashSet};
use std::ffi::{OsStr, OsString};
use std::fmt;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::time::SystemTime;
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum RepoType {
Model,
Dataset,
Space,
Kernel,
}
impl RepoType {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Model => "model",
Self::Dataset => "dataset",
Self::Space => "space",
Self::Kernel => "kernel",
}
}
const fn directory_prefix(self) -> &'static str {
match self {
Self::Model => "models",
Self::Dataset => "datasets",
Self::Space => "spaces",
Self::Kernel => "kernels",
}
}
fn from_directory_prefix(prefix: &str) -> Option<Self> {
match prefix {
"models" => Some(Self::Model),
"datasets" => Some(Self::Dataset),
"spaces" => Some(Self::Space),
"kernels" => Some(Self::Kernel),
_ => None,
}
}
}
impl fmt::Display for RepoType {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CachedFile {
pub relative_path: PathBuf,
pub snapshot_path: PathBuf,
pub blob_path: PathBuf,
pub size_on_disk: u64,
pub last_accessed: SystemTime,
pub last_modified: SystemTime,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CachedRevision {
pub commit_hash: String,
pub snapshot_path: PathBuf,
pub files: Vec<CachedFile>,
pub refs: Vec<String>,
pub size_on_disk: u64,
pub last_modified: SystemTime,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CachedRepo {
pub repo_id: String,
pub repo_type: RepoType,
pub repo_path: PathBuf,
pub revisions: Vec<CachedRevision>,
pub blob_count: usize,
pub size_on_disk: u64,
pub last_accessed: SystemTime,
pub last_modified: SystemTime,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ScanWarning {
pub path: PathBuf,
pub message: String,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CacheInfo {
pub cache_dir: PathBuf,
pub repos: Vec<CachedRepo>,
pub size_on_disk: u64,
pub warnings: Vec<ScanWarning>,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RepoScan {
pub cache_dir: PathBuf,
pub repo: Option<CachedRepo>,
pub warnings: Vec<ScanWarning>,
}
#[non_exhaustive]
#[derive(Debug)]
pub enum Error {
Io {
path: PathBuf,
source: io::Error,
},
InvalidRepoId {
repo_id: String,
},
}
impl fmt::Display for Error {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Io { path, source } => {
write!(formatter, "failed to inspect {}: {source}", path.display())
}
Self::InvalidRepoId { repo_id } => {
write!(
formatter,
"invalid Hugging Face repository identifier {repo_id:?}"
)
}
}
}
}
impl std::error::Error for Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Io { source, .. } => Some(source),
Self::InvalidRepoId { .. } => None,
}
}
}
#[must_use]
pub fn resolve_cache_dir() -> PathBuf {
resolve_cache_dir_with(|name| std::env::var_os(name), &std::env::temp_dir())
}
pub fn scan_default_cache() -> Result<CacheInfo, Error> {
scan_cache(resolve_cache_dir())
}
pub fn scan_cache(cache_dir: impl AsRef<Path>) -> Result<CacheInfo, Error> {
let cache_dir = cache_dir.as_ref().to_path_buf();
let entries = match fs::read_dir(&cache_dir) {
Ok(entries) => entries,
Err(source) if source.kind() == io::ErrorKind::NotFound => {
return Ok(CacheInfo {
cache_dir,
repos: Vec::new(),
size_on_disk: 0,
warnings: Vec::new(),
});
}
Err(source) => {
return Err(Error::Io {
path: cache_dir,
source,
});
}
};
let mut warnings = Vec::new();
let mut repo_entries = collect_entries(entries, &cache_dir, &mut warnings);
repo_entries.sort_by_key(fs::DirEntry::file_name);
let mut repos = Vec::new();
for entry in repo_entries {
if !entry_is_directory(&entry, &mut warnings) {
continue;
}
let Some((repo_type, repo_id)) = parse_repo_folder(&entry.file_name()) else {
continue;
};
repos.push(scan_repo_path(
entry.path(),
repo_type,
repo_id,
&mut warnings,
));
}
repos.sort_by(|left, right| {
left.repo_type
.cmp(&right.repo_type)
.then_with(|| left.repo_id.cmp(&right.repo_id))
});
let size_on_disk = repos
.iter()
.fold(0_u64, |total, repo| total.saturating_add(repo.size_on_disk));
sort_warnings(&mut warnings);
Ok(CacheInfo {
cache_dir,
repos,
size_on_disk,
warnings,
})
}
pub fn scan_repo(
cache_dir: impl AsRef<Path>,
repo_type: RepoType,
repo_id: &str,
) -> Result<RepoScan, Error> {
validate_repo_id(repo_id)?;
let cache_dir = cache_dir.as_ref().to_path_buf();
let repo_path = cache_dir.join(repo_folder_name(repo_type, repo_id));
match fs::metadata(&repo_path) {
Ok(metadata) if metadata.is_dir() => {}
Ok(_) => {
return Ok(RepoScan {
cache_dir,
repo: None,
warnings: vec![ScanWarning {
path: repo_path,
message: "repository cache entry is not a directory".to_owned(),
}],
});
}
Err(source) if source.kind() == io::ErrorKind::NotFound => {
return Ok(RepoScan {
cache_dir,
repo: None,
warnings: Vec::new(),
});
}
Err(source) => {
return Err(Error::Io {
path: repo_path,
source,
});
}
}
let mut warnings = Vec::new();
let repo = scan_repo_path(repo_path, repo_type, repo_id.to_owned(), &mut warnings);
sort_warnings(&mut warnings);
Ok(RepoScan {
cache_dir,
repo: Some(repo),
warnings,
})
}
fn resolve_cache_dir_with(
mut read_env: impl FnMut(&str) -> Option<OsString>,
temporary_dir: &Path,
) -> PathBuf {
if let Some(path) = read_env("HF_HUB_CACHE") {
return PathBuf::from(path);
}
if let Some(path) = read_env("HUGGINGFACE_HUB_CACHE") {
return PathBuf::from(path);
}
if let Some(path) = read_env("HF_HOME") {
return PathBuf::from(path).join("hub");
}
if let Some(path) = read_env("XDG_CACHE_HOME") {
return PathBuf::from(path).join("huggingface").join("hub");
}
if let Some(path) = read_env("HOME").or_else(|| read_env("USERPROFILE")) {
return PathBuf::from(path)
.join(".cache")
.join("huggingface")
.join("hub");
}
temporary_dir.join("huggingface").join("hub")
}
fn validate_repo_id(repo_id: &str) -> Result<(), Error> {
let mut component_count = 0_usize;
for component in repo_id.split('/') {
component_count += 1;
if component.is_empty()
|| matches!(component, "." | "..")
|| component.contains('\\')
|| component.contains("--")
{
return Err(Error::InvalidRepoId {
repo_id: repo_id.to_owned(),
});
}
}
if component_count > 2 {
return Err(Error::InvalidRepoId {
repo_id: repo_id.to_owned(),
});
}
Ok(())
}
fn repo_folder_name(repo_type: RepoType, repo_id: &str) -> String {
format!(
"{}--{}",
repo_type.directory_prefix(),
repo_id.replace('/', "--")
)
}
fn parse_repo_folder(name: &OsStr) -> Option<(RepoType, String)> {
let name = name.to_str()?;
let (prefix, encoded_id) = name.split_once("--")?;
let repo_type = RepoType::from_directory_prefix(prefix)?;
if encoded_id.is_empty() {
return None;
}
Some((repo_type, encoded_id.replace("--", "/")))
}
fn scan_repo_path(
repo_path: PathBuf,
repo_type: RepoType,
repo_id: String,
warnings: &mut Vec<ScanWarning>,
) -> CachedRepo {
let commit_refs = read_refs(&repo_path.join("refs"), warnings);
let mut revisions = Vec::new();
let snapshots_path = repo_path.join("snapshots");
for entry in read_directory_or_warn(&snapshots_path, warnings) {
if !entry_is_directory(&entry, warnings) {
continue;
}
let snapshot_path = entry.path();
let commit_hash = entry.file_name().to_string_lossy().into_owned();
let mut files = scan_snapshot(&snapshot_path, warnings);
files.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
let size_on_disk = files
.iter()
.fold(0_u64, |total, file| total.saturating_add(file.size_on_disk));
let last_modified = files
.iter()
.map(|file| file.last_modified)
.max()
.unwrap_or(SystemTime::UNIX_EPOCH);
let mut refs = commit_refs.get(&commit_hash).cloned().unwrap_or_default();
refs.sort();
revisions.push(CachedRevision {
commit_hash,
snapshot_path,
files,
refs,
size_on_disk,
last_modified,
});
}
revisions.sort_by(|left, right| left.commit_hash.cmp(&right.commit_hash));
let mut blobs = HashMap::<PathBuf, u64>::new();
let mut last_accessed = SystemTime::UNIX_EPOCH;
let mut last_modified = SystemTime::UNIX_EPOCH;
for file in revisions.iter().flat_map(|revision| &revision.files) {
blobs
.entry(file.blob_path.clone())
.or_insert(file.size_on_disk);
last_accessed = last_accessed.max(file.last_accessed);
last_modified = last_modified.max(file.last_modified);
}
let size_on_disk = blobs
.values()
.fold(0_u64, |total, size| total.saturating_add(*size));
CachedRepo {
repo_id,
repo_type,
repo_path,
revisions,
blob_count: blobs.len(),
size_on_disk,
last_accessed,
last_modified,
}
}
fn read_refs(refs_path: &Path, warnings: &mut Vec<ScanWarning>) -> HashMap<String, Vec<String>> {
let mut refs = HashMap::<String, Vec<String>>::new();
for path in walk_files(refs_path, warnings) {
let commit_hash = match fs::read_to_string(&path) {
Ok(contents) => contents.trim().to_owned(),
Err(error) => {
warn(warnings, &path, error.to_string());
continue;
}
};
if commit_hash.is_empty() {
warn(warnings, &path, "ref file is empty");
continue;
}
let name = path
.strip_prefix(refs_path)
.unwrap_or(&path)
.to_string_lossy()
.replace(std::path::MAIN_SEPARATOR, "/");
refs.entry(commit_hash).or_default().push(name);
}
refs
}
fn scan_snapshot(snapshot_path: &Path, warnings: &mut Vec<ScanWarning>) -> Vec<CachedFile> {
let mut files = Vec::new();
for path in walk_files(snapshot_path, warnings) {
let blob_path = match fs::canonicalize(&path) {
Ok(path) => path,
Err(error) => {
warn(warnings, &path, error.to_string());
continue;
}
};
let metadata = match fs::metadata(&blob_path) {
Ok(metadata) => metadata,
Err(error) => {
warn(warnings, &path, error.to_string());
continue;
}
};
if !metadata.is_file() {
warn(warnings, &path, "snapshot entry does not resolve to a file");
continue;
}
files.push(CachedFile {
relative_path: path
.strip_prefix(snapshot_path)
.unwrap_or(&path)
.to_path_buf(),
snapshot_path: path,
blob_path,
size_on_disk: metadata.len(),
last_accessed: metadata.accessed().unwrap_or(SystemTime::UNIX_EPOCH),
last_modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
});
}
files
}
fn walk_files(path: &Path, warnings: &mut Vec<ScanWarning>) -> Vec<PathBuf> {
let mut files = Vec::new();
let mut pending = vec![path.to_path_buf()];
let mut visited_directories = HashSet::new();
while let Some(directory) = pending.pop() {
let canonical_directory = match fs::canonicalize(&directory) {
Ok(path) => path,
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
Err(error) => {
warn(warnings, &directory, error.to_string());
continue;
}
};
if !visited_directories.insert(canonical_directory) {
warn(warnings, &directory, "directory cycle ignored");
continue;
}
let mut entries = read_directory_or_warn(&directory, warnings);
entries.reverse();
for entry in entries {
let file_type = match entry.file_type() {
Ok(file_type) => file_type,
Err(error) => {
warn(warnings, entry.path(), error.to_string());
continue;
}
};
if file_type.is_dir() {
pending.push(entry.path());
} else {
files.push(entry.path());
}
}
}
files
}
fn read_directory_or_warn(path: &Path, warnings: &mut Vec<ScanWarning>) -> Vec<fs::DirEntry> {
match fs::read_dir(path) {
Ok(entries) => {
let mut entries = collect_entries(entries, path, warnings);
entries.sort_by_key(fs::DirEntry::file_name);
entries
}
Err(error) if error.kind() == io::ErrorKind::NotFound => Vec::new(),
Err(error) => {
warn(warnings, path, error.to_string());
Vec::new()
}
}
}
fn collect_entries(
entries: fs::ReadDir,
path: &Path,
warnings: &mut Vec<ScanWarning>,
) -> Vec<fs::DirEntry> {
entries
.filter_map(|entry| match entry {
Ok(entry) => Some(entry),
Err(error) => {
warn(warnings, path, error.to_string());
None
}
})
.collect()
}
fn entry_is_directory(entry: &fs::DirEntry, warnings: &mut Vec<ScanWarning>) -> bool {
match entry.file_type() {
Ok(file_type) => file_type.is_dir(),
Err(error) => {
warn(warnings, entry.path(), error.to_string());
false
}
}
}
fn warn(warnings: &mut Vec<ScanWarning>, path: impl AsRef<Path>, message: impl Into<String>) {
warnings.push(ScanWarning {
path: path.as_ref().to_path_buf(),
message: message.into(),
});
}
fn sort_warnings(warnings: &mut [ScanWarning]) {
warnings.sort_by(|left, right| {
left.path
.cmp(&right.path)
.then_with(|| left.message.cmp(&right.message))
});
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::ffi::OsString;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use super::{resolve_cache_dir_with, scan_cache, scan_repo, Error, RepoType};
static NEXT_TEMPORARY_DIRECTORY: AtomicU64 = AtomicU64::new(0);
struct TestDir(PathBuf);
impl TestDir {
fn new() -> Self {
loop {
let sequence = NEXT_TEMPORARY_DIRECTORY.fetch_add(1, Ordering::Relaxed);
let path = std::env::temp_dir().join(format!(
"hf-cache-reader-tests-{}-{sequence}",
std::process::id()
));
match fs::create_dir(&path) {
Ok(()) => return Self(path),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(error) => panic!("failed to create test directory: {error}"),
}
}
}
fn path(&self) -> &Path {
&self.0
}
}
impl Drop for TestDir {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
#[test]
fn cache_resolution_obeys_environment_precedence() {
let values = HashMap::from([
("HF_HUB_CACHE", OsString::from("/explicit")),
("HF_HOME", OsString::from("/home")),
("HOME", OsString::from("/user")),
]);
assert_eq!(
resolve_cache_dir_with(|name| values.get(name).cloned(), Path::new("/temporary")),
Path::new("/explicit")
);
let values = HashMap::from([("HF_HOME", OsString::from("/home"))]);
assert_eq!(
resolve_cache_dir_with(|name| values.get(name).cloned(), Path::new("/temporary")),
Path::new("/home/hub")
);
assert_eq!(
resolve_cache_dir_with(|_| None, Path::new("/temporary")),
Path::new("/temporary/huggingface/hub")
);
}
#[test]
fn missing_cache_is_empty() {
let temporary = TestDir::new();
let missing = temporary.path().join("missing");
let cache = scan_cache(&missing).expect("scan missing cache");
assert_eq!(cache.cache_dir, missing);
assert!(cache.repos.is_empty());
assert!(cache.warnings.is_empty());
}
#[test]
fn scans_revisions_nested_refs_and_nested_files() {
let temporary = TestDir::new();
let cache_dir = temporary.path().join("hub");
let repo_dir = cache_dir.join("models--acme--tiny-model");
write_ref(&repo_dir, "main", "commit-a");
write_ref(&repo_dir, "refs/pr/7", "commit-a");
write_cached_file(
&repo_dir,
"commit-a",
"weights/model-Q4_K_M.gguf",
"blob-a",
b"weights",
);
let cache = scan_cache(&cache_dir).expect("scan cache");
assert!(cache.warnings.is_empty(), "{:?}", cache.warnings);
assert_eq!(cache.repos.len(), 1);
let repo = &cache.repos[0];
assert_eq!(repo.repo_type, RepoType::Model);
assert_eq!(repo.repo_id, "acme/tiny-model");
assert_eq!(repo.revisions.len(), 1);
let revision = &repo.revisions[0];
assert_eq!(revision.commit_hash, "commit-a");
assert_eq!(
revision.refs,
vec!["main".to_owned(), "refs/pr/7".to_owned()]
);
assert_eq!(revision.files.len(), 1);
assert_eq!(
revision.files[0].relative_path,
Path::new("weights/model-Q4_K_M.gguf")
);
assert_eq!(revision.files[0].size_on_disk, 7);
}
#[test]
fn targeted_scan_ignores_unrelated_repositories() {
let temporary = TestDir::new();
let cache_dir = temporary.path().join("hub");
let wanted = cache_dir.join("models--acme--wanted");
let other = cache_dir.join("models--acme--other");
write_cached_file(&wanted, "wanted-rev", "model.gguf", "wanted", b"wanted");
write_cached_file(&other, "other-rev", "model.gguf", "other", b"other");
let scan = scan_repo(&cache_dir, RepoType::Model, "acme/wanted")
.expect("scan requested repository");
assert!(scan.warnings.is_empty());
let repo = scan.repo.expect("cached repository");
assert_eq!(repo.repo_id, "acme/wanted");
assert_eq!(repo.revisions[0].commit_hash, "wanted-rev");
let missing = scan_repo(&cache_dir, RepoType::Model, "acme/missing")
.expect("scan missing repository");
assert!(missing.repo.is_none());
}
#[test]
fn unsafe_or_ambiguous_repository_ids_are_rejected() {
let temporary = TestDir::new();
for repo_id in ["", "../escape", "a/b/c", "a\\b", "owner/repo--name"] {
assert!(matches!(
scan_repo(temporary.path(), RepoType::Model, repo_id),
Err(Error::InvalidRepoId { .. })
));
}
}
#[cfg(unix)]
#[test]
fn shared_blobs_are_counted_once() {
let temporary = TestDir::new();
let cache_dir = temporary.path().join("hub");
let repo_dir = cache_dir.join("models--acme--shared");
write_cached_file(&repo_dir, "one", "model.gguf", "shared", b"12345");
link_cached_file(&repo_dir, "two", "model.gguf", "shared");
let cache = scan_cache(&cache_dir).expect("scan cache");
let repo = &cache.repos[0];
assert_eq!(repo.revisions.len(), 2);
assert_eq!(repo.blob_count, 1);
assert_eq!(repo.size_on_disk, 5);
}
#[cfg(unix)]
#[test]
fn dangling_snapshot_links_are_warnings() {
use std::os::unix::fs::symlink;
let temporary = TestDir::new();
let cache_dir = temporary.path().join("hub");
let snapshot = cache_dir
.join("models--acme--broken")
.join("snapshots/revision");
fs::create_dir_all(&snapshot).expect("snapshot directory");
symlink("../../blobs/missing", snapshot.join("model.gguf"))
.expect("dangling snapshot link");
let cache = scan_cache(&cache_dir).expect("scan cache");
assert_eq!(cache.repos[0].revisions[0].files.len(), 0);
assert_eq!(cache.warnings.len(), 1);
assert!(cache.warnings[0].message.contains("No such file"));
}
fn write_ref(repo_dir: &Path, name: &str, commit: &str) {
let path = repo_dir.join("refs").join(name);
fs::create_dir_all(path.parent().expect("ref parent")).expect("create ref parent");
fs::write(path, commit).expect("write ref");
}
fn write_cached_file(
repo_dir: &Path,
revision: &str,
relative_path: &str,
blob_name: &str,
contents: &[u8],
) {
let blob = repo_dir.join("blobs").join(blob_name);
fs::create_dir_all(blob.parent().expect("blob parent")).expect("create blob parent");
fs::write(&blob, contents).expect("write blob");
link_cached_file(repo_dir, revision, relative_path, blob_name);
}
fn link_cached_file(repo_dir: &Path, revision: &str, relative_path: &str, blob_name: &str) {
let blob = repo_dir.join("blobs").join(blob_name);
let pointer = repo_dir
.join("snapshots")
.join(revision)
.join(relative_path);
fs::create_dir_all(pointer.parent().expect("snapshot parent"))
.expect("create snapshot parent");
create_pointer(&blob, &pointer);
}
#[cfg(unix)]
fn create_pointer(blob: &Path, pointer: &Path) {
std::os::unix::fs::symlink(blob, pointer).expect("create snapshot symlink");
}
#[cfg(windows)]
fn create_pointer(blob: &Path, pointer: &Path) {
fs::copy(blob, pointer).expect("copy snapshot file");
}
}