1#![deny(missing_docs)]
2#![cfg_attr(feature="ureq", doc = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/README.md")))]
3#![cfg_attr(
4 not(feature = "ureq"),
5 doc = "Documentation is meant to be compiled with default features (at least ureq)"
6)]
7use std::io::Write;
8use std::path::PathBuf;
9
10#[cfg(any(feature = "tokio", feature = "ureq"))]
12pub mod api;
13
14const HF_HOME: &str = "HF_HOME";
15
16#[derive(Debug, Clone, Copy)]
18pub enum RepoType {
19 Model,
22 Dataset,
24 Space,
26}
27
28#[derive(Clone, Debug)]
30pub struct Cache {
31 path: PathBuf,
32}
33
34impl Cache {
35 pub fn new(path: PathBuf) -> Self {
37 Self { path }
38 }
39
40 pub fn from_env() -> Self {
43 match std::env::var(HF_HOME) {
44 Ok(home) => {
45 let mut path: PathBuf = home.into();
46 path.push("hub");
47 Self::new(path)
48 }
49 Err(_) => Self::default(),
50 }
51 }
52
53 pub fn path(&self) -> &PathBuf {
55 &self.path
56 }
57
58 pub fn token_path(&self) -> PathBuf {
60 let mut path = self.path.clone();
61 path.pop();
63 path.push("token");
64 path
65 }
66
67 pub fn token(&self) -> Option<String> {
70 let token_filename = self.token_path();
71 if token_filename.exists() {
72 log::info!("Using token file found {token_filename:?}");
73 }
74 match std::fs::read_to_string(token_filename) {
75 Ok(token_content) => {
76 let token_content = token_content.trim();
77 if token_content.is_empty() {
78 None
79 } else {
80 Some(token_content.to_string())
81 }
82 }
83 Err(_) => None,
84 }
85 }
86
87 pub fn repo(&self, repo: Repo) -> CacheRepo {
90 CacheRepo::new(self.clone(), repo)
91 }
92
93 pub fn model(&self, model_id: String) -> CacheRepo {
101 self.repo(Repo::new(model_id, RepoType::Model))
102 }
103
104 pub fn dataset(&self, model_id: String) -> CacheRepo {
112 self.repo(Repo::new(model_id, RepoType::Dataset))
113 }
114
115 pub fn space(&self, model_id: String) -> CacheRepo {
123 self.repo(Repo::new(model_id, RepoType::Space))
124 }
125}
126
127#[derive(Debug)]
129pub struct CacheRepo {
130 cache: Cache,
131 repo: Repo,
132}
133
134impl CacheRepo {
135 fn new(cache: Cache, repo: Repo) -> Self {
136 Self { cache, repo }
137 }
138
139 pub fn get(&self, filename: &str) -> Option<PathBuf> {
142 let commit_path = self.ref_path();
143 let commit_hash = std::fs::read_to_string(commit_path).ok()?;
144 let mut pointer_path = self.pointer_path(&commit_hash);
145 pointer_path.push(filename);
146 if pointer_path.exists() {
147 Some(pointer_path)
148 } else {
149 None
150 }
151 }
152
153 fn path(&self) -> PathBuf {
154 let mut ref_path = self.cache.path.clone();
155 ref_path.push(self.repo.folder_name());
156 ref_path
157 }
158
159 fn ref_path(&self) -> PathBuf {
160 let mut ref_path = self.path();
161 ref_path.push("refs");
162 ref_path.push(self.repo.revision());
163 ref_path
164 }
165
166 pub fn create_ref(&self, commit_hash: &str) -> Result<(), std::io::Error> {
169 let ref_path = self.ref_path();
170 std::fs::create_dir_all(ref_path.parent().unwrap())?;
172 let mut file = std::fs::OpenOptions::new()
173 .write(true)
174 .create(true)
175 .truncate(true)
176 .open(&ref_path)?;
177 file.write_all(commit_hash.trim().as_bytes())?;
178 Ok(())
179 }
180
181 #[cfg(any(feature = "tokio", feature = "ureq"))]
182 pub(crate) fn blob_path(&self, etag: &str) -> PathBuf {
183 let mut blob_path = self.path();
184 blob_path.push("blobs");
185 blob_path.push(etag);
186 blob_path
187 }
188
189 pub(crate) fn pointer_path(&self, commit_hash: &str) -> PathBuf {
190 let mut pointer_path = self.path();
191 pointer_path.push("snapshots");
192 pointer_path.push(commit_hash);
193 pointer_path
194 }
195}
196
197impl Default for Cache {
198 fn default() -> Self {
199 let mut path = dirs::home_dir().expect("Cache directory cannot be found");
200 path.push(".cache");
201 path.push("huggingface");
202 path.push("hub");
203 Self::new(path)
204 }
205}
206
207#[derive(Clone, Debug)]
209pub struct Repo {
210 repo_id: String,
211 repo_type: RepoType,
212 revision: String,
213}
214
215impl Repo {
216 pub fn new(repo_id: String, repo_type: RepoType) -> Self {
218 Self::with_revision(repo_id, repo_type, "main".to_string())
219 }
220
221 pub fn with_revision(repo_id: String, repo_type: RepoType, revision: String) -> Self {
223 Self {
224 repo_id,
225 repo_type,
226 revision,
227 }
228 }
229
230 pub fn model(repo_id: String) -> Self {
232 Self::new(repo_id, RepoType::Model)
233 }
234
235 pub fn dataset(repo_id: String) -> Self {
237 Self::new(repo_id, RepoType::Dataset)
238 }
239
240 pub fn space(repo_id: String) -> Self {
242 Self::new(repo_id, RepoType::Space)
243 }
244
245 pub fn folder_name(&self) -> String {
247 let prefix = match self.repo_type {
248 RepoType::Model => "models",
249 RepoType::Dataset => "datasets",
250 RepoType::Space => "spaces",
251 };
252 format!("{prefix}--{}", self.repo_id).replace('/', "--")
253 }
254
255 pub fn revision(&self) -> &str {
257 &self.revision
258 }
259
260 #[cfg(any(feature = "tokio", feature = "ureq"))]
262 pub fn url(&self) -> String {
263 match self.repo_type {
264 RepoType::Model => self.repo_id.to_string(),
265 RepoType::Dataset => {
266 format!("datasets/{}", self.repo_id)
267 }
268 RepoType::Space => {
269 format!("spaces/{}", self.repo_id)
270 }
271 }
272 }
273
274 #[cfg(any(feature = "tokio", feature = "ureq"))]
276 pub fn url_revision(&self) -> String {
277 self.revision.replace('/', "%2F")
278 }
279
280 #[cfg(any(feature = "tokio", feature = "ureq"))]
282 pub fn api_url(&self) -> String {
283 let prefix = match self.repo_type {
284 RepoType::Model => "models",
285 RepoType::Dataset => "datasets",
286 RepoType::Space => "spaces",
287 };
288 format!("{prefix}/{}/revision/{}", self.repo_id, self.url_revision())
289 }
290}
291
292#[cfg(test)]
293mod tests {
294 use super::*;
295
296 #[macro_export]
299 macro_rules! assert_no_diff {
300 ($left: expr, $right: expr) => {
301 let left = serde_json::to_string_pretty(&$left).unwrap();
302 let right = serde_json::to_string_pretty(&$right).unwrap();
303 if left != right {
304 use rand::RngExt;
305 use std::io::Write;
306 use std::process::Command;
307 let rand_string: String = rand::rng()
308 .sample_iter(&rand::distr::Alphanumeric)
309 .take(6)
310 .map(char::from)
311 .collect();
312 let left_filename = format!("/tmp/left-{rand_string}.txt");
313 let mut file = std::fs::File::create(&left_filename).unwrap();
314 file.write_all(left.as_bytes()).unwrap();
315 let right_filename = format!("/tmp/right-{rand_string}.txt");
316 let mut file = std::fs::File::create(&right_filename).unwrap();
317 file.write_all(right.as_bytes()).unwrap();
318 let output = Command::new("diff")
319 .args(["-U5", &right_filename, &left_filename])
321 .output()
322 .expect("Failed to diff")
323 .stdout;
324 let diff = String::from_utf8(output).expect("Invalid utf-8 diff output");
325 assert!(false, "{diff}")
327 };
328 };
329 }
330
331 #[test]
332 #[cfg(not(target_os = "windows"))]
333 fn token_path() {
334 let cache = Cache::from_env();
335 let token_path = cache.token_path().to_str().unwrap().to_string();
336 if let Ok(hf_home) = std::env::var(HF_HOME) {
337 assert_eq!(token_path, format!("{hf_home}/token"));
338 } else {
339 let n = "huggingface/token".len();
340 assert_eq!(&token_path[token_path.len() - n..], "huggingface/token");
341 }
342 }
343
344 #[test]
345 #[cfg(target_os = "windows")]
346 fn token_path() {
347 let cache = Cache::from_env();
348 let token_path = cache.token_path().to_str().unwrap().to_string();
349 if let Ok(hf_home) = std::env::var(HF_HOME) {
350 assert_eq!(token_path, format!("{hf_home}\\token"));
351 } else {
352 let n = "huggingface/token".len();
353 assert_eq!(&token_path[token_path.len() - n..], "huggingface\\token");
354 }
355 }
356}