Skip to main content

gs_hf/
lib.rs

1#![deny(missing_docs)]
2#![cfg_attr(feature="ureq", doc = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/README.md")))]
3#![cfg_attr(
4    not(feature = "ureq"),
5    doc = "Documentation is meant to be compiled with default features (at least ureq)"
6)]
7use std::io::Write;
8use std::path::PathBuf;
9
10/// The actual Api to interact with the hub.
11#[cfg(any(feature = "tokio", feature = "ureq"))]
12pub mod api;
13
14const HF_HOME: &str = "HF_HOME";
15
16/// The type of repo to interact with
17#[derive(Debug, Clone, Copy)]
18pub enum RepoType {
19    /// This is a model, usually it consists of weight files and some configuration
20    /// files
21    Model,
22    /// This is a dataset, usually contains data within parquet files
23    Dataset,
24    /// This is a space, usually a demo showcashing a given model or dataset
25    Space,
26}
27
28/// A local struct used to fetch information from the cache folder.
29#[derive(Clone, Debug)]
30pub struct Cache {
31    path: PathBuf,
32}
33
34impl Cache {
35    /// Creates a new cache object location
36    pub fn new(path: PathBuf) -> Self {
37        Self { path }
38    }
39
40    /// Creates cache from environment variable HF_HOME (if defined) otherwise
41    /// defaults to [`home_dir`]/.cache/huggingface/
42    pub fn from_env() -> Self {
43        match std::env::var(HF_HOME) {
44            Ok(home) => {
45                let mut path: PathBuf = home.into();
46                path.push("hub");
47                Self::new(path)
48            }
49            Err(_) => Self::default(),
50        }
51    }
52
53    /// Creates a new cache object location
54    pub fn path(&self) -> &PathBuf {
55        &self.path
56    }
57
58    /// Returns the location of the token file
59    pub fn token_path(&self) -> PathBuf {
60        let mut path = self.path.clone();
61        // Remove `"hub"`
62        path.pop();
63        path.push("token");
64        path
65    }
66
67    /// Returns the token value if it exists in the cache
68    /// Use `huggingface-cli login` to set it up.
69    pub fn token(&self) -> Option<String> {
70        let token_filename = self.token_path();
71        if token_filename.exists() {
72            log::info!("Using token file found {token_filename:?}");
73        }
74        match std::fs::read_to_string(token_filename) {
75            Ok(token_content) => {
76                let token_content = token_content.trim();
77                if token_content.is_empty() {
78                    None
79                } else {
80                    Some(token_content.to_string())
81                }
82            }
83            Err(_) => None,
84        }
85    }
86
87    /// Creates a new handle [`CacheRepo`] which contains operations
88    /// on a particular [`Repo`]
89    pub fn repo(&self, repo: Repo) -> CacheRepo {
90        CacheRepo::new(self.clone(), repo)
91    }
92
93    /// Simple wrapper over
94    /// ```
95    /// # use gs_hf::{Cache, Repo, RepoType};
96    /// # let model_id = "gpt2".to_string();
97    /// let cache = Cache::new("/tmp/".into());
98    /// let cache = cache.repo(Repo::new(model_id, RepoType::Model));
99    /// ```
100    pub fn model(&self, model_id: String) -> CacheRepo {
101        self.repo(Repo::new(model_id, RepoType::Model))
102    }
103
104    /// Simple wrapper over
105    /// ```
106    /// # use gs_hf::{Cache, Repo, RepoType};
107    /// # let model_id = "gpt2".to_string();
108    /// let cache = Cache::new("/tmp/".into());
109    /// let cache = cache.repo(Repo::new(model_id, RepoType::Dataset));
110    /// ```
111    pub fn dataset(&self, model_id: String) -> CacheRepo {
112        self.repo(Repo::new(model_id, RepoType::Dataset))
113    }
114
115    /// Simple wrapper over
116    /// ```
117    /// # use gs_hf::{Cache, Repo, RepoType};
118    /// # let model_id = "gpt2".to_string();
119    /// let cache = Cache::new("/tmp/".into());
120    /// let cache = cache.repo(Repo::new(model_id, RepoType::Space));
121    /// ```
122    pub fn space(&self, model_id: String) -> CacheRepo {
123        self.repo(Repo::new(model_id, RepoType::Space))
124    }
125}
126
127/// Shorthand for accessing things within a particular repo
128#[derive(Debug)]
129pub struct CacheRepo {
130    cache: Cache,
131    repo: Repo,
132}
133
134impl CacheRepo {
135    fn new(cache: Cache, repo: Repo) -> Self {
136        Self { cache, repo }
137    }
138
139    /// This will get the location of the file within the cache for the remote
140    /// `filename`. Will return `None` if file is not already present in cache.
141    pub fn get(&self, filename: &str) -> Option<PathBuf> {
142        let commit_path = self.ref_path();
143        let commit_hash = std::fs::read_to_string(commit_path).ok()?;
144        let mut pointer_path = self.pointer_path(&commit_hash);
145        pointer_path.push(filename);
146        if pointer_path.exists() {
147            Some(pointer_path)
148        } else {
149            None
150        }
151    }
152
153    fn path(&self) -> PathBuf {
154        let mut ref_path = self.cache.path.clone();
155        ref_path.push(self.repo.folder_name());
156        ref_path
157    }
158
159    fn ref_path(&self) -> PathBuf {
160        let mut ref_path = self.path();
161        ref_path.push("refs");
162        ref_path.push(self.repo.revision());
163        ref_path
164    }
165
166    /// Creates a reference in the cache directory that points branches to the correct
167    /// commits within the blobs.
168    pub fn create_ref(&self, commit_hash: &str) -> Result<(), std::io::Error> {
169        let ref_path = self.ref_path();
170        // Needs to be done like this because revision might contain `/` creating subfolders here.
171        std::fs::create_dir_all(ref_path.parent().unwrap())?;
172        let mut file = std::fs::OpenOptions::new()
173            .write(true)
174            .create(true)
175            .truncate(true)
176            .open(&ref_path)?;
177        file.write_all(commit_hash.trim().as_bytes())?;
178        Ok(())
179    }
180
181    #[cfg(any(feature = "tokio", feature = "ureq"))]
182    pub(crate) fn blob_path(&self, etag: &str) -> PathBuf {
183        let mut blob_path = self.path();
184        blob_path.push("blobs");
185        blob_path.push(etag);
186        blob_path
187    }
188
189    pub(crate) fn pointer_path(&self, commit_hash: &str) -> PathBuf {
190        let mut pointer_path = self.path();
191        pointer_path.push("snapshots");
192        pointer_path.push(commit_hash);
193        pointer_path
194    }
195}
196
197impl Default for Cache {
198    fn default() -> Self {
199        let mut path = dirs::home_dir().expect("Cache directory cannot be found");
200        path.push(".cache");
201        path.push("huggingface");
202        path.push("hub");
203        Self::new(path)
204    }
205}
206
207/// The representation of a repo on the hub.
208#[derive(Clone, Debug)]
209pub struct Repo {
210    repo_id: String,
211    repo_type: RepoType,
212    revision: String,
213}
214
215impl Repo {
216    /// Repo with the default branch ("main").
217    pub fn new(repo_id: String, repo_type: RepoType) -> Self {
218        Self::with_revision(repo_id, repo_type, "main".to_string())
219    }
220
221    /// fully qualified Repo
222    pub fn with_revision(repo_id: String, repo_type: RepoType, revision: String) -> Self {
223        Self {
224            repo_id,
225            repo_type,
226            revision,
227        }
228    }
229
230    /// Shortcut for [`Repo::new`] with [`RepoType::Model`]
231    pub fn model(repo_id: String) -> Self {
232        Self::new(repo_id, RepoType::Model)
233    }
234
235    /// Shortcut for [`Repo::new`] with [`RepoType::Dataset`]
236    pub fn dataset(repo_id: String) -> Self {
237        Self::new(repo_id, RepoType::Dataset)
238    }
239
240    /// Shortcut for [`Repo::new`] with [`RepoType::Space`]
241    pub fn space(repo_id: String) -> Self {
242        Self::new(repo_id, RepoType::Space)
243    }
244
245    /// The normalized folder nameof the repo within the cache directory
246    pub fn folder_name(&self) -> String {
247        let prefix = match self.repo_type {
248            RepoType::Model => "models",
249            RepoType::Dataset => "datasets",
250            RepoType::Space => "spaces",
251        };
252        format!("{prefix}--{}", self.repo_id).replace('/', "--")
253    }
254
255    /// The revision
256    pub fn revision(&self) -> &str {
257        &self.revision
258    }
259
260    /// The actual URL part of the repo
261    #[cfg(any(feature = "tokio", feature = "ureq"))]
262    pub fn url(&self) -> String {
263        match self.repo_type {
264            RepoType::Model => self.repo_id.to_string(),
265            RepoType::Dataset => {
266                format!("datasets/{}", self.repo_id)
267            }
268            RepoType::Space => {
269                format!("spaces/{}", self.repo_id)
270            }
271        }
272    }
273
274    /// Revision needs to be url escaped before being used in a URL
275    #[cfg(any(feature = "tokio", feature = "ureq"))]
276    pub fn url_revision(&self) -> String {
277        self.revision.replace('/', "%2F")
278    }
279
280    /// Used to compute the repo's url part when accessing the metadata of the repo
281    #[cfg(any(feature = "tokio", feature = "ureq"))]
282    pub fn api_url(&self) -> String {
283        let prefix = match self.repo_type {
284            RepoType::Model => "models",
285            RepoType::Dataset => "datasets",
286            RepoType::Space => "spaces",
287        };
288        format!("{prefix}/{}/revision/{}", self.repo_id, self.url_revision())
289    }
290}
291
292#[cfg(test)]
293mod tests {
294    use super::*;
295
296    /// Internal macro used to show cleaners errors
297    /// on the payloads received from the hub.
298    #[macro_export]
299    macro_rules! assert_no_diff {
300        ($left: expr, $right: expr) => {
301            let left = serde_json::to_string_pretty(&$left).unwrap();
302            let right = serde_json::to_string_pretty(&$right).unwrap();
303            if left != right {
304                use rand::RngExt;
305                use std::io::Write;
306                use std::process::Command;
307                let rand_string: String = rand::rng()
308                    .sample_iter(&rand::distr::Alphanumeric)
309                    .take(6)
310                    .map(char::from)
311                    .collect();
312                let left_filename = format!("/tmp/left-{rand_string}.txt");
313                let mut file = std::fs::File::create(&left_filename).unwrap();
314                file.write_all(left.as_bytes()).unwrap();
315                let right_filename = format!("/tmp/right-{rand_string}.txt");
316                let mut file = std::fs::File::create(&right_filename).unwrap();
317                file.write_all(right.as_bytes()).unwrap();
318                let output = Command::new("diff")
319                    // Reverse order seems to be more appropriate for how we set up the tests.
320                    .args(["-U5", &right_filename, &left_filename])
321                    .output()
322                    .expect("Failed to diff")
323                    .stdout;
324                let diff = String::from_utf8(output).expect("Invalid utf-8 diff output");
325                // eprintln!("assertion `left == right` failed\n{diff}");
326                assert!(false, "{diff}")
327            };
328        };
329    }
330
331    #[test]
332    #[cfg(not(target_os = "windows"))]
333    fn token_path() {
334        let cache = Cache::from_env();
335        let token_path = cache.token_path().to_str().unwrap().to_string();
336        if let Ok(hf_home) = std::env::var(HF_HOME) {
337            assert_eq!(token_path, format!("{hf_home}/token"));
338        } else {
339            let n = "huggingface/token".len();
340            assert_eq!(&token_path[token_path.len() - n..], "huggingface/token");
341        }
342    }
343
344    #[test]
345    #[cfg(target_os = "windows")]
346    fn token_path() {
347        let cache = Cache::from_env();
348        let token_path = cache.token_path().to_str().unwrap().to_string();
349        if let Ok(hf_home) = std::env::var(HF_HOME) {
350            assert_eq!(token_path, format!("{hf_home}\\token"));
351        } else {
352            let n = "huggingface/token".len();
353            assert_eq!(&token_path[token_path.len() - n..], "huggingface\\token");
354        }
355    }
356}