fetch_data/
lib.rs

1#![warn(clippy::all)]
2#![warn(clippy::pedantic)]
3#![warn(clippy::nursery)]
4#![warn(missing_docs)]
5#![allow(clippy::missing_errors_doc)]
6#![doc = include_str!("../README.md")]
7
8use anyinput::anyinput;
9/// Used to construct global `FetchData` instance.
10///
11/// This is a re-export from crate [`ctor`](https://crates.io/crates/ctor).
12pub use ctor::ctor;
13use directories::ProjectDirs;
14
15use sha2::{Digest, Sha256};
16use std::{
17    collections::HashMap,
18    env,
19    ffi::OsString,
20    fs::{self, read_dir, File},
21    io::{self, ErrorKind},
22    path::{Path, PathBuf},
23    sync::{Mutex, MutexGuard},
24    thread,
25    time::Duration,
26};
27use tempfile::NamedTempFile;
28use thiserror::Error;
29
30/// Used to fetch data files from a URL, if needed. It verifies file contents via a hash.
31///
32/// # Thread Safety
33///
34/// `FetchData` works well with multithreaded testing, It is thread safe (via a Mutex).
35///
36pub struct FetchData {
37    mutex: Mutex<Result<Internals, Box<FetchDataError>>>,
38}
39
40impl FetchData {
41    /// Create a new `FetchData` object.
42    ///
43    /// # Errors
44    ///
45    /// To make `FetchData` work well as a static global, `new` never fails. Instead, `FetchData` stores any error
46    /// and returns it when the first call to `fetch_file`, etc., is made.
47    ///
48    /// # Arguments
49    ///  *all inputs are string-like*
50    ///
51    /// * `registry_contents` - Whitespace delimited list of files and hashes.
52    ///           Use Rust's [`std::include_str`](https://doc.rust-lang.org/std/macro.include_str.html)
53    ///           macro to include the contents of a file.
54    /// * `url_root` - Base URL for remote files.
55    /// * `env_key` - Environment variable that may contain the path to the data directory.
56    ///           If not set, the data directory will be create via
57    ///           [`ProjectDirs`](https://docs.rs/directories/latest/directories/struct.ProjectDirs.html#method.from_path)
58    ///           and the next three arguments.
59    /// * `qualifier` - The reverse domain name notation of the application, excluding the organization or application name itself.
60    /// * `organization` - The name of the organization that develops this application.
61    /// * `application` - The name of the application itself.
62    ///
63    /// # Example
64    /// ```
65    /// use fetch_data::{FetchData};
66    ///
67    /// // Create a new FetchData instance.
68    /// let fetch_data = FetchData::new(
69    ///     "small.fam 36e0086c0353ff336d0533330dbacb12c75e37dc3cba174313635b98dfe86ed2
70    ///      small.bim 56b6657a3766e2e52273f89d28be6135f9424ca1d204d29f3fa1c5a90eca794e",
71    ///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/",
72    ///     "BAR_APP_DATA_DIR",
73    ///     "com",
74    ///     "Foo Corp",
75    ///     "Bar App",
76    ///     );
77    ///
78    /// // If the local file exists and has the right hash, just return its path.
79    /// // Otherwise, download the file, confirm its hash, and return its path.
80    /// let local_path = fetch_data.fetch_file("small.bim")?;
81    /// assert!(local_path.exists());
82    /// # use fetch_data::FetchDataError;
83    /// # Ok::<(), Box<FetchDataError>>(())
84    /// ```
85    #[anyinput]
86    pub fn new(
87        registry_contents: AnyString,
88        url_root: AnyString,
89        env_key: AnyString,
90        qualifier: AnyString,
91        organization: AnyString,
92        application: AnyString,
93    ) -> Self {
94        Self {
95            mutex: Mutex::new(Internals::new(
96                registry_contents,
97                url_root,
98                env_key,
99                qualifier,
100                organization,
101                application,
102            )),
103        }
104    }
105
106    fn lock(&self) -> MutexGuard<'_, Result<Internals, Box<FetchDataError>>> {
107        match self.mutex.lock() {
108            Ok(lock) => lock,
109            Err(err) => err.into_inner(),
110        }
111    }
112
113    /// Fetch data files from a URL, but only if needed. Verify contents via a hash.
114    ///
115    /// # Example
116    /// ```
117    /// use fetch_data::{FetchData};
118    ///
119    /// // Create a new FetchData object.
120    /// let fetch_data = FetchData::new(
121    ///     "small.fam 36e0086c0353ff336d0533330dbacb12c75e37dc3cba174313635b98dfe86ed2
122    ///      small.bim 56b6657a3766e2e52273f89d28be6135f9424ca1d204d29f3fa1c5a90eca794e",
123    ///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/",
124    ///     "BAR_APP_DATA_DIR",
125    ///     "com",
126    ///     "Foo Corp",
127    ///     "Bar App",
128    ///     );
129    ///
130    /// // If the local file exists and has the right hash, just return its path.
131    /// // Otherwise, download the file, confirm its hash, and return its path.
132    /// let local_path = fetch_data.fetch_file("small.bim")?;
133    /// assert!(local_path.exists());
134    /// # use fetch_data::FetchDataError;
135    /// # Ok::<(), Box<FetchDataError>>(())
136    /// ```
137    #[anyinput]
138    pub fn fetch_file(&self, path: AnyPath) -> Result<PathBuf, Box<FetchDataError>> {
139        let path_list = vec![path.to_path_buf()];
140        let vec = self.fetch_files(path_list)?;
141        Ok(vec[0].clone())
142    }
143
144    /// Given a list of files, returns a list of their local paths. If necessary, the files will be downloaded.
145    ///
146    /// # Example
147    /// ```
148    /// use fetch_data::{FetchData};
149    ///
150    /// // Create a new FetchData instance.
151    /// let fetch_data = FetchData::new(
152    ///     "small.fam 36e0086c0353ff336d0533330dbacb12c75e37dc3cba174313635b98dfe86ed2
153    ///      small.bim 56b6657a3766e2e52273f89d28be6135f9424ca1d204d29f3fa1c5a90eca794e",
154    ///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/",
155    ///     "BAR_APP_DATA_DIR",
156    ///     "com",
157    ///     "Foo Corp",
158    ///     "Bar App",
159    ///     );
160    ///
161    /// // If a local file exists and has the right hash, just return its path
162    /// // in a list. Otherwise, download the file, confirm its hash, and return
163    /// //  its path in the list.
164    /// let local_path_list = fetch_data.fetch_files(["small.bim", "small.bim"])?;
165    /// assert!(local_path_list[0].exists() && local_path_list[1].exists());
166    /// # use fetch_data::FetchDataError;
167    /// # Ok::<(), Box<FetchDataError>>(())
168    /// ```
169    #[anyinput]
170    #[allow(clippy::significant_drop_tightening)]
171    pub fn fetch_files(
172        &self,
173        path_list: AnyIter<AnyPath>,
174    ) -> Result<Vec<PathBuf>, Box<FetchDataError>> {
175        let lock = self.lock();
176        // Convert Result to reference the error inside the Box for compatibility with internals()
177        let lock_ref = lock.as_ref().map_err(|e| &**e);
178        let internals = Self::internals(lock_ref)?;
179        let hash_registry = &internals.hash_registry;
180        let cache_dir = &internals.cache_dir;
181        let url_root = &internals.url_root;
182
183        let mut local_list: Vec<PathBuf> = Vec::new();
184        for path in path_list {
185            let path = path.as_ref();
186
187            let path_as_string = path.to_str().ok_or_else(|| {
188                Box::new(FetchDataSpecificError::UnknownOrBadFile("???".to_string()).into())
189            })?;
190
191            let Some(hash) = hash_registry.get(path) else {
192                return Err(Box::new(
193                    FetchDataSpecificError::UnknownOrBadFile(path_as_string.to_string()).into(),
194                ));
195            };
196
197            let local_path = cache_dir.join(path);
198            let url = format!("{url_root}{path_as_string}");
199            fetch(url, hash, &local_path)?;
200            local_list.push(local_path);
201        }
202
203        Ok(local_list)
204    }
205
206    fn internals<'a>(
207        lock_ref: Result<&'a Internals, &FetchDataError>,
208    ) -> Result<&'a Internals, Box<FetchDataError>> {
209        match lock_ref {
210            Ok(internals) => Ok(internals),
211            Err(e) => Err(Box::new(
212                FetchDataSpecificError::FetchDataNewFailed(e.to_string()).into(),
213            )),
214        }
215    }
216    /// Compute registry contents by downloading items and hashing them.
217    ///
218    /// # Tips
219    ///
220    /// * If you put the returned contents into a file, you can use Rust's [`std::include_str`](https://doc.rust-lang.org/std/macro.include_str.html)
221    ///   macro to include the contents of that file in [`FetchData::new`](struct.FetchData.html#method.new).
222    ///
223    /// * Use utility function [`fetch_data::dir_to_file_list`](fn.dir_to_file_list.html) to create a list of files in any local directory.
224    /// Note the hash is computed on download files, not any original local files.
225    ///
226    /// # Example
227    ///
228    /// ```
229    /// use fetch_data::{FetchData};
230    ///
231    /// // Create a new FetchData object.
232    /// let fetch_data = FetchData::new(
233    ///     "", // ignored
234    ///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/",
235    ///     "BAR_APP_DATA_DIR",
236    ///     "com",
237    ///     "Foo Corp",
238    ///     "Bar App",
239    ///     );
240    ///
241    /// // Even if local files exist, download each file. Hash each file. Return the results as a string.
242    /// let registry_contents = fetch_data.gen_registry_contents(["small.fam", "small.bim"])?;
243    /// println!("{registry_contents}"); // Prints:
244    ///                                  // small.fam 36e0086c0353ff336d0533330dbacb12c75e37dc3cba174313635b98dfe86ed2
245    ///                                  // small.bim 56b6657a3766e2e52273f89d28be6135f9424ca1d204d29f3fa1c5a90eca794e
246    /// # use fetch_data::FetchDataError;
247    /// # Ok::<(), Box<FetchDataError>>(())
248    /// ```
249    #[anyinput]
250    #[allow(clippy::significant_drop_tightening)]
251    pub fn gen_registry_contents(
252        &self,
253        path_list: AnyIter<AnyPath>,
254    ) -> Result<String, Box<FetchDataError>> {
255        let lock = self.lock();
256        // Convert Result to reference the error inside the Box for compatibility with internals()
257        let lock_ref = lock.as_ref().map_err(|e| &**e);
258        let internals = Self::internals(lock_ref)?;
259        let cache_dir = &internals.cache_dir;
260        let url_root = &internals.url_root;
261
262        let mut s = String::new();
263        for path in path_list {
264            let path = path.as_ref();
265
266            let Some(path_as_string) = path.to_str() else {
267                return Err(Box::new(
268                    FetchDataSpecificError::UnknownOrBadFile("???".to_string()).into(),
269                ));
270            };
271
272            let local_path = cache_dir.join(path);
273            let url = format!("{url_root}{path_as_string}");
274            download(url, &local_path)?;
275            let hash = hash_file(&local_path)?;
276            s.push_str(path_as_string);
277            s.push(' ');
278            s.push_str(&hash);
279            s.push('\n');
280        }
281
282        Ok(s)
283    }
284
285    /// Return the path to the local cache directory.
286    #[allow(clippy::significant_drop_tightening)]
287    pub fn cache_dir(&self) -> Result<PathBuf, Box<FetchDataError>> {
288        let lock = self.lock();
289        // Convert Result to reference the error inside the Box for compatibility with internals()
290        let lock_ref = lock.as_ref().map_err(|e| &**e);
291        let internals = Self::internals(lock_ref)?;
292        let cache_dir = &internals.cache_dir;
293        Ok(cache_dir.to_owned())
294    }
295}
296
297/// All possible errors returned by this crate and the crates it depends on.
298// Based on `<https://nick.groenen.me/posts/rust-error-handling/#the-library-error-type>`
299#[derive(Error, Debug)]
300pub enum FetchDataError {
301    #[allow(missing_docs)]
302    #[error(transparent)]
303    FetchDataError(#[from] FetchDataSpecificError),
304
305    #[allow(missing_docs)]
306    #[error(transparent)]
307    IOError(#[from] io::Error),
308
309    #[allow(missing_docs)]
310    #[error(transparent)]
311    UreqError(#[from] ureq::Error),
312}
313/// All errors specific to this crate.
314#[derive(Error, Debug, Clone)]
315pub enum FetchDataSpecificError {
316    #[allow(missing_docs)]
317    #[error("Unknown or bad file '{0}'")]
318    UnknownOrBadFile(String),
319
320    #[allow(missing_docs)]
321    #[error("The registry of files is invalid")]
322    RegistryProblem(),
323
324    #[allow(missing_docs)]
325    #[error("FetchData new failed with error: {0}")]
326    FetchDataNewFailed(String),
327
328    #[allow(missing_docs)]
329    #[error("Downloaded file not seen: {0}")]
330    DownloadedFileNotSeen(String),
331
332    #[allow(missing_docs)]
333    #[error("Downloaded file has wrong hash: {0},expected: {1}, actual: {2}")]
334    DownloadedFileWrongHash(String, String, String),
335
336    #[allow(missing_docs)]
337    #[error("Cannot create cache directory")]
338    CannotCreateCacheDir(),
339}
340
341/// If necessary, retrieve a file from a URL, checking its hash.
342/// # Example
343/// ```
344/// use fetch_data::fetch;
345/// use temp_testdir::TempDir;
346///
347/// // Create a temporary local directory.
348/// let temp_dir = TempDir::default();
349/// // Download the file and check its hash.
350/// let path = temp_dir.join("small.fam");
351/// fetch(
352///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/small.fam",
353///     "36e0086c0353ff336d0533330dbacb12c75e37dc3cba174313635b98dfe86ed2",
354///     &path,
355/// )?;
356/// assert!(&path.exists());
357/// // This time, because the local file exists and has the correct hash, no download is performed.
358/// fetch(
359///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/small.fam",
360///     "36e0086c0353ff336d0533330dbacb12c75e37dc3cba174313635b98dfe86ed2",
361///     &path,
362/// )?;
363/// assert!(&path.exists());
364/// # use fetch_data::FetchDataError;
365/// # Ok::<(), Box<FetchDataError>>(())
366/// ```
367#[anyinput]
368pub fn fetch(url: AnyString, hash: AnyString, path: AnyPath) -> Result<(), Box<FetchDataError>> {
369    if !path.exists() {
370        download(url, path)?;
371    }
372    let mut actual_hash = hash_file(path)?;
373    if !actual_hash.eq(hash) {
374        // A stale or corrupt local copy: replace it once before reporting a mismatch.
375        download(url, path)?;
376        actual_hash = hash_file(path)?;
377    }
378    if !actual_hash.eq(hash) {
379        return Err(Box::new(
380            FetchDataSpecificError::DownloadedFileWrongHash(
381                path.display().to_string(),
382                hash.to_string(),
383                actual_hash,
384            )
385            .into(),
386        ));
387    }
388    Ok(())
389}
390
391/// Download a file from a URL and compute its hash.
392///
393/// # Example
394/// ```
395/// use fetch_data::hash_download;
396/// use temp_testdir::TempDir;
397///
398/// // Create a temporary local directory.
399/// let temp_dir = TempDir::default();
400/// let path = temp_dir.join("small.fam");
401/// // Download a file and compute its hash.
402/// let hash = hash_download(
403///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/small.fam",
404///    &path,
405/// )?;
406/// assert!(hash.eq("36e0086c0353ff336d0533330dbacb12c75e37dc3cba174313635b98dfe86ed2"));
407/// # use fetch_data::FetchDataError;
408/// # Ok::<(), Box<FetchDataError>>(())
409/// ```
410#[anyinput]
411pub fn hash_download(url: AnyString, path: AnyPath) -> Result<String, Box<FetchDataError>> {
412    download(url, path)?;
413    hash_file(path)
414}
415
416/// Compute the hash (SHA256) of a local file.
417///
418/// # Example
419/// ```
420/// use fetch_data::{hash_file, download};
421/// use temp_testdir::TempDir;
422///
423/// // Download a file to a temporary directory.
424/// let temp_dir = TempDir::default();
425/// let path = temp_dir.join("small.fam");
426/// download(
427///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/small.fam",
428///     &path,
429/// )?;
430/// // Compute the hash of the file.
431/// let hash = hash_file(&path)?;
432/// assert!(hash.eq("36e0086c0353ff336d0533330dbacb12c75e37dc3cba174313635b98dfe86ed2"));
433/// # use fetch_data::FetchDataError;
434/// # Ok::<(), Box<FetchDataError>>(())
435#[anyinput]
436pub fn hash_file(path: AnyPath) -> Result<String, Box<FetchDataError>> {
437    let mut sha256 = Sha256::new();
438    let mut file = File::open(path).map_err(|e| Box::new(e.into()))?;
439
440    io::copy(&mut file, &mut sha256).map_err(|e| Box::new(e.into()))?;
441    let hash_bytes = sha256.finalize();
442
443    let hex_hash = base16ct::lower::encode_string(&hash_bytes);
444    Ok(hex_hash)
445}
446
447/// Download a file from a URL.
448///
449/// # Example
450/// ```
451/// use fetch_data::download;
452/// use temp_testdir::TempDir;
453///
454/// // Create a temporary local directory.
455/// let temp_dir = TempDir::default();
456/// // Download a file to the temporary directory.
457/// let path = temp_dir.join("small.fam");
458/// download(
459///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/small.fam",
460///     &path,
461/// )?;
462/// assert!(path.exists());
463/// # use fetch_data::FetchDataError;
464/// # Ok::<(), Box<FetchDataError>>(())
465/// ```
466///
467/// Transient failures (network errors, timeouts, HTTP 429 and 5xx) are retried
468/// up to three times with exponential backoff. The file is written under a
469/// temporary name and renamed into place only when complete, so an interrupted
470/// download never leaves a partial file at `path`.
471#[anyinput]
472pub fn download(url: AnyString, path: AnyPath) -> Result<(), Box<FetchDataError>> {
473    let mut attempt = 1;
474    loop {
475        match download_once(url, path) {
476            Ok(()) => break,
477            Err(error) if attempt < DOWNLOAD_ATTEMPTS && is_transient(&error) => {
478                thread::sleep(RETRY_BASE_DELAY * (1 << (attempt - 1)));
479                attempt += 1;
480            }
481            Err(error) => return Err(error),
482        }
483    }
484    if !path.exists() {
485        return Err(Box::new(
486            FetchDataSpecificError::DownloadedFileNotSeen(path.display().to_string()).into(),
487        ));
488    }
489    Ok(())
490}
491
492/// A download is tried this many times in all before its error is returned.
493const DOWNLOAD_ATTEMPTS: u32 = 4;
494/// Wait before the first retry; doubled before each later one.
495#[cfg(not(test))]
496const RETRY_BASE_DELAY: Duration = Duration::from_secs(1);
497#[cfg(test)]
498const RETRY_BASE_DELAY: Duration = Duration::from_millis(10);
499const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
500/// Longest wait for more data before an attempt is abandoned and retried.
501const READ_TIMEOUT: Duration = Duration::from_secs(60);
502
503fn download_once(url: &str, path: &Path) -> Result<(), Box<FetchDataError>> {
504    let agent = ureq::AgentBuilder::new()
505        .timeout_connect(CONNECT_TIMEOUT)
506        .timeout_read(READ_TIMEOUT)
507        .build();
508    let response = agent
509        .get(url)
510        .call()
511        .map_err(|error| Box::new(error.into()))?;
512
513    let directory = match path.parent() {
514        Some(parent) if !parent.as_os_str().is_empty() => parent,
515        _ => Path::new("."),
516    };
517    // The temporary file is deleted if it is dropped before `persist`, so any
518    // error below leaves no partial file behind.
519    let mut temp_file = NamedTempFile::new_in(directory).map_err(|error| Box::new(error.into()))?;
520    io::copy(&mut response.into_reader(), &mut temp_file)
521        .map_err(|error| Box::new(error.into()))?;
522    temp_file
523        .as_file()
524        .sync_all()
525        .map_err(|error| Box::new(error.into()))?;
526    temp_file
527        .persist(path)
528        .map_err(|persist_error| Box::new(persist_error.error.into()))?;
529    Ok(())
530}
531
532fn is_transient(error: &FetchDataError) -> bool {
533    match error {
534        FetchDataError::UreqError(ureq_error) => match ureq_error {
535            ureq::Error::Status(code, _) => *code == 429 || *code >= 500,
536            ureq::Error::Transport(_) => true,
537        },
538        // Reading the response body failed partway (reset, timeout, truncation).
539        FetchDataError::IOError(io_error) => matches!(
540            io_error.kind(),
541            ErrorKind::ConnectionReset
542                | ErrorKind::ConnectionAborted
543                | ErrorKind::BrokenPipe
544                | ErrorKind::TimedOut
545                | ErrorKind::UnexpectedEof
546                | ErrorKind::Interrupted
547                | ErrorKind::WouldBlock
548        ),
549        FetchDataError::FetchDataError(_) => false,
550    }
551}
552
553fn hash_registry(registry_contents: &str) -> Result<HashMap<PathBuf, String>, Box<FetchDataError>> {
554    let mut hash_map = HashMap::new();
555    for line in registry_contents.lines() {
556        let mut parts = line.split_whitespace();
557
558        let url = if let Some(url) = parts.next() {
559            if url.is_empty() {
560                return Err(Box::new(FetchDataSpecificError::RegistryProblem().into()));
561            }
562            PathBuf::from(url)
563        } else {
564            return Err(Box::new(FetchDataSpecificError::RegistryProblem().into()));
565        };
566        let hash = if let Some(hash) = parts.next() {
567            hash.to_string()
568        } else {
569            return Err(Box::new(FetchDataSpecificError::RegistryProblem().into()));
570        };
571        if hash.is_empty() || parts.next().is_some() {
572            return Err(Box::new(FetchDataSpecificError::RegistryProblem().into()));
573        }
574
575        hash_map.insert(url, hash.clone());
576    }
577    Ok(hash_map)
578}
579
580/// List all the files in a local directory.
581///
582/// # Example
583/// ```
584/// use fetch_data::{dir_to_file_list, download};
585/// use temp_testdir::TempDir;
586///
587/// // Create a local directory and download two files to it.
588/// let temp_dir = TempDir::default();
589/// download(
590///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/small.fam",
591///     temp_dir.join("small.fam"),
592/// )?;
593/// download(
594///     "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/small.bim",
595///     temp_dir.join("small.bim"),
596/// )?;
597/// // List the files in the directory.
598/// let file_list = dir_to_file_list(temp_dir)?;
599/// println!("{file_list:?}"); // Prints ["small.bim", "small.fam"]
600/// # use fetch_data::FetchDataError;
601/// # Ok::<(), Box<FetchDataError>>(())
602/// ```
603#[anyinput]
604pub fn dir_to_file_list(path: AnyPath) -> Result<Vec<OsString>, Box<FetchDataError>> {
605    let file_list = read_dir(path)
606        .map_err(|error| Box::new(error.into()))?
607        .map(|res| res.map(|e| e.file_name()))
608        .collect::<Result<Vec<_>, io::Error>>()
609        .map_err(|error| Box::new(error.into()))?;
610    Ok(file_list)
611}
612struct Internals {
613    cache_dir: PathBuf,
614    hash_registry: HashMap<PathBuf, String>,
615    url_root: String,
616}
617
618impl Internals {
619    fn new(
620        registry_contents: &str,
621        url_root: &str,
622        env_key: &str,
623        qualifier: &str,
624        organization: &str,
625        application: &str,
626    ) -> Result<Self, Box<FetchDataError>> {
627        let cache_dir = Self::cache_dir(env_key, qualifier, organization, application)?;
628        let hash_registry = hash_registry(registry_contents)?;
629
630        Ok(Self {
631            cache_dir,
632            hash_registry,
633            url_root: url_root.to_string(),
634        })
635    }
636
637    fn cache_dir(
638        env_key: &str,
639        qualifier: &str,
640        organization: &str,
641        application: &str,
642    ) -> Result<PathBuf, Box<FetchDataError>> {
643        let cache_dir = if let Ok(cache_dir) = env::var(env_key) {
644            PathBuf::from(cache_dir)
645        } else if let Some(proj_dirs) = ProjectDirs::from(qualifier, organization, application) {
646            proj_dirs.cache_dir().to_owned()
647        } else {
648            return Err(Box::new(
649                FetchDataSpecificError::CannotCreateCacheDir().into(),
650            ));
651        };
652        if !cache_dir.exists() {
653            fs::create_dir_all(&cache_dir).map_err(|error| Box::new(error.into()))?;
654        }
655        Ok(cache_dir)
656    }
657}
658
659#[ctor]
660static STATIC_FETCH_DATA: FetchData = FetchData::new(
661    include_str!("../registry.txt"),
662    "https://raw.githubusercontent.com/CarlKCarlK/fetch-data/main/tests/data/",
663    "BAR_APP_DATA_DIR",
664    "com",
665    "Foo Corp",
666    "Bar App",
667);
668
669/// A sample `sample_file`. Don't use this. Instead, define your own `sample_file` function
670/// that knows how to fetch your data files.
671#[anyinput]
672pub fn sample_file(path: AnyPath) -> Result<PathBuf, Box<FetchDataError>> {
673    STATIC_FETCH_DATA.fetch_file(path)
674}
675
676#[cfg(test)]
677mod download_tests {
678    use super::{download, fetch, hash_file, DOWNLOAD_ATTEMPTS};
679    use std::{
680        fs,
681        io::{self, BufRead, BufReader, Write},
682        net::TcpListener,
683        path::Path,
684        thread::{self, JoinHandle},
685    };
686    use temp_testdir::TempDir;
687
688    enum Reply {
689        Status(u16),
690        Body(&'static [u8]),
691        /// Promise `len` bytes but send only `body` before closing.
692        Truncated(usize, &'static [u8]),
693    }
694
695    /// Serve one scripted reply per connection on 127.0.0.1, then stop.
696    /// The thread returns how many replies it served.
697    fn serve(replies: Vec<Reply>) -> io::Result<(String, JoinHandle<io::Result<usize>>)> {
698        let listener = TcpListener::bind("127.0.0.1:0")?;
699        let url = format!("http://{}/small.fam", listener.local_addr()?);
700        let server = thread::spawn(move || {
701            let mut served = 0;
702            for reply in replies {
703                let (mut stream, _) = listener.accept()?;
704                let mut reader = BufReader::new(stream.try_clone()?);
705                let mut line = String::new();
706                while reader.read_line(&mut line)? > 2 {
707                    line.clear();
708                }
709                let response = match reply {
710                    Reply::Status(code) => format!(
711                        "HTTP/1.1 {code} Simulated\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
712                    )
713                    .into_bytes(),
714                    Reply::Body(body) => [
715                        format!(
716                            "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
717                            body.len()
718                        )
719                        .as_bytes(),
720                        body,
721                    ]
722                    .concat(),
723                    Reply::Truncated(len, body) => [
724                        format!("HTTP/1.1 200 OK\r\nContent-Length: {len}\r\nConnection: close\r\n\r\n")
725                            .as_bytes(),
726                        body,
727                    ]
728                    .concat(),
729                };
730                stream.write_all(&response)?;
731                served += 1;
732            }
733            Ok(served)
734        });
735        Ok((url, server))
736    }
737
738    fn served(server: JoinHandle<io::Result<usize>>) -> io::Result<usize> {
739        server.join().expect("server thread panicked")
740    }
741
742    fn file_names(dir: &Path) -> io::Result<Vec<String>> {
743        let mut names = fs::read_dir(dir)?
744            .map(|entry| Ok(entry?.file_name().to_string_lossy().into_owned()))
745            .collect::<io::Result<Vec<String>>>()?;
746        names.sort();
747        Ok(names)
748    }
749
750    #[test]
751    fn retries_server_errors() -> anyhow::Result<()> {
752        let (url, server) = serve(vec![
753            Reply::Status(503),
754            Reply::Status(502),
755            Reply::Body(b"hello"),
756        ])?;
757        let temp_dir = TempDir::default();
758        let path = temp_dir.join("small.fam");
759        download(&url, &path)?;
760        assert_eq!(fs::read(&path)?, b"hello");
761        assert_eq!(served(server)?, 3);
762        Ok(())
763    }
764
765    #[test]
766    fn does_not_retry_not_found() -> anyhow::Result<()> {
767        let (url, server) = serve(vec![Reply::Status(404)])?;
768        let temp_dir = TempDir::default();
769        let path = temp_dir.join("small.fam");
770        assert!(download(&url, &path).is_err());
771        assert_eq!(served(server)?, 1);
772        assert!(!path.exists());
773        Ok(())
774    }
775
776    #[test]
777    fn truncated_body_is_retried_without_leaving_partial_file() -> anyhow::Result<()> {
778        let (url, server) = serve(vec![
779            Reply::Truncated(10, b"0123"),
780            Reply::Body(b"complete"),
781        ])?;
782        let temp_dir = TempDir::default();
783        let path = temp_dir.join("small.fam");
784        fs::write(&path, b"old")?;
785        download(&url, &path)?;
786        assert_eq!(fs::read(&path)?, b"complete");
787        assert_eq!(served(server)?, 2);
788        assert_eq!(file_names(&temp_dir)?, ["small.fam"]);
789        Ok(())
790    }
791
792    #[test]
793    fn gives_up_and_keeps_previous_file() -> anyhow::Result<()> {
794        let replies = (0..DOWNLOAD_ATTEMPTS).map(|_| Reply::Status(503)).collect();
795        let (url, server) = serve(replies)?;
796        let temp_dir = TempDir::default();
797        let path = temp_dir.join("small.fam");
798        fs::write(&path, b"old")?;
799        assert!(download(&url, &path).is_err());
800        assert_eq!(served(server)?, usize::try_from(DOWNLOAD_ATTEMPTS)?);
801        assert_eq!(fs::read(&path)?, b"old");
802        assert_eq!(file_names(&temp_dir)?, ["small.fam"]);
803        Ok(())
804    }
805
806    #[test]
807    fn fetch_replaces_corrupt_local_file() -> anyhow::Result<()> {
808        let (url, server) = serve(vec![Reply::Body(b"good")])?;
809        let temp_dir = TempDir::default();
810        let good = temp_dir.join("good");
811        fs::write(&good, b"good")?;
812        let hash = hash_file(&good)?;
813        let path = temp_dir.join("small.fam");
814        fs::write(&path, b"partial from an old interrupted download")?;
815        fetch(&url, &hash, &path)?;
816        assert_eq!(fs::read(&path)?, b"good");
817        assert_eq!(served(server)?, 1);
818        Ok(())
819    }
820}
fetch_data/lib.rs

fetch_data/
lib.rs
