pyoe2_craftpath/external_api/
fetch_json_from_urls.rs

1use std::{
2    fs,
3    io::Write,
4    path::Path,
5    time::{Duration, SystemTime},
6};
7
8use anyhow::Result;
9use tracing::instrument;
10
11use crate::api::types::THashMap;
12
13#[instrument(skip_all)]
14pub fn retrieve_contents_from_urls_with_cache_unstable_order(
15    cache_url_map: THashMap<String, String>,
16    max_cache_duration_in_sec: u64,
17) -> Result<Vec<String>> {
18    let mut results = Vec::new();
19    let max_age = Duration::from_secs(max_cache_duration_in_sec);
20
21    for (cache_path, url) in cache_url_map {
22        let path = Path::new(&cache_path);
23
24        let should_download = if !path.exists() {
25            true
26        } else {
27            let metadata = fs::metadata(path)?;
28            let modified = metadata.modified()?;
29            let age = SystemTime::now().duration_since(modified)?;
30            age > max_age
31        };
32
33        let data = if should_download {
34            tracing::info!("Downloading fresh data for {}...", cache_path);
35            let response = reqwest::blocking::get(&url)?.text()?;
36
37            if let Some(parent) = path.parent() {
38                fs::create_dir_all(parent)?;
39            }
40
41            let mut file = fs::File::create(&path)?;
42            file.write_all(response.as_bytes())?;
43            response
44        } else {
45            tracing::info!("Loading cached contents from '{}'", path.display());
46            fs::read_to_string(&path)?
47        };
48
49        results.push(data);
50    }
51
52    Ok(results)
53}
pyoe2_craftpath/external_api/fetch_json_from_urls.rs

pyoe2_craftpath/external_api/
fetch_json_from_urls.rs
