wax/
fetch.rs

1use std::path::PathBuf;
2use std::time::Duration;
3
4use reqwest::Client;
5use tokio::time::sleep;
6
7use crate::cache::{Cache, CacheStats};
8use crate::config::Settings;
9use crate::error::{AppError, Result};
10
11#[derive(Debug)]
12pub struct Fetcher {
13    client: Client,
14    cache: Cache,
15    request_delay_ms: u64,
16    timeout_ms: u64,
17    pub stats: CacheStats,
18}
19
20impl Fetcher {
21    pub async fn new(settings: &Settings) -> Result<Self> {
22        let client = Client::builder()
23            .timeout(Duration::from_millis(settings.timeout_ms))
24            .user_agent(settings.user_agent.clone())
25            .build()?;
26
27        Ok(Self {
28            client,
29            cache: Cache::new(settings.cache_dir.clone()).await?,
30            request_delay_ms: settings.request_delay_ms,
31            timeout_ms: settings.timeout_ms,
32            stats: CacheStats::default(),
33        })
34    }
35
36    pub fn cache(&self) -> &Cache {
37        &self.cache
38    }
39
40    pub async fn fetch_text(&mut self, url: &str) -> Result<String> {
41        if let Some(path) = file_path_from_url(url) {
42            return Ok(tokio::fs::read_to_string(path).await?);
43        }
44
45        if let Some(body) = self.cache.get(url, self.timeout_ms * 60).await? {
46            self.stats.hits += 1;
47            return Ok(body);
48        }
49
50        self.stats.misses += 1;
51        sleep(Duration::from_millis(self.request_delay_ms)).await;
52        let response = self.client.get(url).send().await?;
53        let status = response.status();
54        if !status.is_success() {
55            return Err(AppError::Network(format!(
56                "request failed for {url}: {status}"
57            )));
58        }
59
60        let body = response.text().await?;
61        self.cache.put(url, &body).await?;
62        Ok(body)
63    }
64}
65
66fn file_path_from_url(url: &str) -> Option<PathBuf> {
67    url.strip_prefix("file://").map(PathBuf::from)
68}
wax/fetch.rs

wax/
fetch.rs
