Skip to main content

cfrp_detector/
cache.rs

1//! Local day-rollover file cache for external data sources (CIDR ranges, colo JSON, ASN HTML).
2//!
3//! The `FileCache` stores fetched resources as `<prefix>-day-<NNN><extension>` inside
4//! a configurable directory, rolling over to a fresh file every calendar day (UTC).
5//! A fetch-with-retry loop handles transient network errors and HTTP 429 rate limits
6//! using exponential backoff + jitter.
7
8use crate::{DetectorError, Result, RetryConfig, is_retryable_error};
9use std::{
10    path::{Path, PathBuf},
11    time::{Duration, SystemTime},
12};
13use tokio::{fs, io::AsyncWriteExt, time::sleep};
14
15/// Configuration for the on-disk HTTP response cache.
16#[derive(Debug, Clone)]
17pub struct CacheConfig {
18    /// Directory where cached files are stored (created on demand).
19    pub directory: PathBuf,
20    /// Files older than this are considered stale and refetched.
21    pub max_age: Duration,
22    /// Retry policy used for network fetches.
23    pub retry: RetryConfig,
24    /// Whether to treat HTTP 429 (Too Many Requests) as a retryable response.
25    pub retry_on_429: bool,
26}
27
28impl Default for CacheConfig {
29    fn default() -> Self {
30        Self {
31            directory: PathBuf::from("data/cfrpdata"),
32            max_age: Duration::from_secs(7 * 24 * 3600),
33            retry: RetryConfig {
34                max_attempts: 3,
35                initial_backoff_ms: 500,
36                max_backoff_ms: 5000,
37                backoff_multiplier: 2.0,
38            },
39            retry_on_429: true,
40        }
41    }
42}
43
44/// Simple day-rolling file cache that sits in front of data-source fetches.
45#[derive(Debug, Clone)]
46pub struct FileCache {
47    /// Active configuration (public for inspection / overrides).
48    pub cfg: CacheConfig,
49}
50
51impl FileCache {
52    /// Wraps a [`CacheConfig`] into a new cache instance.
53    pub fn new(cfg: CacheConfig) -> Self {
54        Self { cfg }
55    }
56
57    /// Returns cached bytes if a fresh file exists, otherwise fetches `url`,
58    /// stores the result under the day-stamped filename, and returns the bytes.
59    ///
60    /// Stale files (older than `max_age`) for the same `prefix`/`extension`
61    /// pair are automatically cleaned up after a successful fetch.
62    pub async fn load_or_fetch(
63        &self,
64        prefix: &str,
65        extension: &str,
66        url: &str,
67        max_age: Duration,
68        client: &reqwest::Client,
69    ) -> Result<Vec<u8>> {
70        fs::create_dir_all(&self.cfg.directory).await?;
71        if let Some(path) = self.find_fresh(prefix, extension, max_age).await? {
72            return Ok(fs::read(path).await?);
73        }
74        let bytes = self.fetch_with_retry(url, client).await?;
75        let name = format!("{}-{}{}", prefix, chrono_like_date(), extension);
76        let path = self.cfg.directory.join(name);
77        let mut file = fs::File::create(&path).await?;
78        file.write_all(&bytes).await?;
79        file.flush().await?;
80        self.cleanup(prefix, extension, &path).await;
81        Ok(bytes)
82    }
83
84    async fn fetch_with_retry(&self, url: &str, client: &reqwest::Client) -> Result<Vec<u8>> {
85        let policy = &self.cfg.retry;
86        let mut last_err: Option<DetectorError> = None;
87        let mut backoff_ms = policy.initial_backoff_ms;
88
89        for attempt in 1..=policy.max_attempts {
90            match self.fetch_once(url, client).await {
91                Ok(bytes) => return Ok(bytes),
92                Err(e) => {
93                    let should_retry = self.should_retry_error(&e);
94                    if !should_retry || attempt == policy.max_attempts {
95                        last_err = Some(e);
96                        break;
97                    }
98                    last_err = Some(e);
99                    let jitter = (backoff_ms as f64 * 0.3 * (rand_like_jitter() - 0.5)) as i64;
100                    let delay_ms = (backoff_ms as i64 + jitter).max(0) as u64;
101                    sleep(Duration::from_millis(delay_ms)).await;
102                    backoff_ms = ((backoff_ms as f64 * policy.backoff_multiplier) as u64)
103                        .min(policy.max_backoff_ms);
104                }
105            }
106        }
107
108        let attempts = policy.max_attempts;
109        let source =
110            Box::new(last_err.unwrap_or_else(|| {
111                DetectorError::DataSource("unknown error after retries".into())
112            }));
113        Err(DetectorError::RetriesExceeded { source, attempts })
114    }
115
116    async fn fetch_once(&self, url: &str, client: &reqwest::Client) -> Result<Vec<u8>> {
117        let resp = client.get(url).send().await?;
118        let status = resp.status();
119        if status.is_success() {
120            Ok(resp.bytes().await?.to_vec())
121        } else {
122            let status_code = status.as_u16();
123            let msg = format!("HTTP {} when fetching {}", status_code, url);
124            Err(DetectorError::Http(msg))
125        }
126    }
127
128    fn should_retry_error(&self, err: &DetectorError) -> bool {
129        if is_retryable_error(err) {
130            return true;
131        }
132        if let DetectorError::Http(msg) = err {
133            if msg.contains("HTTP 5") {
134                return true;
135            }
136            if self.cfg.retry_on_429 && msg.contains("HTTP 429") {
137                return true;
138            }
139        }
140        false
141    }
142
143    async fn find_fresh(
144        &self,
145        prefix: &str,
146        extension: &str,
147        max_age: Duration,
148    ) -> Result<Option<PathBuf>> {
149        let mut dir = fs::read_dir(&self.cfg.directory).await?;
150        let mut latest: Option<(PathBuf, SystemTime)> = None;
151        while let Some(entry) = dir.next_entry().await? {
152            let name = entry.file_name();
153            let name = name.to_string_lossy();
154            if !name.starts_with(&format!("{}-", prefix)) || !name.ends_with(extension) {
155                continue;
156            }
157            let modified = entry
158                .metadata()
159                .await?
160                .modified()
161                .unwrap_or(SystemTime::UNIX_EPOCH);
162            if latest.as_ref().map(|(_, t)| modified > *t).unwrap_or(true) {
163                latest = Some((entry.path(), modified));
164            }
165        }
166        if let Some((path, modified)) = latest
167            && modified.elapsed().unwrap_or(Duration::MAX) <= max_age
168        {
169            return Ok(Some(path));
170        }
171        Ok(None)
172    }
173
174    async fn cleanup(&self, prefix: &str, extension: &str, keep: &Path) {
175        let Ok(mut dir) = fs::read_dir(&self.cfg.directory).await else {
176            return;
177        };
178        while let Ok(Some(entry)) = dir.next_entry().await {
179            let name = entry.file_name().to_string_lossy().into_owned();
180            if name.starts_with(&format!("{}-", prefix))
181                && name.ends_with(extension)
182                && entry.path() != keep
183            {
184                let _ = fs::remove_file(entry.path()).await;
185            }
186        }
187    }
188}
189
190fn chrono_like_date() -> String {
191    let days = SystemTime::now()
192        .duration_since(SystemTime::UNIX_EPOCH)
193        .unwrap_or_default()
194        .as_secs()
195        / 86_400;
196    format!("day-{}", days)
197}
198
199fn rand_like_jitter() -> f64 {
200    use std::collections::hash_map::DefaultHasher;
201    use std::hash::{Hash, Hasher};
202    use std::sync::atomic::{AtomicU64, Ordering};
203    static COUNTER: AtomicU64 = AtomicU64::new(0x9e37_79b9_7f4a_7c15);
204    let mut h = DefaultHasher::new();
205    SystemTime::now().hash(&mut h);
206    let nonce = COUNTER.fetch_add(0x517c_c15b_d185_4c0d, Ordering::Relaxed);
207    h.write_u64(nonce);
208    let val = h.finish();
209    (val as f64) / (u64::MAX as f64)
210}
211
212#[cfg(test)]
213mod tests {
214    use super::*;
215
216    #[test]
217    fn cache_config_default_values() {
218        let cfg = CacheConfig::default();
219        assert_eq!(cfg.directory, PathBuf::from("data/cfrpdata"));
220        assert_eq!(cfg.max_age, Duration::from_secs(7 * 24 * 3600));
221        assert_eq!(cfg.retry.max_attempts, 3);
222        assert_eq!(cfg.retry.initial_backoff_ms, 500);
223        assert!(cfg.retry_on_429);
224    }
225
226    #[test]
227    fn cache_config_clone() {
228        let cfg = CacheConfig {
229            directory: PathBuf::from("/tmp/test"),
230            max_age: Duration::from_secs(60),
231            retry: RetryConfig::default(),
232            retry_on_429: false,
233        };
234        let cfg2 = cfg.clone();
235        assert_eq!(cfg.directory, cfg2.directory);
236        assert_eq!(cfg.max_age, cfg2.max_age);
237        assert_eq!(cfg.retry_on_429, cfg2.retry_on_429);
238    }
239
240    #[test]
241    fn file_cache_new_constructs() {
242        let cfg = CacheConfig::default();
243        let cache = FileCache::new(cfg.clone());
244        assert_eq!(cache.cfg.directory, cfg.directory);
245    }
246
247    #[test]
248    fn chrono_like_date_format() {
249        let s = chrono_like_date();
250        assert!(s.starts_with("day-"));
251        let days_part = &s[4..];
252        assert!(days_part.parse::<u64>().is_ok());
253    }
254
255    #[test]
256    fn rand_like_jitter_within_unit_range() {
257        for _ in 0..10 {
258            let j = rand_like_jitter();
259            assert!((0.0..=1.0).contains(&j), "jitter out of range: {}", j);
260        }
261    }
262
263    #[test]
264    fn should_retry_error_5xx() {
265        let cfg = CacheConfig::default();
266        let cache = FileCache::new(cfg);
267        let e503 = DetectorError::Http("HTTP 503 when fetching /x".into());
268        assert!(cache.should_retry_error(&e503));
269        let e404 = DetectorError::Http("HTTP 404 when fetching /x".into());
270        assert!(!cache.should_retry_error(&e404));
271    }
272
273    #[test]
274    fn should_retry_error_429_when_enabled() {
275        let mut cfg = CacheConfig::default();
276        cfg.retry_on_429 = true;
277        let cache = FileCache::new(cfg);
278        let e429 = DetectorError::Http("HTTP 429 when fetching /x".into());
279        assert!(cache.should_retry_error(&e429));
280
281        let mut cfg_no = CacheConfig::default();
282        cfg_no.retry_on_429 = false;
283        let cache_no = FileCache::new(cfg_no);
284        assert!(!cache_no.should_retry_error(&e429));
285    }
286
287    #[test]
288    fn retries_exceeded_error_display() {
289        let e = DetectorError::RetriesExceeded {
290            source: Box::new(DetectorError::Http("HTTP 503".into())),
291            attempts: 3,
292        };
293        let msg = e.to_string();
294        assert!(msg.contains("3 attempts"));
295        assert!(msg.contains("HTTP 503"));
296    }
297
298    #[tokio::test]
299    async fn file_cache_cleanup_does_not_panic_on_missing_dir() {
300        let cfg = CacheConfig {
301            directory: PathBuf::from("/tmp/nonexistent-cfrp-cache-dir"),
302            max_age: Duration::from_secs(60),
303            retry: RetryConfig::default(),
304            retry_on_429: true,
305        };
306        let cache = FileCache::new(cfg);
307        let keep = PathBuf::from("/tmp/nonexistent-cfrp-cache-dir/keep.txt");
308        cache.cleanup("ips-v4", ".txt", &keep).await;
309    }
310}