codebook_downloader 0.3.42

use anyhow::Result;
use base16ct::lower;
use chrono::{DateTime, Utc};
use log::{error, info};
use reqwest::blocking::Client;
use reqwest::header::{IF_MODIFIED_SINCE, LAST_MODIFIED};
use rustls::ClientConfig;
#[cfg(not(target_os = "android"))]
use rustls_platform_verifier::BuilderVerifierExt;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::fs::{self, File};
use std::io::{BufReader, Read};
use std::path::{Path, PathBuf};
use std::sync::{Arc, OnceLock, RwLock};
use tempfile::NamedTempFile;

const METADATA_FILE: &str = "_metadata.json";
const TWO_WEEKS: u64 = 14 * 24 * 3600;

/// A definitive HTTP client-error response (4xx, e.g. 404). The server
/// answered; retrying won't change the result. Downloads fail fast on these,
/// and callers can back off much longer than for transient failures.
/// Retrieve via `err.downcast_ref::<PermanentHttpError>()`.
#[derive(Debug)]
pub struct PermanentHttpError {
    pub status: u16,
    pub url: String,
}

impl std::fmt::Display for PermanentHttpError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "Download failed with status {} for {}",
            self.status, self.url
        )
    }
}

impl std::error::Error for PermanentHttpError {}

/// A response from an [`HttpTransport`]: status code, the Last-Modified
/// header if present, and a streaming body.
pub struct TransportResponse {
    pub status: u16,
    pub last_modified: Option<String>,
    pub body: Box<dyn Read>,
}

/// The HTTP layer behind the [`Downloader`]. Production uses the reqwest
/// implementation; tests inject an in-memory fake so `cargo test` never
/// opens a socket.
pub trait HttpTransport: Send + Sync {
    /// Perform a GET request, optionally with an If-Modified-Since header.
    fn get(&self, url: &str, if_modified_since: Option<&str>) -> Result<TransportResponse>;
}

/// Test-only network guard, in the style of pytest-socket: any request
/// through the real transport panics. Compiled in for this crate's own unit
/// tests and whenever the `deny-network` feature is enabled — workspace
/// crates enable that feature from their dev-dependencies, so every
/// `cargo test` build denies network access while release builds carry no
/// guard at all.
#[cfg(any(test, feature = "deny-network"))]
fn assert_network_allowed(url: &str) {
    panic!(
        "Blocked network request to {url}: tests run with networking denied \
         (deny-network feature). Inject a fake HttpTransport via \
         Downloader::with_transport, or use the dictionary fixtures \
         (refresh with `make fetch_fixtures`)."
    );
}

#[cfg(not(any(test, feature = "deny-network")))]
fn assert_network_allowed(_url: &str) {}

/// Production transport backed by a lazily-built reqwest client with
/// platform-verified TLS (bundled Mozilla roots as fallback).
pub struct ReqwestTransport {
    client: OnceLock<Client>,
}

impl ReqwestTransport {
    pub fn new() -> Self {
        Self {
            client: OnceLock::new(),
        }
    }

    fn client(&self) -> &Client {
        self.client.get_or_init(|| {
            let arc_crypto_provider =
                std::sync::Arc::new(rustls::crypto::aws_lc_rs::default_provider());
            let config = Self::build_tls_config(arc_crypto_provider);
            reqwest::blocking::Client::builder()
                .use_preconfigured_tls(config)
                .build()
                .expect("codebook: failed to build HTTP client")
        })
    }

    #[cfg(not(target_os = "android"))]
    fn build_tls_config(
        crypto_provider: std::sync::Arc<rustls::crypto::CryptoProvider>,
    ) -> ClientConfig {
        // Try OS cert chains first (proxy support), fall back to bundled Mozilla roots
        ClientConfig::builder_with_provider(crypto_provider.clone())
            .with_safe_default_protocol_versions()
            .expect("codebook: failed to configure TLS protocol versions")
            .with_platform_verifier()
            .map(|config| config.with_no_client_auth())
            .unwrap_or_else(|e| {
                log::warn!("Platform verifier unavailable, using bundled roots: {e}");
                Self::build_webpki_tls_config(crypto_provider)
            })
    }

    #[cfg(target_os = "android")]
    fn build_tls_config(
        crypto_provider: std::sync::Arc<rustls::crypto::CryptoProvider>,
    ) -> ClientConfig {
        // Android (Termux) doesn't support rustls-platform-verifier without JNI,
        // so use bundled Mozilla CA roots directly.
        Self::build_webpki_tls_config(crypto_provider)
    }

    fn build_webpki_tls_config(
        crypto_provider: std::sync::Arc<rustls::crypto::CryptoProvider>,
    ) -> ClientConfig {
        let mut root_store = rustls::RootCertStore::empty();
        root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
        ClientConfig::builder_with_provider(crypto_provider)
            .with_safe_default_protocol_versions()
            .expect("codebook: failed to configure TLS protocol versions")
            .with_root_certificates(root_store)
            .with_no_client_auth()
    }
}

impl Default for ReqwestTransport {
    fn default() -> Self {
        Self::new()
    }
}

impl HttpTransport for ReqwestTransport {
    fn get(&self, url: &str, if_modified_since: Option<&str>) -> Result<TransportResponse> {
        assert_network_allowed(url);
        let mut request = self.client().get(url);
        if let Some(ims) = if_modified_since {
            request = request.header(IF_MODIFIED_SINCE, ims);
        }
        let response = request.send()?;
        let status = response.status().as_u16();
        let last_modified = response
            .headers()
            .get(LAST_MODIFIED)
            .and_then(|hv| hv.to_str().ok())
            .map(String::from);
        Ok(TransportResponse {
            status,
            last_modified,
            body: Box::new(response),
        })
    }
}

#[derive(Debug, Serialize, Deserialize, Default)]
struct Metadata {
    files: HashMap<String, FileEntry>,
}

#[derive(Debug, Serialize, Deserialize, Clone)]
struct FileEntry {
    path: PathBuf,
    last_checked: DateTime<Utc>,
    last_modified: Option<DateTime<Utc>>,
    content_hash: String,
}

pub struct Downloader {
    cache_dir: PathBuf,
    metadata_path: PathBuf,
    metadata: OnceLock<RwLock<Metadata>>,
    transport: Arc<dyn HttpTransport>,
}

impl Downloader {
    pub fn new(cache_dir: impl AsRef<Path>) -> Self {
        Self::with_transport(cache_dir, Arc::new(ReqwestTransport::new()))
    }

    /// Create a downloader with a custom HTTP transport. Tests use this to
    /// inject an in-memory fake so no sockets are involved.
    pub fn with_transport(cache_dir: impl AsRef<Path>, transport: Arc<dyn HttpTransport>) -> Self {
        let cache_dir = cache_dir.as_ref().to_path_buf();
        info!("Cache folder at: {cache_dir:?}");

        let metadata_path = cache_dir.join(METADATA_FILE);

        Self {
            cache_dir,
            metadata_path,
            metadata: OnceLock::new(),
            transport,
        }
    }

    fn metadata(&self) -> &RwLock<Metadata> {
        let metadata_path = self.metadata_path.clone();
        let cache_dir = self.cache_dir.clone();
        self.metadata.get_or_init(move || {
            // Best-effort: if the directory can't be created, downloads fail
            // later with a real error instead of panicking here.
            if let Err(e) = fs::create_dir_all(&cache_dir) {
                error!(
                    "Failed to create cache directory {}: {e}",
                    cache_dir.display()
                );
            }
            let metadata = Self::load_metadata(&metadata_path);
            RwLock::new(metadata)
        })
    }

    fn load_metadata(metadata_path: &Path) -> Metadata {
        match File::open(metadata_path) {
            Ok(file) => match serde_json::from_reader(file) {
                Ok(metadata) => metadata,
                Err(err) => {
                    log::warn!("Failed to parse metadata file {metadata_path:?}: {err}");
                    Metadata::default()
                }
            },
            Err(err) => {
                if err.kind() != std::io::ErrorKind::NotFound {
                    log::warn!("Failed to open metadata file {metadata_path:?}: {err}");
                }
                Metadata::default()
            }
        }
    }

    fn persist_metadata(&self, metadata: &mut Metadata) -> Result<()> {
        self.persist_metadata_with_removals(metadata, &[])
    }

    /// Write metadata to disk atomically (temp file + rename). The cache dir
    /// is shared by every codebook process on the machine and each rewrites
    /// the whole file, so entries written by other processes since our last
    /// read are merged in first — persisting must never erase them.
    /// `removed_urls` are dropped after the merge so a deliberate removal
    /// isn't resurrected from disk.
    fn persist_metadata_with_removals(
        &self,
        metadata: &mut Metadata,
        removed_urls: &[&str],
    ) -> Result<()> {
        let disk = Self::load_metadata(&self.metadata_path);
        for (url, entry) in disk.files {
            metadata.files.entry(url).or_insert(entry);
        }
        for url in removed_urls {
            metadata.files.remove(*url);
        }

        let temp_file = NamedTempFile::new_in(&self.cache_dir)?;
        serde_json::to_writer_pretty(&temp_file, metadata)?;
        temp_file.persist(&self.metadata_path)?;
        Ok(())
    }

    fn purge_stale_entry(&self, url: &str, stale_path: &Path) {
        let mut metadata = self.metadata().write().unwrap();
        if metadata
            .files
            .get(url)
            .map(|entry| entry.path == stale_path)
            .unwrap_or(false)
        {
            metadata.files.remove(url);
            if let Err(err) = self.persist_metadata_with_removals(&mut metadata, &[url]) {
                log::error!(
                    "Failed to persist metadata after removing stale entry for {url}: {err}"
                );
            }
        }
    }

    pub fn get(&self, url: &str) -> Result<PathBuf> {
        let entry = {
            let metadata = self.metadata().read().unwrap();
            metadata.files.get(url).cloned()
        };

        let result = match entry {
            Some(entry) => {
                if !entry.path.exists() {
                    self.purge_stale_entry(url, &entry.path);
                    self.download_new(url)
                } else {
                    let needs_update =
                        entry.last_checked.timestamp() + TWO_WEEKS as i64 <= Utc::now().timestamp();
                    if needs_update {
                        self.try_update(url)
                    } else {
                        Ok(entry.path)
                    }
                }
            }
            None => self.download_new(url),
        };

        // On failure, fall back to a cached copy when one exists on disk
        // (e.g. offline revalidation). Every failing branch above was itself
        // a download attempt, so re-downloading here would only repeat it.
        result.or_else(|e| {
            let cached = {
                let metadata = self.metadata().read().unwrap();
                metadata
                    .files
                    .get(url)
                    .map(|file_info| file_info.path.clone())
            };
            match cached {
                Some(path) if path.exists() => {
                    log::error!("Failed to update, using cached version: {e}");
                    Ok(path)
                }
                _ => Err(e),
            }
        })
    }

    fn try_update(&self, url: &str) -> Result<PathBuf> {
        // Get last modified time with read lock
        let last_modified = {
            self.metadata()
                .read()
                .unwrap()
                .files
                .get(url)
                .and_then(|e| e.last_modified)
        };

        let if_modified_since = last_modified.map(|lm| lm.with_timezone(&Utc).to_rfc2822());
        let response = self.transport.get(url, if_modified_since.as_deref())?;

        match response.status {
            304 => self.update_check_time(url),
            200 => self.handle_updated_response(url, response),
            status => {
                let _ = self.update_check_time(url);
                Err(anyhow::anyhow!("Unexpected status code: {}", status))
            }
        }
    }

    fn handle_updated_response(&self, url: &str, response: TransportResponse) -> Result<PathBuf> {
        let last_modified = parse_last_modified(response.last_modified.as_deref());
        let temp_file = self.download_to_temp(response.body)?;
        let new_hash = compute_file_hash(temp_file.path())?;
        // The entry can disappear concurrently (another thread purging a
        // stale path); a missing entry just means "changed".
        let old_hash = {
            let metadata = self
                .metadata()
                .read()
                .map_err(|e| anyhow::anyhow!("Lock error: {}", e))?;
            metadata.files.get(url).map(|e| e.content_hash.clone())
        };
        if old_hash.as_deref() == Some(new_hash.as_str()) {
            self.update_check_time(url)
        } else {
            self.replace_file(url, temp_file, last_modified, new_hash)
        }
    }

    fn download_new(&self, url: &str) -> Result<PathBuf> {
        let max_retries = 2;
        let mut last_err = None;
        for attempt in 0..=max_retries {
            if attempt > 0 {
                std::thread::sleep(std::time::Duration::from_millis(500 * attempt as u64));
            }
            match self.try_download_new(url) {
                Ok(path) => return Ok(path),
                Err(e) => {
                    if e.downcast_ref::<PermanentHttpError>().is_some() {
                        log::warn!("Not retrying download for {url}: {e}");
                        return Err(e);
                    }
                    log::warn!(
                        "Download attempt {}/{} failed for {url}: {e}",
                        attempt + 1,
                        max_retries + 1
                    );
                    last_err = Some(e);
                }
            }
        }
        Err(last_err.unwrap())
    }

    fn try_download_new(&self, url: &str) -> Result<PathBuf> {
        let response = self.transport.get(url, None)?;
        let status = response.status;
        if (400..500).contains(&status) {
            return Err(anyhow::Error::new(PermanentHttpError {
                status,
                url: url.to_string(),
            }));
        }
        if !(200..300).contains(&status) {
            return Err(anyhow::anyhow!(
                "Download failed with status {status} for {url}"
            ));
        }
        let last_modified = parse_last_modified(response.last_modified.as_deref());
        let temp_file = self.download_to_temp(response.body)?;
        let new_hash = compute_file_hash(temp_file.path())?;
        self.store_new_file(url, temp_file, last_modified, new_hash)
    }

    fn download_to_temp(&self, mut body: Box<dyn Read>) -> Result<NamedTempFile> {
        let mut temp_file = NamedTempFile::new_in(&self.cache_dir)?;
        std::io::copy(&mut body, &mut temp_file)?;
        Ok(temp_file)
    }

    fn store_new_file(
        &self,
        url: &str,
        temp_file: NamedTempFile,
        last_modified: Option<DateTime<Utc>>,
        content_hash: String,
    ) -> Result<PathBuf> {
        let filename = hash_url(url);
        let path = self.cache_dir.join(filename);
        temp_file.persist(&path)?;

        let entry = FileEntry {
            path: path.clone(),
            last_checked: Utc::now(),
            last_modified,
            content_hash,
        };
        {
            let mut metadata = self.metadata().write().unwrap();
            metadata.files.insert(url.to_string(), entry);
            self.persist_metadata(&mut metadata)?;
        }
        Ok(path)
    }

    fn replace_file(
        &self,
        url: &str,
        temp_file: NamedTempFile,
        last_modified: Option<DateTime<Utc>>,
        content_hash: String,
    ) -> Result<PathBuf> {
        let new_path: PathBuf;
        {
            let mut metadata = self.metadata().write().unwrap();
            // Upsert: the entry can disappear concurrently (stale-path purge)
            let old_path = metadata.files.get(url).map(|e| e.path.clone());

            new_path = self.cache_dir.join(hash_url(url));
            temp_file.persist(&new_path)?;

            // Remove old file if it's different
            if let Some(old_path) = old_path
                && old_path != new_path
                && old_path.exists()
            {
                fs::remove_file(old_path)?;
            }

            let entry = FileEntry {
                path: new_path.clone(),
                last_checked: Utc::now(),
                last_modified,
                content_hash,
            };
            metadata.files.insert(url.to_string(), entry);
            self.persist_metadata(&mut metadata)?;
        }

        Ok(new_path)
    }

    fn update_check_time(&self, url: &str) -> Result<PathBuf> {
        let path: PathBuf;
        {
            let mut metadata = self.metadata().write().unwrap();
            let entry = metadata
                .files
                .get_mut(url)
                .ok_or_else(|| anyhow::anyhow!("No cache entry for {url}"))?;
            entry.last_checked = Utc::now();
            path = entry.path.clone();
            self.persist_metadata(&mut metadata)?;
        }
        Ok(path)
    }
}

fn hash_url(url: &str) -> String {
    let hash = Sha256::digest(url.as_bytes());
    lower::encode_string(&hash)
}

fn compute_file_hash(path: &Path) -> Result<String> {
    let file = File::open(path)?;
    let mut reader = BufReader::new(file);
    let mut hasher = Sha256::new();
    let mut buffer = [0; 1024];

    loop {
        let count = reader.read(&mut buffer)?;
        if count == 0 {
            break;
        }
        hasher.update(&buffer[..count]);
    }

    Ok(lower::encode_string(&hasher.finalize()))
}

fn parse_last_modified(header: Option<&str>) -> Option<DateTime<Utc>> {
    header
        .and_then(|s| DateTime::parse_from_rfc2822(s).ok())
        .map(|dt| dt.with_timezone(&Utc))
}

#[cfg(test)]
mod tests {
    use super::*;
    use chrono::Duration;
    use std::collections::VecDeque;
    use std::io::Cursor;
    use std::sync::Mutex;
    use tempfile::tempdir;

    const URL: &str = "https://example.com/test.txt";

    struct FakeResponse {
        status: u16,
        body: &'static str,
        last_modified: Option<&'static str>,
    }

    fn ok(status: u16, body: &'static str, last_modified: Option<&'static str>) -> ScriptedResult {
        Ok(FakeResponse {
            status,
            body,
            last_modified,
        })
    }

    fn connection_error() -> ScriptedResult {
        Err(anyhow::anyhow!("connection refused"))
    }

    type ScriptedResult = Result<FakeResponse>;

    /// In-memory HttpTransport: pops one scripted response per request and
    /// records every request. Panics if a request arrives with no scripted
    /// response left — an unexpected request is a test failure.
    struct FakeTransport {
        responses: Mutex<VecDeque<ScriptedResult>>,
        requests: Mutex<Vec<(String, Option<String>)>>,
    }

    impl FakeTransport {
        fn new(responses: Vec<ScriptedResult>) -> Arc<Self> {
            Arc::new(Self {
                responses: Mutex::new(responses.into()),
                requests: Mutex::new(Vec::new()),
            })
        }

        fn push(&self, response: ScriptedResult) {
            self.responses.lock().unwrap().push_back(response);
        }

        fn requests(&self) -> Vec<(String, Option<String>)> {
            self.requests.lock().unwrap().clone()
        }
    }

    impl HttpTransport for FakeTransport {
        fn get(&self, url: &str, if_modified_since: Option<&str>) -> Result<TransportResponse> {
            self.requests
                .lock()
                .unwrap()
                .push((url.to_string(), if_modified_since.map(String::from)));
            let next = self
                .responses
                .lock()
                .unwrap()
                .pop_front()
                .expect("unexpected HTTP request in test (no scripted response left)");
            next.map(|r| TransportResponse {
                status: r.status,
                last_modified: r.last_modified.map(String::from),
                body: Box::new(Cursor::new(r.body.as_bytes().to_vec())),
            })
        }
    }

    #[test]
    fn test_download_new_file() {
        // NB: chrono's RFC 2822 parser validates the weekday, so the date
        // must be a real Wednesday
        let transport = FakeTransport::new(vec![ok(
            200,
            "test content",
            Some("Wed, 18 Oct 2023 07:28:00 GMT"),
        )]);
        let temp_dir = tempdir().unwrap();
        let downloader = Downloader::with_transport(temp_dir.path(), transport.clone());

        let path = downloader.get(URL).unwrap();

        assert_eq!(transport.requests().len(), 1);
        assert!(path.exists());
        assert_eq!(std::fs::read_to_string(path).unwrap(), "test content");
        let metadata = downloader.metadata().read().unwrap();
        let entry = metadata.files.get(URL).unwrap();
        assert_eq!(entry.content_hash, compute_file_hash(&entry.path).unwrap());
        assert!(entry.last_modified.is_some());
    }

    #[test]
    #[should_panic(expected = "Blocked network request")]
    fn test_network_guard_blocks_real_transport() {
        let temp_dir = tempdir().unwrap();
        let downloader = Downloader::new(temp_dir.path());
        let _ = downloader.get("https://example.invalid/dict.txt");
    }

    #[test]
    fn test_404_fails_fast_without_retries() {
        let transport = FakeTransport::new(vec![ok(404, "", None)]);
        let temp_dir = tempdir().unwrap();
        let downloader = Downloader::with_transport(temp_dir.path(), transport.clone());

        let err = downloader.get(URL).unwrap_err();

        // A definitive 4xx must not be retried, and must be identifiable
        assert_eq!(transport.requests().len(), 1);
        let permanent = err
            .downcast_ref::<PermanentHttpError>()
            .expect("4xx should downcast to PermanentHttpError");
        assert_eq!(permanent.status, 404);
    }

    #[test]
    fn test_server_error_is_retried() {
        let transport = FakeTransport::new(vec![
            ok(500, "", None),
            ok(500, "", None),
            ok(500, "", None),
        ]);
        let temp_dir = tempdir().unwrap();
        let downloader = Downloader::with_transport(temp_dir.path(), transport.clone());

        let err = downloader.get(URL).unwrap_err();

        // Transient 5xx keeps the retry behavior and is not "permanent"
        assert_eq!(transport.requests().len(), 3);
        assert!(err.downcast_ref::<PermanentHttpError>().is_none());
    }

    #[test]
    fn test_returns_cached_file_within_two_weeks_without_request() {
        let transport = FakeTransport::new(vec![ok(200, "cached content", None)]);
        let temp_dir = tempdir().unwrap();
        let downloader = Downloader::with_transport(temp_dir.path(), transport.clone());
        let path = downloader.get(URL).unwrap();
        assert_eq!(transport.requests().len(), 1);

        // A fresh instance re-reads metadata from disk; the entry is recent,
        // so no request may happen (the fake would panic on one).
        let offline_transport = FakeTransport::new(vec![]);
        let downloader = Downloader::with_transport(temp_dir.path(), offline_transport.clone());
        let cached_path = downloader.get(URL).unwrap();

        assert_eq!(path, cached_path);
        assert_eq!(
            std::fs::read_to_string(cached_path).unwrap(),
            "cached content"
        );
        assert!(offline_transport.requests().is_empty());
    }

    #[test]
    fn test_updates_file_when_modified() {
        let initial_last_modified = "Wed, 21 Oct 2020 07:28:00 GMT";
        let transport = FakeTransport::new(vec![ok(200, "v1", Some(initial_last_modified))]);
        let temp_dir = tempdir().unwrap();
        let downloader = Downloader::with_transport(temp_dir.path(), transport.clone());
        let path_v1 = downloader.get(URL).unwrap();

        // Get stored metadata and age the entry past the revalidation window
        let stored_last_modified = {
            let metadata = downloader.metadata().read().unwrap();
            metadata.files[URL].last_modified
        };
        {
            let mut metadata = downloader.metadata().write().unwrap();
            let entry = metadata.files.get_mut(URL).unwrap();
            entry.last_checked = stored_last_modified.unwrap() - Duration::weeks(3);
        }

        transport.push(ok(200, "v2", Some("Fri, 23 Oct 2020 07:28:00 GMT")));
        let path_v2 = downloader.get(URL).unwrap();

        // The revalidation request must carry If-Modified-Since
        let requests = transport.requests();
        assert_eq!(requests.len(), 2);
        assert_eq!(
            requests[1].1.as_deref(),
            Some(stored_last_modified.unwrap().to_rfc2822().as_str())
        );
        assert_eq!(path_v1, path_v2);
        assert_eq!(std::fs::read_to_string(path_v2).unwrap(), "v2");
    }

    #[test]
    fn test_uses_stale_file_when_update_fails() {
        let transport = FakeTransport::new(vec![ok(200, "original", None)]);
        let temp_dir = tempdir().unwrap();
        let downloader = Downloader::with_transport(temp_dir.path(), transport.clone());
        let original_path = downloader.get(URL).unwrap();

        // Age the entry, then go "offline"
        {
            let mut metadata = downloader.metadata().write().unwrap();
            let entry = metadata.files.get_mut(URL).unwrap();
            entry.last_checked = Utc::now() - Duration::seconds(TWO_WEEKS as i64 * 2);
        }
        transport.push(connection_error());

        let cached_path = downloader.get(URL).unwrap();
        assert_eq!(original_path, cached_path);
        assert_eq!(std::fs::read_to_string(cached_path).unwrap(), "original");
    }

    #[test]
    fn test_doesnt_check_within_two_weeks() {
        let transport = FakeTransport::new(vec![ok(200, "content", None)]);
        let temp_dir = tempdir().unwrap();
        let downloader = Downloader::with_transport(temp_dir.path(), transport.clone());

        downloader.get(URL).unwrap();
        downloader.get(URL).unwrap();

        // The second get must be served from cache without a request
        assert_eq!(transport.requests().len(), 1);
    }

    #[test]
    fn test_handles_304_not_modified() {
        let last_modified = "Wed, 21 Oct 2020 07:28:00 GMT";
        let transport = FakeTransport::new(vec![ok(200, "content", Some(last_modified))]);
        let temp_dir = tempdir().unwrap();
        let downloader = Downloader::with_transport(temp_dir.path(), transport.clone());
        let original_path = downloader.get(URL).unwrap();

        // Age the entry past the revalidation window
        {
            let mut metadata = downloader.metadata().write().unwrap();
            let entry = metadata.files.get_mut(URL).unwrap();
            entry.last_checked = DateTime::parse_from_rfc2822(last_modified)
                .unwrap()
                .with_timezone(&Utc);
        }

        transport.push(ok(304, "", None));
        let cached_path = downloader.get(URL).unwrap();

        let requests = transport.requests();
        assert_eq!(requests.len(), 2);
        assert_eq!(
            requests[1].1.as_deref(),
            Some("Wed, 21 Oct 2020 07:28:00 +0000")
        );
        assert_eq!(original_path, cached_path);
        let metadata = downloader.metadata().read().unwrap();
        let entry = metadata.files.get(URL).unwrap();
        assert!(entry.last_checked > Utc::now() - Duration::seconds(1));
    }

    #[test]
    fn test_concurrent_processes_dont_clobber_metadata() {
        let temp_dir = tempdir().unwrap();
        let url_a = "https://example.com/a.txt";
        let url_b = "https://example.com/b.txt";

        // Two downloaders sharing the cache dir, simulating two processes.
        // B loads (empty) metadata before A writes, so B's in-memory view is
        // stale when it persists.
        let transport_a = FakeTransport::new(vec![ok(200, "a", None)]);
        let downloader_a = Downloader::with_transport(temp_dir.path(), transport_a);
        let transport_b = FakeTransport::new(vec![ok(200, "b", None)]);
        let downloader_b = Downloader::with_transport(temp_dir.path(), transport_b);
        let _ = downloader_b.metadata();

        downloader_a.get(url_a).unwrap();
        downloader_b.get(url_b).unwrap();

        // B's persist must not have erased A's entry from disk
        let disk = Downloader::load_metadata(&temp_dir.path().join(METADATA_FILE));
        assert!(disk.files.contains_key(url_a), "A's entry was clobbered");
        assert!(disk.files.contains_key(url_b));
    }

    #[test]
    fn test_file_hashing() {
        let url1 = "https://example.com/file1";
        let url2 = "https://example.com/file2";

        assert_ne!(hash_url(url1), hash_url(url2));

        let same_url = "https://example.com/same";
        assert_eq!(hash_url(same_url), hash_url(same_url));
    }

    #[test]
    fn test_redownloads_when_file_missing() {
        let transport = FakeTransport::new(vec![ok(200, "content", None)]);
        let temp_dir = tempdir().unwrap();
        let downloader = Downloader::with_transport(temp_dir.path(), transport.clone());
        let path = downloader.get(URL).unwrap();

        // Simulate file deletion but keep metadata
        std::fs::remove_file(&path).unwrap();
        assert!(!path.exists());

        transport.push(ok(200, "redownloaded content", None));
        let new_path = downloader.get(URL).unwrap();

        assert_eq!(transport.requests().len(), 2);
        assert!(new_path.exists());
        assert_eq!(
            std::fs::read_to_string(&new_path).unwrap(),
            "redownloaded content"
        );
    }
}