domain-status 0.2.1

Concurrent URL status checker that captures comprehensive metadata in SQLite.
Documentation
//! URL fetching operations for fingerprint rulesets.
//!
//! This module handles fetching technologies and categories from URLs,
//! including GitHub directory fetching.
//!
//! Security features:
//! - URL validation (SSRF protection)
//! - Size limits to prevent `DoS`
//! - Retry logic for transient failures

use anyhow::{Context, Result};
use std::collections::HashMap;
use std::time::Duration;

use crate::config::MAX_RULESET_DOWNLOAD_SIZE;
use crate::fingerprint::models::Technology;
use crate::security::validate_url_safe;
use crate::utils::retry::with_network_download_retry;

use super::github::fetch_from_github_directory;

/// Fetches technologies from a URL (handles both single file and directory)
///
/// # Security
///
/// This function validates URLs to prevent SSRF attacks and enforces size limits
/// to prevent `DoS` attacks via extremely large files.
pub(crate) async fn fetch_from_url(url: &str) -> Result<HashMap<String, Technology>> {
    // SSRF protection: validate URL before fetching
    validate_url_safe(url).with_context(|| format!("Unsafe ruleset URL rejected: {url}"))?;

    let client = crate::initialization::build_download_client(Duration::from_secs(60))?;

    // Check if URL points to a directory (GitHub) or a file
    // For HTTP Archive, we need to fetch all JSON files from the directory
    // raw.githubusercontent.com URLs that don't end in .json are directories
    if url.contains("raw.githubusercontent.com")
        && !std::path::Path::new(url)
            .extension()
            .is_some_and(|ext| ext.eq_ignore_ascii_case("json"))
    {
        // It's a directory - fetch all JSON files via GitHub API
        log::debug!("Detected GitHub directory URL, fetching via API");
        return fetch_from_github_directory(url, &client).await;
    }

    // Single file - fetch with retries and size limits
    log::debug!("Fetching single file from: {url}");

    with_network_download_retry(
        &format!("fetch ruleset from {url}"),
        1, // Exponential backoff: 1s, 2s, 4s
        || fetch_single_file_with_size_limit(&client, url),
    )
    .await
}

/// Fetches a single file with size limit enforcement
async fn fetch_single_file_with_size_limit(
    client: &reqwest::Client,
    url: &str,
) -> Result<HashMap<String, Technology>> {
    use crate::fetch::stream::reject_if_content_length_exceeds;

    let response = client.get(url).send().await?;

    if !response.status().is_success() {
        return Err(anyhow::anyhow!(
            "Failed to fetch {}: {}",
            url,
            response.status()
        ));
    }

    reject_if_content_length_exceeds(&response, MAX_RULESET_DOWNLOAD_SIZE, "Ruleset file")?;

    // Read response with size limit
    let bytes = response.bytes().await?;

    if bytes.len() > MAX_RULESET_DOWNLOAD_SIZE {
        return Err(anyhow::anyhow!(
            "Ruleset file too large: {} bytes (max: {} bytes)",
            bytes.len(),
            MAX_RULESET_DOWNLOAD_SIZE
        ));
    }

    let json_text =
        String::from_utf8(bytes.to_vec()).context("Ruleset file contains invalid UTF-8")?;

    // Parse as a map of technology name -> Technology
    let technologies: HashMap<String, Technology> =
        serde_json::from_str(&json_text).context("Failed to parse technologies JSON")?;

    Ok(technologies)
}

#[cfg(test)]
mod tests {
    use super::*;
    use httptest::{matchers::*, responders::*, Expectation, Server};

    #[tokio::test]
    async fn test_fetch_from_url_blocks_private_ip() {
        // Test that fetching from private IP is blocked (SSRF protection)
        let result = fetch_from_url("http://127.0.0.1/technologies.json").await;
        assert!(result.is_err());
        assert!(result
            .unwrap_err()
            .to_string()
            .contains("Unsafe ruleset URL rejected"));
    }

    #[tokio::test]
    async fn test_fetch_from_url_blocks_localhost() {
        // Test that fetching from localhost is blocked
        let result = fetch_from_url("http://localhost/technologies.json").await;
        assert!(result.is_err());
        assert!(result
            .unwrap_err()
            .to_string()
            .contains("Unsafe ruleset URL rejected"));
    }

    #[tokio::test]
    async fn test_fetch_from_url_blocks_unsafe_scheme() {
        // Test that unsafe schemes are blocked
        let result = fetch_from_url("file:///etc/passwd").await;
        assert!(result.is_err());
        assert!(result
            .unwrap_err()
            .to_string()
            .contains("Unsafe ruleset URL rejected"));
    }

    #[tokio::test]
    async fn test_fetch_from_url_allows_public_urls() {
        // Note: This test would require a real public URL since httptest uses localhost
        // which is blocked by SSRF protection. Instead, we test URL validation separately
        // and the fetch logic is tested via integration tests with real URLs.
        // This test verifies that the URL validation is working correctly.

        // Test that a public URL format is accepted by validation
        use crate::security::validate_url_safe;
        assert!(validate_url_safe("https://example.com/technologies.json").is_ok());
        assert!(validate_url_safe("http://8.8.8.8/technologies.json").is_ok()); // Public IP
    }

    #[tokio::test]
    async fn test_fetch_single_file_with_size_limit_enforces_max_size() {
        // Test that files exceeding size limit are rejected
        let server = Server::run();
        let url = server.url("/large.json").to_string();

        // Create a response larger than MAX_RULESET_DOWNLOAD_SIZE
        let large_body = vec![0u8; MAX_RULESET_DOWNLOAD_SIZE + 1];

        server.expect(
            Expectation::matching(request::method_path("GET", "/large.json")).respond_with(
                status_code(200)
                    .insert_header(
                        "Content-Length",
                        (MAX_RULESET_DOWNLOAD_SIZE + 1).to_string(),
                    )
                    .body(large_body),
            ),
        );

        let client = reqwest::Client::builder()
            .timeout(Duration::from_secs(60))
            .build()
            .unwrap();

        let result = fetch_single_file_with_size_limit(&client, &url).await;
        assert!(result.is_err());
        assert!(result.unwrap_err().to_string().contains("too large"));
    }

    #[tokio::test]
    async fn test_fetch_single_file_content_length_mismatch() {
        // Test that actual body size is checked even if content-length header is valid
        // This is critical - servers might send incorrect content-length headers
        // The code at line 106 checks actual bytes.len() after reading
        // Note: Hyper will panic if content-length header doesn't match body, so we test
        // the case where content-length is missing but body is too large
        let server = Server::run();
        let url = server.url("/mismatch.json").to_string();

        // Server sends no content-length header but body is too large
        let large_body = vec![0u8; MAX_RULESET_DOWNLOAD_SIZE + 1];

        server.expect(
            Expectation::matching(request::method_path("GET", "/mismatch.json")).respond_with(
                status_code(200)
                    // No Content-Length header - body size will be checked after reading
                    .body(large_body),
            ),
        );

        let client = reqwest::Client::builder()
            .timeout(Duration::from_secs(60))
            .build()
            .unwrap();

        let result = fetch_single_file_with_size_limit(&client, &url).await;
        assert!(result.is_err());
        assert!(result.unwrap_err().to_string().contains("too large"));
    }

    #[test]
    fn test_retry_exponential_backoff_calculation() {
        // Test that exponential backoff calculation is correct
        // The code at line 61 uses 1 << (attempt - 1) for backoff
        // This is critical - ensures we don't hammer failing servers
        // attempt 1: 1 << 0 = 1 second
        // attempt 2: 1 << 1 = 2 seconds
        // attempt 3: 1 << 2 = 4 seconds
        assert_eq!(1 << (1 - 1), 1);
        assert_eq!(1 << (2 - 1), 2);
        assert_eq!(1 << (3 - 1), 4);
    }

    #[tokio::test]
    async fn test_fetch_single_file_invalid_utf8() {
        // Test that invalid UTF-8 in response body is handled gracefully
        // This is critical - prevents panics from malformed responses
        let server = Server::run();
        let url = server.url("/invalid-utf8.json").to_string();

        // Send invalid UTF-8 bytes
        let invalid_utf8 = vec![0xFF, 0xFE, 0xFD];

        server.expect(
            Expectation::matching(request::method_path("GET", "/invalid-utf8.json")).respond_with(
                status_code(200)
                    .insert_header("Content-Length", "3")
                    .body(invalid_utf8),
            ),
        );

        let client = reqwest::Client::builder()
            .timeout(Duration::from_secs(60))
            .build()
            .unwrap();

        let result = fetch_single_file_with_size_limit(&client, &url).await;
        assert!(result.is_err());
        assert!(result.unwrap_err().to_string().contains("invalid UTF-8"));
    }

    #[tokio::test]
    async fn test_fetch_single_file_invalid_json() {
        // Test that invalid JSON is handled gracefully
        // This is critical - prevents panics from malformed ruleset files
        let server = Server::run();
        let url = server.url("/invalid.json").to_string();

        let invalid_json = "{invalid json}";
        server.expect(
            Expectation::matching(request::method_path("GET", "/invalid.json")).respond_with(
                status_code(200)
                    .insert_header("Content-Length", invalid_json.len().to_string()) // Match actual body size
                    .body(invalid_json),
            ),
        );

        let client = reqwest::Client::builder()
            .timeout(Duration::from_secs(60))
            .build()
            .unwrap();

        let result = fetch_single_file_with_size_limit(&client, &url).await;
        assert!(result.is_err());
        assert!(result
            .unwrap_err()
            .to_string()
            .contains("parse technologies JSON"));
    }
}