domain-status 0.2.1

Concurrent URL status checker that captures comprehensive metadata in SQLite.
Documentation
//! GitHub directory fetching operations.

use anyhow::{Context, Result};
use reqwest::Client;
use serde::Deserialize;
use std::collections::HashMap;

use crate::config::MAX_RULESET_DOWNLOAD_SIZE;
use crate::fingerprint::models::Technology;
use crate::security::validate_url_safe;

/// Fetches all JSON files from a GitHub directory and merges them
#[allow(clippy::too_many_lines)] // Sequential GitHub API: list directory, fetch each JSON, parse and merge into ruleset
pub(crate) async fn fetch_from_github_directory(
    dir_url: &str,
    client: &Client,
) -> Result<HashMap<String, Technology>> {
    // Convert raw.githubusercontent.com URL to GitHub API URL
    // e.g., https://raw.githubusercontent.com/HTTPArchive/wappalyzer/main/src/technologies
    // -> https://api.github.com/repos/HTTPArchive/wappalyzer/contents/src/technologies
    let (api_url, branch) = if dir_url.contains("raw.githubusercontent.com") {
        // Extract: owner/repo/branch/path
        // e.g., raw.githubusercontent.com/HTTPArchive/wappalyzer/main/src/technologies
        let parts: Vec<&str> = dir_url.split('/').collect();
        if parts.len() >= 7 {
            let owner = parts[3];
            let repo = parts[4];
            let branch = parts.get(5).copied().unwrap_or("main"); // Extract branch, default to "main"
            let path = parts[6..].join("/");
            let constructed_url =
                format!("https://api.github.com/repos/{owner}/{repo}/contents/{path}");
            // Validate constructed URL (defense in depth)
            validate_url_safe(&constructed_url)
                .with_context(|| format!("Unsafe GitHub API URL constructed: {constructed_url}"))?;
            (constructed_url, branch)
        } else {
            return Err(anyhow::anyhow!(
                "Invalid GitHub URL format. Expected: \
                'https://raw.githubusercontent.com/owner/repo/branch/path', got: '{dir_url}'"
            ));
        }
    } else {
        // Already an API URL or different format - validate it
        validate_url_safe(dir_url).with_context(|| format!("Unsafe GitHub API URL: {dir_url}"))?;
        (dir_url.to_string(), "main") // Default branch if not specified
    };

    log::info!("Fetching technology files from GitHub directory: {api_url}");

    // Fetch directory listing with branch reference
    let api_url_with_ref = format!("{api_url}?ref={branch}");
    log::debug!("Fetching directory listing from: {api_url_with_ref} (branch: {branch})");

    let response = super::github_api_get(client, &api_url_with_ref).await?;

    if !response.status().is_success() {
        let status = response.status();
        let error_text = response
            .text()
            .await
            .unwrap_or_else(|_| "Unknown error".to_string());

        // Provide helpful error message for rate limits
        if status.as_u16() == 403 && error_text.contains("rate limit") {
            return Err(anyhow::anyhow!(
                "GitHub API rate limit exceeded. \
                Unauthenticated requests are limited to 60 requests/hour. \
                To increase the limit to 5000 requests/hour, set the GITHUB_TOKEN environment variable. \
                \
                Get a token at: https://github.com/settings/tokens \
                (no special permissions needed for public repositories). \
                \
                Alternatively, use a cached ruleset or wait before retrying. \
                \
                Error: {status} - {error_text}"
            ));
        }

        return Err(anyhow::anyhow!(
            "Failed to fetch directory listing: {status} - {error_text}"
        ));
    }

    #[derive(Deserialize)]
    struct FileEntry {
        name: String,
        download_url: Option<String>,
        #[serde(rename = "type")]
        file_type: String,
    }

    let json_text = response.text().await?;
    let entries: Vec<FileEntry> =
        serde_json::from_str(&json_text).context("Failed to parse GitHub API response")?;

    // Filter for JSON files only
    let json_files: Vec<_> = entries
        .into_iter()
        .filter(|f| {
            f.file_type == "file"
                && std::path::Path::new(&f.name)
                    .extension()
                    .is_some_and(|ext| ext.eq_ignore_ascii_case("json"))
        })
        .collect();

    log::info!("Found {} JSON files to fetch", json_files.len());

    // Fetch all JSON files in parallel with SSRF protection and size limits
    let mut tasks = Vec::new();
    for file in json_files {
        if let Some(download_url) = file.download_url {
            // SSRF protection: validate download URL from GitHub API
            // GitHub API should only return raw.githubusercontent.com URLs, but validate to be safe
            if let Err(e) = validate_url_safe(&download_url) {
                log::warn!("Skipping unsafe download URL from GitHub API: {download_url} - {e}");
                continue;
            }

            // Additional check: ensure it's actually a GitHub URL (defense in depth)
            if !download_url.starts_with("https://raw.githubusercontent.com/") {
                log::warn!(
                    "Suspicious download URL from GitHub API (not raw.githubusercontent.com): {download_url}"
                );
                continue;
            }

            let client = client.clone();
            tasks.push(tokio::spawn(async move {
                match client.get(&download_url).send().await {
                    Ok(resp) => {
                        if resp.status().is_success() {
                            // Check content-length header if available
                            if let Some(content_length) = resp.content_length() {
                                if content_length > MAX_RULESET_DOWNLOAD_SIZE as u64 {
                                    log::warn!(
                                        "File {download_url} too large ({content_length} bytes, max: {MAX_RULESET_DOWNLOAD_SIZE}), skipping"
                                    );
                                    return None;
                                }
                            }

                            match resp.text().await {
                                Ok(text) => {
                                    // Double-check size after download (in case content-length was missing/wrong)
                                    if text.len() > MAX_RULESET_DOWNLOAD_SIZE {
                                        log::warn!(
                                            "File {} too large after download ({} bytes, max: {}), skipping",
                                            download_url,
                                            text.len(),
                                            MAX_RULESET_DOWNLOAD_SIZE
                                        );
                                        return None;
                                    }

                                    match serde_json::from_str::<HashMap<String, Technology>>(&text)
                                    {
                                        Ok(techs) => Some(techs),
                                        Err(e) => {
                                            log::warn!("Failed to parse {download_url}: {e}");
                                            None
                                        }
                                    }
                                }
                                Err(e) => {
                                    log::warn!("Failed to read {download_url}: {e}");
                                    None
                                }
                            }
                        } else {
                            log::warn!("Failed to fetch {}: {}", download_url, resp.status());
                            None
                        }
                    }
                    Err(e) => {
                        log::warn!("Failed to request {download_url}: {e}");
                        None
                    }
                }
            }));
        }
    }

    // Collect all results and merge
    let mut all_technologies = HashMap::new();
    for task in tasks {
        if let Ok(Some(techs)) = task.await {
            all_technologies.extend(techs);
        }
    }

    log::info!(
        "Successfully loaded {} technologies",
        all_technologies.len()
    );
    Ok(all_technologies)
}

#[cfg(test)]
mod tests {
    use super::*;
    use reqwest::Client;

    fn create_test_client() -> Client {
        Client::builder()
            .timeout(std::time::Duration::from_secs(5))
            .build()
            .expect("Failed to create test client")
    }

    #[tokio::test]
    async fn test_fetch_from_github_directory_invalid_url() {
        let client = create_test_client();
        let result = fetch_from_github_directory("not-a-valid-url", &client).await;
        assert!(result.is_err());
    }

    #[tokio::test]
    async fn test_fetch_from_github_directory_short_url() {
        let client = create_test_client();
        let result =
            fetch_from_github_directory("https://raw.githubusercontent.com/owner", &client).await;
        assert!(result.is_err());
        let error_msg = result.unwrap_err().to_string();
        assert!(
            error_msg.contains("Invalid GitHub URL format"),
            "Expected invalid URL format error, got: {}",
            error_msg
        );
    }

    #[tokio::test]
    async fn test_fetch_from_github_directory_ssrf_protection() {
        let client = create_test_client();
        // Test that SSRF protection blocks private IPs
        // This would be caught when constructing the API URL
        let result = fetch_from_github_directory("http://192.168.1.1/path", &client).await;
        assert!(result.is_err());
        let error_msg = result.unwrap_err().to_string();
        assert!(
            error_msg.contains("Unsafe") || error_msg.contains("private"),
            "Expected SSRF protection error, got: {}",
            error_msg
        );
    }

    #[tokio::test]
    async fn test_fetch_from_github_directory_nonexistent_repo() {
        let client = create_test_client();
        let url = "https://raw.githubusercontent.com/nonexistent/repo/main/path";
        let result = fetch_from_github_directory(url, &client).await;
        // Should fail with 404 or similar
        assert!(result.is_err());
    }
}