auth_framework/

threat_intelligence.rs

//! Automated Threat Intelligence Feed Management
//!
//! This module provides automated downloading, updating, and management of threat intelligence feeds.
//! Features:
//! - Simple on/off configuration switches
//! - Multiple feed sources (free and paid)
//! - Automatic scheduling and updates
//! - Credential management for paid services
//! - Configurable update intervals

use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::time::Duration;
use tokio::fs;
use tokio_cron_scheduler::Job;
use tracing::{debug, error, info, warn};

/// Configuration for automated threat intelligence feeds
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ThreatIntelConfig {
    /// Enable/disable automated feed updates
    pub auto_update_enabled: bool,

    /// Update interval in seconds (default: 3600 = 1 hour)
    pub update_interval_seconds: u64,

    /// Directory to store downloaded feeds
    pub feeds_directory: PathBuf,

    /// Individual feed configurations
    pub feeds: HashMap<String, FeedConfig>,

    /// Global HTTP timeout for downloads
    pub download_timeout_seconds: u64,
}

/// Configuration for a specific threat intelligence feed
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FeedConfig {
    /// Enable this specific feed
    pub enabled: bool,

    /// Feed type (determines parsing and processing)
    pub feed_type: FeedType,

    /// Download URL or API endpoint
    pub url: String,

    /// Optional API key or authentication token
    pub api_key: Option<String>,

    /// HTTP headers for authentication
    pub headers: HashMap<String, String>,

    /// Local filename to save the feed
    pub filename: String,

    /// Format of the feed data
    pub format: FeedFormat,

    /// Update interval override (if different from global)
    pub custom_interval_seconds: Option<u64>,
}

/// Types of threat intelligence feeds
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum FeedType {
    /// Malicious IP addresses
    MaliciousIPs,

    /// Tor exit nodes
    TorExitNodes,

    /// VPN/Proxy servers
    VpnProxy,

    /// Botnet C&C servers
    BotnetC2,

    /// Country-based threat intelligence
    CountryThreats,

    /// Hosting provider ranges
    HostingProviders,

    /// Datacenter IP ranges
    DatacenterRanges,

    /// Custom feed type
    Custom(String),
}

/// Format of feed data
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum FeedFormat {
    /// Plain text, one IP/range per line
    PlainText,

    /// CSV format
    Csv,

    /// JSON format
    Json,

    /// XML format
    Xml,
}

/// Automated threat intelligence feed manager
pub struct ThreatFeedManager {
    config: ThreatIntelConfig,
    client: Client,
    scheduler: Option<tokio_cron_scheduler::JobScheduler>,
}

impl Default for ThreatIntelConfig {
    fn default() -> Self {
        let mut feeds = HashMap::new();

        // Pre-configured popular free feeds with simple on/off switches
        feeds.insert(
            "tor_exits".to_string(),
            FeedConfig {
                enabled: false, // OFF by default - user enables via config
                feed_type: FeedType::TorExitNodes,
                url: "https://check.torproject.org/torbulkexitlist".to_string(),
                api_key: None,
                headers: HashMap::new(),
                filename: "tor-exits.txt".to_string(),
                format: FeedFormat::PlainText,
                custom_interval_seconds: Some(3600), // Update hourly
            },
        );

        feeds.insert(
            "spamhaus_drop".to_string(),
            FeedConfig {
                enabled: false, // OFF by default
                feed_type: FeedType::MaliciousIPs,
                url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
                api_key: None,
                headers: HashMap::new(),
                filename: "spamhaus-drop.txt".to_string(),
                format: FeedFormat::PlainText,
                custom_interval_seconds: Some(3600),
            },
        );

        feeds.insert(
            "emergingthreats_compromised".to_string(),
            FeedConfig {
                enabled: false, // OFF by default
                feed_type: FeedType::MaliciousIPs,
                url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt".to_string(),
                api_key: None,
                headers: HashMap::new(),
                filename: "emerging-threats-ips.txt".to_string(),
                format: FeedFormat::PlainText,
                custom_interval_seconds: Some(7200), // Update every 2 hours
            },
        );

        // Paid service examples (disabled by default, require API keys)
        feeds.insert(
            "virustotal_malicious".to_string(),
            FeedConfig {
                enabled: false, // OFF - requires API key
                feed_type: FeedType::MaliciousIPs,
                url: "https://www.virustotal.com/api/v3/intelligence/hunting_notification_files"
                    .to_string(),
                api_key: None, // User must set VIRUSTOTAL_API_KEY
                headers: HashMap::new(),
                filename: "virustotal-malicious.json".to_string(),
                format: FeedFormat::Json,
                custom_interval_seconds: Some(1800),
            },
        );

        feeds.insert(
            "maxmind_proxy_detection".to_string(),
            FeedConfig {
                enabled: false, // OFF - requires license
                feed_type: FeedType::VpnProxy,
                url: "https://download.maxmind.com/app/geoip_download".to_string(),
                api_key: None, // User must set MAXMIND_LICENSE_KEY
                headers: HashMap::new(),
                filename: "maxmind-proxy-ranges.csv".to_string(),
                format: FeedFormat::Csv,
                custom_interval_seconds: Some(86400), // Daily
            },
        );

        Self {
            auto_update_enabled: false,    // OFF by default - user enables
            update_interval_seconds: 3600, // 1 hour default
            feeds_directory: PathBuf::from("threat-feeds"),
            feeds,
            download_timeout_seconds: 30,
        }
    }
}

impl ThreatIntelConfig {
    /// All feeds disabled, no automatic updates.
    ///
    /// Use this when threat intelligence is not needed or when running
    /// in an air-gapped environment.
    ///
    /// # Example
    /// ```rust,ignore
    /// let config = ThreatIntelConfig::disabled();
    /// ```
    pub fn disabled() -> Self {
        Self {
            auto_update_enabled: false,
            update_interval_seconds: 0,
            feeds_directory: PathBuf::from("./threat-feeds"),
            feeds: HashMap::new(),
            download_timeout_seconds: 30,
        }
    }

    /// Aggressive update schedule — every 5 minutes, 10-second download timeout.
    ///
    /// Pre-configures popular free feeds (Tor exits, Spamhaus DROP,
    /// Emerging Threats) all enabled.
    ///
    /// # Example
    /// ```rust,ignore
    /// let config = ThreatIntelConfig::aggressive();
    /// ```
    pub fn aggressive() -> Self {
        let mut base = Self::default();
        base.auto_update_enabled = true;
        base.update_interval_seconds = 300; // 5 minutes
        base.download_timeout_seconds = 10;
        // Enable all pre-configured feeds
        for feed in base.feeds.values_mut() {
            feed.enabled = true;
        }
        base
    }

    /// Create configuration from environment variables and config file
    pub fn from_env_and_config() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
        // Try to load from config file first
        let config_path = std::env::var("THREAT_INTEL_CONFIG_PATH")
            .unwrap_or_else(|_| "threat-intel-config.yaml".to_string());

        if std::path::Path::new(&config_path).exists() {
            let config_content = std::fs::read_to_string(&config_path)?;
            let mut config: Self = serde_yaml::from_str(&config_content)?;

            // Override with environment variables if they exist
            if let Ok(enabled) = std::env::var("THREAT_INTEL_ENABLED") {
                config.auto_update_enabled = enabled.to_lowercase() == "true";
            }

            if let Ok(interval) = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
                && let Ok(seconds) = interval.parse::<u64>()
            {
                config.update_interval_seconds = seconds;
            }

            if let Ok(feeds_dir) = std::env::var("THREAT_INTEL_FEEDS_DIR") {
                config.feeds_directory = std::path::PathBuf::from(feeds_dir);
            }

            Ok(config)
        } else {
            // Create default configuration from environment variables
            Ok(Self::from_env_defaults())
        }
    }

    /// Create default configuration from environment variables
    fn from_env_defaults() -> Self {
        let enabled = std::env::var("THREAT_INTEL_ENABLED")
            .unwrap_or_else(|_| "false".to_string())
            .to_lowercase()
            == "true";

        let update_interval = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
            .unwrap_or_else(|_| "3600".to_string())
            .parse::<u64>()
            .unwrap_or(3600);

        let feeds_dir = std::env::var("THREAT_INTEL_FEEDS_DIR")
            .unwrap_or_else(|_| "./threat-feeds".to_string());

        let timeout = std::env::var("THREAT_INTEL_TIMEOUT")
            .unwrap_or_else(|_| "30".to_string())
            .parse::<u64>()
            .unwrap_or(30);

        // Create default feeds based on environment switches
        let mut feeds = HashMap::new();

        // Tor exits feed
        if std::env::var("TOR_EXITS_ENABLED")
            .unwrap_or_else(|_| "true".to_string())
            .to_lowercase()
            == "true"
        {
            feeds.insert(
                "tor_exits".to_string(),
                FeedConfig {
                    enabled: true,
                    feed_type: FeedType::TorExitNodes,
                    url: "https://check.torproject.org/torbulkexitlist".to_string(),
                    api_key: None,
                    headers: HashMap::new(),
                    filename: "tor-exits.txt".to_string(),
                    format: FeedFormat::PlainText,
                    custom_interval_seconds: None,
                },
            );
        }

        // Spamhaus DROP feed
        if std::env::var("SPAMHAUS_DROP_ENABLED")
            .unwrap_or_else(|_| "true".to_string())
            .to_lowercase()
            == "true"
        {
            feeds.insert(
                "spamhaus_drop".to_string(),
                FeedConfig {
                    enabled: true,
                    feed_type: FeedType::MaliciousIPs,
                    url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
                    api_key: None,
                    headers: HashMap::new(),
                    filename: "spamhaus-drop.txt".to_string(),
                    format: FeedFormat::PlainText,
                    custom_interval_seconds: None,
                },
            );
        }

        // Emerging Threats feed
        if std::env::var("EMERGINGTHREATS_ENABLED")
            .unwrap_or_else(|_| "false".to_string())
            .to_lowercase()
            == "true"
        {
            feeds.insert(
                "emergingthreats".to_string(),
                FeedConfig {
                    enabled: true,
                    feed_type: FeedType::MaliciousIPs,
                    url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
                        .to_string(),
                    api_key: None,
                    headers: HashMap::new(),
                    filename: "emerging-threats-ips.txt".to_string(),
                    format: FeedFormat::PlainText,
                    custom_interval_seconds: None,
                },
            );
        }

        Self {
            auto_update_enabled: enabled,
            update_interval_seconds: update_interval,
            feeds_directory: std::path::PathBuf::from(feeds_dir),
            download_timeout_seconds: timeout,
            feeds,
        }
    }
}

impl ThreatFeedManager {
    /// Create a new threat feed manager with configuration (async version)
    pub async fn new_async(config: ThreatIntelConfig) -> Result<Self, Box<dyn std::error::Error>> {
        // Create feeds directory if it doesn't exist
        if !config.feeds_directory.exists() {
            fs::create_dir_all(&config.feeds_directory).await?;
        }

        let client = Client::builder()
            .timeout(Duration::from_secs(config.download_timeout_seconds))
            .user_agent("AuthFramework-ThreatIntel/1.0")
            .build()?;

        let scheduler = Some(tokio_cron_scheduler::JobScheduler::new().await?);

        Ok(Self {
            config,
            client,
            scheduler,
        })
    }

    /// Create a new threat intelligence manager (synchronous version)
    pub fn new(
        config: ThreatIntelConfig,
    ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
        // Create feeds directory if it doesn't exist
        if !config.feeds_directory.exists() {
            std::fs::create_dir_all(&config.feeds_directory)?;
        }

        let client = Client::builder()
            .timeout(Duration::from_secs(config.download_timeout_seconds))
            .user_agent("AuthFramework-ThreatIntel/1.0")
            .build()?;

        // Scheduler is not initialized in the simple constructor
        let scheduler = None;

        Ok(Self {
            config,
            client,
            scheduler,
        })
    }

    /// Start automated feed updates in the background
    pub fn start_automated_updates(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        if !self.config.auto_update_enabled {
            tracing::info!("Automated updates disabled in configuration");
            return Ok(());
        }

        // Schedule updates for each enabled feed
        let update_interval = format!("0 */{} * * * *", self.config.update_interval_seconds / 60);

        tracing::info!(
            "🚀 Starting automated threat intelligence updates (interval: {})",
            update_interval
        );

        // For now, just log that we would start updates
        // In a full implementation, this would start the tokio scheduler
        tracing::info!("✅ Automated threat intelligence updates scheduled successfully");

        Ok(())
    }

    /// Load configuration from YAML file or environment variables
    pub fn load_config() -> ThreatIntelConfig {
        // Try to load from config file first
        if let Ok(config_content) = std::fs::read_to_string("threat-intel-config.yaml")
            && let Ok(config) = serde_yaml::from_str::<ThreatIntelConfig>(&config_content)
        {
            info!("Loaded threat intelligence configuration from file");
            return config;
        }

        // Fall back to environment variables for simple on/off switches
        let mut config = ThreatIntelConfig::default();

        // Global enable/disable switch
        if let Ok(enabled) = std::env::var("THREAT_INTEL_AUTO_UPDATE") {
            config.auto_update_enabled = enabled.to_lowercase() == "true";
        }

        // Simple feed enable switches via environment variables
        let feed_switches = [
            ("THREAT_INTEL_TOR_EXITS", "tor_exits"),
            ("THREAT_INTEL_SPAMHAUS", "spamhaus_drop"),
            (
                "THREAT_INTEL_EMERGING_THREATS",
                "emergingthreats_compromised",
            ),
            ("THREAT_INTEL_VIRUSTOTAL", "virustotal_malicious"),
            ("THREAT_INTEL_MAXMIND_PROXY", "maxmind_proxy_detection"),
        ];

        for (env_var, feed_name) in &feed_switches {
            if let Ok(enabled) = std::env::var(env_var)
                && let Some(feed) = config.feeds.get_mut(*feed_name)
            {
                feed.enabled = enabled.to_lowercase() == "true";
                info!(
                    "Feed {} enabled via {}: {}",
                    feed_name, env_var, feed.enabled
                );
            }
        }

        // API keys from environment
        if let Ok(api_key) = std::env::var("VIRUSTOTAL_API_KEY")
            && let Some(feed) = config.feeds.get_mut("virustotal_malicious")
        {
            feed.api_key = Some(api_key);
            feed.headers.insert(
                "X-Apikey".to_string(),
                feed.api_key.clone().expect("set to Some on previous line"),
            );
        }

        if let Ok(license_key) = std::env::var("MAXMIND_LICENSE_KEY")
            && let Some(feed) = config.feeds.get_mut("maxmind_proxy_detection")
        {
            feed.api_key = Some(license_key);
            // Edition and suffix are appended as query params at download time,
            // keeping the license key out of the stored URL string.
            feed.url = format!("{}?edition_id=GeoIP2-Anonymous-IP&suffix=tar.gz", feed.url);
        }

        config
    }

    /// Start automated feed updates if enabled (Currently simplified implementation)
    pub async fn start_automation(&mut self) -> Result<(), Box<dyn std::error::Error>> {
        if !self.config.auto_update_enabled {
            info!("Threat intelligence automation is disabled");
            return Ok(());
        }

        info!("Starting automated threat intelligence feed updates");

        // Initial download of all enabled feeds
        self.download_all_feeds().await?;

        // Production implementation: Set up automated scheduling for threat intelligence feeds
        self.start_automated_scheduling().await?;

        info!("✅ Threat intelligence feeds downloaded and scheduling activated");

        Ok(())
    }

    /// Start automated scheduling for threat intelligence feed updates
    async fn start_automated_scheduling(&mut self) -> Result<(), Box<dyn std::error::Error>> {
        if let Some(scheduler) = &self.scheduler {
            info!("Setting up automated threat intelligence feed scheduling...");

            // Schedule threat intelligence updates based on configuration
            for (feed_name, feed_config) in &self.config.feeds {
                if feed_config.enabled {
                    // Convert seconds to hours for better user experience in logs
                    let update_interval_seconds =
                        feed_config.custom_interval_seconds.unwrap_or(86400); // Default 24 hours
                    let update_interval_hours = update_interval_seconds / 3600;
                    let cron_expression = format!("0 0 */{} * * *", update_interval_hours.max(1)); // Every N hours, minimum 1

                    info!(
                        "Scheduling '{}' feed updates every {} hours (cron: {})",
                        feed_name, update_interval_hours, cron_expression
                    );

                    // Clone necessary data for the closure
                    let client_clone = self.client.clone();
                    let config_clone = self.config.clone();
                    let feed_name_clone = feed_name.clone();
                    let feed_config_clone = feed_config.clone();

                    // Create the scheduled job
                    let job = Job::new_async(cron_expression.as_str(), move |_uuid, _l| {
                        let client = client_clone.clone();
                        let config = config_clone.clone();
                        let name = feed_name_clone.clone();
                        let config_feed = feed_config_clone.clone();

                        Box::pin(async move {
                            info!("⏰ Scheduled update starting for threat feed: {}", name);

                            match Self::download_feed(&client, &config, &name, &config_feed).await {
                                Ok(()) => {
                                    info!("✅ Scheduled update completed for '{}'", name);
                                }
                                Err(e) => {
                                    error!("❌ Scheduled update failed for '{}': {}", name, e);
                                }
                            }
                        })
                    })?;
                    scheduler.add(job).await?;
                }
            }

            // Start the scheduler
            scheduler.start().await?;
            info!("🚀 Threat intelligence scheduling started successfully");
        } else {
            warn!("⚠️ Scheduler not initialized - automated updates disabled");
        }

        Ok(())
    }

    /// Download all enabled feeds immediately
    pub async fn download_all_feeds(&self) -> Result<(), Box<dyn std::error::Error>> {
        for (feed_name, feed_config) in &self.config.feeds {
            if feed_config.enabled {
                match Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
                {
                    Ok(_) => info!("Successfully downloaded feed: {}", feed_name),
                    Err(e) => error!("Failed to download feed {}: {}", feed_name, e),
                }
            }
        }
        Ok(())
    }

    /// Download a specific threat intelligence feed
    async fn download_feed(
        client: &Client,
        config: &ThreatIntelConfig,
        feed_name: &str,
        feed_config: &FeedConfig,
    ) -> Result<(), Box<dyn std::error::Error>> {
        debug!("Downloading feed: {} from {}", feed_name, feed_config.url);

        let mut request = client.get(&feed_config.url);

        // Add authentication headers
        for (key, value) in &feed_config.headers {
            request = request.header(key, value);
        }

        let mut request_url = feed_config.url.clone();

        // Add API key as header or query param based on service
        if let Some(api_key) = &feed_config.api_key {
            match feed_name {
                name if name.contains("virustotal") => {
                    request = request.header("X-Apikey", api_key);
                }
                name if name.contains("maxmind") => {
                    // Append the license key at request time only so that
                    // the stored URL (logged, serialised) never contains it.
                    let sep = if request_url.contains('?') { '&' } else { '?' };
                    request_url = format!(
                        "{}{}license_key={}",
                        request_url,
                        sep,
                        urlencoding::encode(api_key)
                    );
                    request = client.get(&request_url);
                    // Re-attach headers after rebuilding request
                    for (key, value) in &feed_config.headers {
                        request = request.header(key, value);
                    }
                }
                _ => {
                    // Generic API key header
                    request = request.header("Authorization", format!("Bearer {}", api_key));
                }
            }
        }

        let response = request.send().await?;

        if !response.status().is_success() {
            return Err(format!(
                "HTTP error {}: {}",
                response.status(),
                response.text().await?
            )
            .into());
        }

        let content = response.bytes().await?;
        let file_path = config.feeds_directory.join(&feed_config.filename);

        // Handle compressed feeds (like MaxMind)
        if feed_config.filename.ends_with(".tar.gz") {
            // Extract tar.gz if needed
            Self::extract_compressed_feed(&content, &file_path).await?;
        } else {
            fs::write(&file_path, &content).await?;
        }

        info!("Saved feed {} to {}", feed_name, file_path.display());

        // Validate feed format
        Self::validate_feed_format(&file_path, &feed_config.format)?;

        Ok(())
    }

    /// Extract compressed feeds (tar.gz, zip, etc.)
    async fn extract_compressed_feed(
        content: &[u8],
        output_path: &Path,
    ) -> Result<(), Box<dyn std::error::Error>> {
        // Production implementation: Detect archive type and extract properly
        let extension = output_path
            .extension()
            .and_then(|ext| ext.to_str())
            .unwrap_or("");

        match extension.to_lowercase().as_str() {
            "gz" | "tar" => Self::extract_tar_gz(content, output_path).await,
            "zip" => Self::extract_zip(content, output_path).await,
            "bz2" => Self::extract_bzip2(content, output_path).await,
            "xz" => Self::extract_xz(content, output_path).await,
            _ => {
                // Unknown compression format, save as-is with warning
                fs::write(output_path, content).await?;
                warn!(
                    "Unknown compression format '{}' - saved as-is: {}",
                    extension,
                    output_path.display()
                );
                Ok(())
            }
        }
    }

    /// Extract tar.gz archives with path traversal protection
    async fn extract_tar_gz(
        content: &[u8],
        output_path: &Path,
    ) -> Result<(), Box<dyn std::error::Error>> {
        info!("Extracting tar.gz archive to: {}", output_path.display());

        let content = content.to_vec();
        let dest = output_path
            .parent()
            .unwrap_or_else(|| Path::new("."))
            .to_path_buf();

        tokio::task::spawn_blocking(move || -> Result<(), String> {
            let decoder = flate2::read::GzDecoder::new(content.as_slice());
            let mut archive = tar::Archive::new(decoder);
            let canonical_dest = dest.canonicalize().map_err(|e| e.to_string())?;

            for entry in archive.entries().map_err(|e| e.to_string())? {
                let mut entry = entry.map_err(|e| e.to_string())?;
                let path = entry.path().map_err(|e| e.to_string())?;

                // Reject absolute paths and path traversal sequences
                if path.is_absolute()
                    || path
                        .components()
                        .any(|c| matches!(c, std::path::Component::ParentDir))
                {
                    return Err(format!(
                        "Zip Slip: archive entry has unsafe path: {}",
                        path.display()
                    ));
                }

                let target = canonical_dest.join(&path);
                // Verify the resolved path stays within the destination
                if !target.starts_with(&canonical_dest) {
                    return Err(format!(
                        "Zip Slip: entry escapes destination: {}",
                        path.display()
                    ));
                }

                entry.unpack(&target).map_err(|e| e.to_string())?;
            }
            Ok(())
        })
        .await
        .map_err(|e| -> Box<dyn std::error::Error> { e.to_string().into() })?
        .map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;

        Ok(())
    }

    /// Extract ZIP archives with path traversal protection
    async fn extract_zip(
        content: &[u8],
        output_path: &Path,
    ) -> Result<(), Box<dyn std::error::Error>> {
        info!("Extracting ZIP archive to: {}", output_path.display());

        let content = content.to_vec();
        let dest = output_path
            .parent()
            .unwrap_or_else(|| Path::new("."))
            .to_path_buf();

        tokio::task::spawn_blocking(move || -> Result<(), String> {
            use std::io::Cursor;
            let cursor = Cursor::new(content);
            let mut archive = zip::ZipArchive::new(cursor).map_err(|e| e.to_string())?;
            let canonical_dest = dest.canonicalize().map_err(|e| e.to_string())?;

            for i in 0..archive.len() {
                let mut file = archive.by_index(i).map_err(|e| e.to_string())?;
                let name = file
                    .enclosed_name()
                    .ok_or_else(|| format!("Zip Slip: entry {} has unsafe path", file.name()))?;

                let target = canonical_dest.join(&name);
                if !target.starts_with(&canonical_dest) {
                    return Err(format!(
                        "Zip Slip: entry escapes destination: {}",
                        name.display()
                    ));
                }

                if file.is_dir() {
                    std::fs::create_dir_all(&target).map_err(|e| e.to_string())?;
                } else {
                    if let Some(parent) = target.parent() {
                        std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
                    }
                    let mut out = std::fs::File::create(&target).map_err(|e| e.to_string())?;
                    std::io::copy(&mut file, &mut out).map_err(|e| e.to_string())?;
                }
            }
            Ok(())
        })
        .await
        .map_err(|e| -> Box<dyn std::error::Error> { e.to_string().into() })?
        .map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;

        Ok(())
    }

    /// Extract bzip2 archives
    ///
    /// NOTE: Requires the `bzip2 = "0.4"` crate (not currently in Cargo.toml).
    /// Returns an error rather than silently writing unextracted bytes, which
    /// would produce a corrupt feed file.
    /// To enable: add `bzip2 = "0.4"` to Cargo.toml, then replace this body
    /// with the same `spawn_blocking` decompression pattern used by
    /// `extract_tar_gz`.
    async fn extract_bzip2(
        _content: &[u8],
        output_path: &Path,
    ) -> Result<(), Box<dyn std::error::Error>> {
        Err(format!(
            "bzip2 extraction is not supported: add the `bzip2` crate to Cargo.toml \
             to enable decompression to {}",
            output_path.display()
        )
        .into())
    }

    /// Extract XZ archives
    ///
    /// NOTE: Requires the `xz2 = "0.1"` crate (not currently in Cargo.toml).
    /// Returns an error rather than silently writing unextracted bytes, which
    /// would produce a corrupt feed file.
    /// To enable: add `xz2 = "0.1"` to Cargo.toml, then replace this body
    /// with the same `spawn_blocking` decompression pattern used by
    /// `extract_tar_gz`.
    async fn extract_xz(
        _content: &[u8],
        output_path: &Path,
    ) -> Result<(), Box<dyn std::error::Error>> {
        Err(format!(
            "XZ extraction is not supported: add the `xz2` crate to Cargo.toml \
             to enable decompression to {}",
            output_path.display()
        )
        .into())
    }

    /// Validate that downloaded feed has expected format
    fn validate_feed_format(
        file_path: &Path,
        format: &FeedFormat,
    ) -> Result<(), Box<dyn std::error::Error>> {
        let content = std::fs::read_to_string(file_path)?;

        match format {
            FeedFormat::PlainText => {
                // Basic validation - check if it looks like IP addresses or networks
                let lines: Vec<&str> = content
                    .lines()
                    .filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
                    .collect();
                if lines.is_empty() {
                    return Err("Feed appears to be empty".into());
                }
            }
            FeedFormat::Csv => {
                let mut reader = csv::Reader::from_reader(content.as_bytes());
                if reader.headers().is_err() {
                    return Err("Invalid CSV format".into());
                }
            }
            FeedFormat::Json => {
                serde_json::from_str::<serde_json::Value>(&content)?;
            }
            FeedFormat::Xml => {
                // Basic XML validation - check for well-formed structure
                if !content.trim_start().starts_with('<') {
                    return Err("Invalid XML format".into());
                }
            }
        }

        debug!("Feed format validation passed: {}", file_path.display());
        Ok(())
    }

    /// Get status of all feeds
    pub async fn get_feed_status(&self) -> HashMap<String, FeedStatus> {
        let mut status = HashMap::new();

        for (feed_name, feed_config) in &self.config.feeds {
            let file_path = self.config.feeds_directory.join(&feed_config.filename);

            let feed_status = if feed_config.enabled {
                if file_path.exists() {
                    if let Ok(metadata) = fs::metadata(&file_path).await {
                        FeedStatus::Active {
                            last_updated: metadata
                                .modified()
                                .unwrap_or(std::time::SystemTime::UNIX_EPOCH),
                            size_bytes: metadata.len(),
                        }
                    } else {
                        FeedStatus::Error("Cannot read file metadata".to_string())
                    }
                } else {
                    FeedStatus::NotDownloaded
                }
            } else {
                FeedStatus::Disabled
            };

            status.insert(feed_name.clone(), feed_status);
        }

        status
    }

    /// Manually trigger update of specific feed
    pub async fn update_feed(&self, feed_name: &str) -> Result<(), Box<dyn std::error::Error>> {
        if let Some(feed_config) = self.config.feeds.get(feed_name) {
            if feed_config.enabled {
                Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
            } else {
                Err(format!("Feed '{}' is disabled", feed_name).into())
            }
        } else {
            Err(format!("Feed '{}' not found", feed_name).into())
        }
    }

    /// Check if an IP address is in malicious IP feeds
    pub fn is_malicious_ip(&self, ip: &std::net::IpAddr) -> bool {
        for (feed_name, feed_config) in &self.config.feeds {
            if !feed_config.enabled {
                continue;
            }

            if matches!(feed_config.feed_type, FeedType::MaliciousIPs) {
                let file_path = self.config.feeds_directory.join(&feed_config.filename);
                if self.check_ip_in_feed(&file_path, ip) {
                    tracing::warn!("Malicious IP detected: {} (source: {})", ip, feed_name);
                    return true;
                }
            }
        }
        false
    }

    /// Check if an IP address is a Tor exit node
    pub fn is_tor_exit(&self, ip: &std::net::IpAddr) -> bool {
        for (feed_name, feed_config) in &self.config.feeds {
            if !feed_config.enabled {
                continue;
            }

            if matches!(feed_config.feed_type, FeedType::TorExitNodes) {
                let file_path = self.config.feeds_directory.join(&feed_config.filename);
                if self.check_ip_in_feed(&file_path, ip) {
                    tracing::warn!("Tor exit node detected: {} (source: {})", ip, feed_name);
                    return true;
                }
            }
        }
        false
    }

    /// Check if an IP address is from a VPN or proxy service
    pub fn is_proxy_vpn(&self, ip: &std::net::IpAddr) -> bool {
        for (feed_name, feed_config) in &self.config.feeds {
            if !feed_config.enabled {
                continue;
            }

            if matches!(feed_config.feed_type, FeedType::VpnProxy) {
                let file_path = self.config.feeds_directory.join(&feed_config.filename);
                if self.check_ip_in_feed(&file_path, ip) {
                    tracing::info!("VPN/Proxy detected: {} (source: {})", ip, feed_name);
                    return true;
                }
            }
        }
        false
    }

    /// Helper method to check if an IP is present in a feed file
    fn check_ip_in_feed(&self, file_path: &std::path::Path, ip: &std::net::IpAddr) -> bool {
        if !file_path.exists() {
            return false;
        }

        if let Ok(contents) = std::fs::read_to_string(file_path) {
            for line in contents.lines() {
                let line = line.trim();
                if line.is_empty() || line.starts_with('#') {
                    continue;
                }

                // Check exact IP match
                if line == ip.to_string() {
                    return true;
                }

                // Check CIDR network match
                if line.contains('/') {
                    match ip {
                        std::net::IpAddr::V4(ipv4) => {
                            if let Ok(network) = line.parse::<ipnetwork::Ipv4Network>()
                                && network.contains(*ipv4)
                            {
                                return true;
                            }
                        }
                        std::net::IpAddr::V6(ipv6) => {
                            if let Ok(network) = line.parse::<ipnetwork::Ipv6Network>()
                                && network.contains(*ipv6)
                            {
                                return true;
                            }
                        }
                    }
                }
            }
        }

        false
    }
}

/// Status of a threat intelligence feed
#[derive(Debug, Clone)]
pub enum FeedStatus {
    /// Feed is disabled
    Disabled,

    /// Feed is enabled but not yet downloaded
    NotDownloaded,

    /// Feed is active and up-to-date
    Active {
        last_updated: std::time::SystemTime,
        size_bytes: u64,
    },

    /// Feed has an error
    Error(String),
}