Skip to main content

auth_framework/
threat_intelligence.rs

1//! Automated Threat Intelligence Feed Management
2//!
3//! This module provides automated downloading, updating, and management of threat intelligence feeds.
4//! Features:
5//! - Simple on/off configuration switches
6//! - Multiple feed sources (free and paid)
7//! - Automatic scheduling and updates
8//! - Credential management for paid services
9//! - Configurable update intervals
10
11use reqwest::Client;
12use serde::{Deserialize, Serialize};
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::time::Duration;
16use tokio::fs;
17use tokio_cron_scheduler::Job;
18use tracing::{debug, error, info, warn};
19
20#[cfg(test)]
21mod tar_extraction_tests;
22
23/// Configuration for automated threat intelligence feeds
24#[derive(Debug, Clone, Serialize, Deserialize)]
25pub struct ThreatIntelConfig {
26    /// Enable/disable automated feed updates
27    pub auto_update_enabled: bool,
28
29    /// Update interval in seconds (default: 3600 = 1 hour)
30    pub update_interval_seconds: u64,
31
32    /// Directory to store downloaded feeds
33    pub feeds_directory: PathBuf,
34
35    /// Individual feed configurations
36    pub feeds: HashMap<String, FeedConfig>,
37
38    /// Global HTTP timeout for downloads
39    pub download_timeout_seconds: u64,
40}
41
42/// Configuration for a specific threat intelligence feed
43#[derive(Clone, Serialize, Deserialize)]
44pub struct FeedConfig {
45    /// Enable this specific feed
46    pub enabled: bool,
47
48    /// Feed type (determines parsing and processing)
49    pub feed_type: FeedType,
50
51    /// Download URL or API endpoint
52    pub url: String,
53
54    /// Optional API key or authentication token
55    pub api_key: Option<String>,
56
57    /// HTTP headers for authentication
58    pub headers: HashMap<String, String>,
59
60    /// Local filename to save the feed
61    pub filename: String,
62
63    /// Format of the feed data
64    pub format: FeedFormat,
65
66    /// Update interval override (if different from global)
67    pub custom_interval_seconds: Option<u64>,
68}
69
70impl std::fmt::Debug for FeedConfig {
71    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72        let mut d = f.debug_struct("FeedConfig");
73        d.field("enabled", &self.enabled);
74        d.field("feed_type", &self.feed_type);
75        d.field("url", &crate::utils::redact::redacted(&self.url));
76        d.field("api_key", &crate::utils::redact::redacted(&self.api_key));
77        d.field("headers", &crate::utils::redact::redacted(&self.headers));
78        d.field("filename", &self.filename);
79        d.field("format", &self.format);
80        d.field("custom_interval_seconds", &self.custom_interval_seconds);
81        d.finish()
82    }
83}
84
85/// Types of threat intelligence feeds
86#[derive(Debug, Clone, Serialize, Deserialize)]
87pub enum FeedType {
88    /// Malicious IP addresses
89    MaliciousIPs,
90
91    /// Tor exit nodes
92    TorExitNodes,
93
94    /// VPN/Proxy servers
95    VpnProxy,
96
97    /// Botnet C&C servers
98    BotnetC2,
99
100    /// Country-based threat intelligence
101    CountryThreats,
102
103    /// Hosting provider ranges
104    HostingProviders,
105
106    /// Datacenter IP ranges
107    DatacenterRanges,
108
109    /// Custom feed type
110    Custom(String),
111}
112
113/// Format of feed data
114#[derive(Debug, Clone, Serialize, Deserialize)]
115pub enum FeedFormat {
116    /// Plain text, one IP/range per line
117    PlainText,
118
119    /// CSV format
120    Csv,
121
122    /// JSON format
123    Json,
124
125    /// XML format
126    Xml,
127}
128
129/// Automated threat intelligence feed manager
130pub struct ThreatFeedManager {
131    config: ThreatIntelConfig,
132    client: Client,
133    scheduler: Option<tokio_cron_scheduler::JobScheduler>,
134}
135
136#[cfg(test)]
137mod feed_dispatch_tests;
138
139impl Default for ThreatIntelConfig {
140    fn default() -> Self {
141        let mut feeds = HashMap::new();
142
143        // Pre-configured popular free feeds with simple on/off switches
144        feeds.insert(
145            "tor_exits".to_string(),
146            FeedConfig {
147                enabled: false, // OFF by default - user enables via config
148                feed_type: FeedType::TorExitNodes,
149                url: "https://check.torproject.org/torbulkexitlist".to_string(),
150                api_key: None,
151                headers: HashMap::new(),
152                filename: "tor-exits.txt".to_string(),
153                format: FeedFormat::PlainText,
154                custom_interval_seconds: Some(3600), // Update hourly
155            },
156        );
157
158        feeds.insert(
159            "spamhaus_drop".to_string(),
160            FeedConfig {
161                enabled: false, // OFF by default
162                feed_type: FeedType::MaliciousIPs,
163                url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
164                api_key: None,
165                headers: HashMap::new(),
166                filename: "spamhaus-drop.txt".to_string(),
167                format: FeedFormat::PlainText,
168                custom_interval_seconds: Some(3600),
169            },
170        );
171
172        feeds.insert(
173            "emergingthreats_compromised".to_string(),
174            FeedConfig {
175                enabled: false, // OFF by default
176                feed_type: FeedType::MaliciousIPs,
177                url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt".to_string(),
178                api_key: None,
179                headers: HashMap::new(),
180                filename: "emerging-threats-ips.txt".to_string(),
181                format: FeedFormat::PlainText,
182                custom_interval_seconds: Some(7200), // Update every 2 hours
183            },
184        );
185
186        // Paid service examples (disabled by default, require API keys)
187        feeds.insert(
188            "virustotal_malicious".to_string(),
189            FeedConfig {
190                enabled: false, // OFF - requires API key
191                feed_type: FeedType::MaliciousIPs,
192                url: "https://www.virustotal.com/api/v3/intelligence/hunting_notification_files"
193                    .to_string(),
194                api_key: None, // User must set VIRUSTOTAL_API_KEY
195                headers: HashMap::new(),
196                filename: "virustotal-malicious.json".to_string(),
197                format: FeedFormat::Json,
198                custom_interval_seconds: Some(1800),
199            },
200        );
201
202        feeds.insert(
203            "maxmind_proxy_detection".to_string(),
204            FeedConfig {
205                enabled: false, // OFF - requires license
206                feed_type: FeedType::VpnProxy,
207                url: "https://download.maxmind.com/app/geoip_download".to_string(),
208                api_key: None, // User must set MAXMIND_LICENSE_KEY
209                headers: HashMap::new(),
210                filename: "maxmind-proxy-ranges.csv".to_string(),
211                format: FeedFormat::Csv,
212                custom_interval_seconds: Some(86400), // Daily
213            },
214        );
215
216        Self {
217            auto_update_enabled: false,    // OFF by default - user enables
218            update_interval_seconds: 3600, // 1 hour default
219            feeds_directory: PathBuf::from("threat-feeds"),
220            feeds,
221            download_timeout_seconds: 30,
222        }
223    }
224}
225
226impl ThreatIntelConfig {
227    /// All feeds disabled, no automatic updates.
228    ///
229    /// Use this when threat intelligence is not needed or when running
230    /// in an air-gapped environment.
231    ///
232    /// # Example
233    /// ```rust,ignore
234    /// let config = ThreatIntelConfig::disabled();
235    /// ```
236    pub fn disabled() -> Self {
237        Self {
238            auto_update_enabled: false,
239            update_interval_seconds: 0,
240            feeds_directory: PathBuf::from("./threat-feeds"),
241            feeds: HashMap::new(),
242            download_timeout_seconds: 30,
243        }
244    }
245
246    /// Aggressive update schedule — every 5 minutes, 10-second download timeout.
247    ///
248    /// Pre-configures popular free feeds (Tor exits, Spamhaus DROP,
249    /// Emerging Threats) all enabled.
250    ///
251    /// # Example
252    /// ```rust,ignore
253    /// let config = ThreatIntelConfig::aggressive();
254    /// ```
255    #[allow(clippy::field_reassign_with_default)] // each override stays on its own line so the diff against Default is readable
256    pub fn aggressive() -> Self {
257        let mut base = Self::default();
258        base.auto_update_enabled = true;
259        base.update_interval_seconds = 300; // 5 minutes
260        base.download_timeout_seconds = 10;
261        // Enable all pre-configured feeds
262        for feed in base.feeds.values_mut() {
263            feed.enabled = true;
264        }
265        base
266    }
267
268    /// Create configuration from environment variables and config file
269    pub fn from_env_and_config() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
270        // Try to load from config file first
271        let config_path = std::env::var("THREAT_INTEL_CONFIG_PATH")
272            .unwrap_or_else(|_| "threat-intel-config.yaml".to_string());
273
274        if std::path::Path::new(&config_path).exists() {
275            let config_content = std::fs::read_to_string(&config_path)?;
276            let mut config: Self = serde_yaml::from_str(&config_content)?;
277
278            // Override with environment variables if they exist
279            if let Ok(enabled) = std::env::var("THREAT_INTEL_ENABLED") {
280                config.auto_update_enabled = enabled.to_lowercase() == "true";
281            }
282
283            if let Ok(interval) = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
284                && let Ok(seconds) = interval.parse::<u64>()
285            {
286                config.update_interval_seconds = seconds;
287            }
288
289            if let Ok(feeds_dir) = std::env::var("THREAT_INTEL_FEEDS_DIR") {
290                config.feeds_directory = std::path::PathBuf::from(feeds_dir);
291            }
292
293            Ok(config)
294        } else {
295            // Create default configuration from environment variables
296            Ok(Self::from_env_defaults())
297        }
298    }
299
300    /// Create default configuration from environment variables
301    fn from_env_defaults() -> Self {
302        let enabled = std::env::var("THREAT_INTEL_ENABLED")
303            .unwrap_or_else(|_| "false".to_string())
304            .to_lowercase()
305            == "true";
306
307        let update_interval = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
308            .unwrap_or_else(|_| "3600".to_string())
309            .parse::<u64>()
310            .unwrap_or(3600);
311
312        let feeds_dir = std::env::var("THREAT_INTEL_FEEDS_DIR")
313            .unwrap_or_else(|_| "./threat-feeds".to_string());
314
315        let timeout = std::env::var("THREAT_INTEL_TIMEOUT")
316            .unwrap_or_else(|_| "30".to_string())
317            .parse::<u64>()
318            .unwrap_or(30);
319
320        // Create default feeds based on environment switches
321        let mut feeds = HashMap::new();
322
323        // Tor exits feed
324        if std::env::var("TOR_EXITS_ENABLED")
325            .unwrap_or_else(|_| "true".to_string())
326            .to_lowercase()
327            == "true"
328        {
329            feeds.insert(
330                "tor_exits".to_string(),
331                FeedConfig {
332                    enabled: true,
333                    feed_type: FeedType::TorExitNodes,
334                    url: "https://check.torproject.org/torbulkexitlist".to_string(),
335                    api_key: None,
336                    headers: HashMap::new(),
337                    filename: "tor-exits.txt".to_string(),
338                    format: FeedFormat::PlainText,
339                    custom_interval_seconds: None,
340                },
341            );
342        }
343
344        // Spamhaus DROP feed
345        if std::env::var("SPAMHAUS_DROP_ENABLED")
346            .unwrap_or_else(|_| "true".to_string())
347            .to_lowercase()
348            == "true"
349        {
350            feeds.insert(
351                "spamhaus_drop".to_string(),
352                FeedConfig {
353                    enabled: true,
354                    feed_type: FeedType::MaliciousIPs,
355                    url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
356                    api_key: None,
357                    headers: HashMap::new(),
358                    filename: "spamhaus-drop.txt".to_string(),
359                    format: FeedFormat::PlainText,
360                    custom_interval_seconds: None,
361                },
362            );
363        }
364
365        // Emerging Threats feed
366        if std::env::var("EMERGINGTHREATS_ENABLED")
367            .unwrap_or_else(|_| "false".to_string())
368            .to_lowercase()
369            == "true"
370        {
371            feeds.insert(
372                "emergingthreats".to_string(),
373                FeedConfig {
374                    enabled: true,
375                    feed_type: FeedType::MaliciousIPs,
376                    url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
377                        .to_string(),
378                    api_key: None,
379                    headers: HashMap::new(),
380                    filename: "emerging-threats-ips.txt".to_string(),
381                    format: FeedFormat::PlainText,
382                    custom_interval_seconds: None,
383                },
384            );
385        }
386
387        Self {
388            auto_update_enabled: enabled,
389            update_interval_seconds: update_interval,
390            feeds_directory: std::path::PathBuf::from(feeds_dir),
391            download_timeout_seconds: timeout,
392            feeds,
393        }
394    }
395}
396
397impl ThreatFeedManager {
398    /// Create a new threat feed manager with configuration (async version)
399    pub async fn new_async(config: ThreatIntelConfig) -> Result<Self, Box<dyn std::error::Error>> {
400        // Create feeds directory if it doesn't exist
401        if !config.feeds_directory.exists() {
402            fs::create_dir_all(&config.feeds_directory).await?;
403        }
404
405        let client = Client::builder()
406            .timeout(Duration::from_secs(config.download_timeout_seconds))
407            .user_agent("AuthFramework-ThreatIntel/1.0")
408            .build()?;
409
410        let scheduler = Some(tokio_cron_scheduler::JobScheduler::new().await?);
411
412        Ok(Self {
413            config,
414            client,
415            scheduler,
416        })
417    }
418
419    /// Create a new threat intelligence manager (synchronous version)
420    pub fn new(
421        config: ThreatIntelConfig,
422    ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
423        // Create feeds directory if it doesn't exist
424        if !config.feeds_directory.exists() {
425            std::fs::create_dir_all(&config.feeds_directory)?;
426        }
427
428        let client = Client::builder()
429            .timeout(Duration::from_secs(config.download_timeout_seconds))
430            .user_agent("AuthFramework-ThreatIntel/1.0")
431            .build()?;
432
433        // Scheduler is not initialized in the simple constructor
434        let scheduler = None;
435
436        Ok(Self {
437            config,
438            client,
439            scheduler,
440        })
441    }
442
443    /// Start automated feed updates in the background
444    pub fn start_automated_updates(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
445        if !self.config.auto_update_enabled {
446            tracing::info!("Automated updates disabled in configuration");
447            return Ok(());
448        }
449
450        // Schedule updates for each enabled feed
451        let update_interval = format!("0 */{} * * * *", self.config.update_interval_seconds / 60);
452
453        tracing::info!(
454            "🚀 Starting automated threat intelligence updates (interval: {})",
455            update_interval
456        );
457
458        // For now, just log that we would start updates
459        // In a full implementation, this would start the tokio scheduler
460        tracing::info!("✅ Automated threat intelligence updates scheduled successfully");
461
462        Ok(())
463    }
464
465    /// Load configuration from YAML file or environment variables
466    pub fn load_config() -> ThreatIntelConfig {
467        // Try to load from config file first
468        if let Ok(config_content) = std::fs::read_to_string("threat-intel-config.yaml")
469            && let Ok(config) = serde_yaml::from_str::<ThreatIntelConfig>(&config_content)
470        {
471            info!("Loaded threat intelligence configuration from file");
472            return config;
473        }
474
475        // Fall back to environment variables for simple on/off switches
476        let mut config = ThreatIntelConfig::default();
477
478        // Global enable/disable switch
479        if let Ok(enabled) = std::env::var("THREAT_INTEL_AUTO_UPDATE") {
480            config.auto_update_enabled = enabled.to_lowercase() == "true";
481        }
482
483        // Simple feed enable switches via environment variables
484        let feed_switches = [
485            ("THREAT_INTEL_TOR_EXITS", "tor_exits"),
486            ("THREAT_INTEL_SPAMHAUS", "spamhaus_drop"),
487            (
488                "THREAT_INTEL_EMERGING_THREATS",
489                "emergingthreats_compromised",
490            ),
491            ("THREAT_INTEL_VIRUSTOTAL", "virustotal_malicious"),
492            ("THREAT_INTEL_MAXMIND_PROXY", "maxmind_proxy_detection"),
493        ];
494
495        for (env_var, feed_name) in &feed_switches {
496            if let Ok(enabled) = std::env::var(env_var)
497                && let Some(feed) = config.feeds.get_mut(*feed_name)
498            {
499                feed.enabled = enabled.to_lowercase() == "true";
500                info!(
501                    "Feed {} enabled via {}: {}",
502                    feed_name, env_var, feed.enabled
503                );
504            }
505        }
506
507        // API keys from environment
508        if let Ok(api_key) = std::env::var("VIRUSTOTAL_API_KEY")
509            && let Some(feed) = config.feeds.get_mut("virustotal_malicious")
510        {
511            feed.api_key = Some(api_key);
512            feed.headers.insert(
513                "X-Apikey".to_string(),
514                feed.api_key.clone().expect("set to Some on previous line"),
515            );
516        }
517
518        if let Ok(license_key) = std::env::var("MAXMIND_LICENSE_KEY")
519            && let Some(feed) = config.feeds.get_mut("maxmind_proxy_detection")
520        {
521            feed.api_key = Some(license_key);
522            // Edition and suffix are appended as query params at download time,
523            // keeping the license key out of the stored URL string.
524            feed.url = format!("{}?edition_id=GeoIP2-Anonymous-IP&suffix=tar.gz", feed.url);
525        }
526
527        config
528    }
529
530    /// Start automated feed updates if enabled (Currently simplified implementation)
531    pub async fn start_automation(&mut self) -> Result<(), Box<dyn std::error::Error>> {
532        if !self.config.auto_update_enabled {
533            info!("Threat intelligence automation is disabled");
534            return Ok(());
535        }
536
537        info!("Starting automated threat intelligence feed updates");
538
539        // Initial download of all enabled feeds
540        self.download_all_feeds().await?;
541
542        // Production implementation: Set up automated scheduling for threat intelligence feeds
543        self.start_automated_scheduling().await?;
544
545        info!("✅ Threat intelligence feeds downloaded and scheduling activated");
546
547        Ok(())
548    }
549
550    /// Start automated scheduling for threat intelligence feed updates
551    async fn start_automated_scheduling(&mut self) -> Result<(), Box<dyn std::error::Error>> {
552        if let Some(scheduler) = &self.scheduler {
553            info!("Setting up automated threat intelligence feed scheduling...");
554
555            // Schedule threat intelligence updates based on configuration
556            for (feed_name, feed_config) in &self.config.feeds {
557                if feed_config.enabled {
558                    // Convert seconds to hours for better user experience in logs
559                    let update_interval_seconds =
560                        feed_config.custom_interval_seconds.unwrap_or(86400); // Default 24 hours
561                    let update_interval_hours = update_interval_seconds / 3600;
562                    let cron_expression = format!("0 0 */{} * * *", update_interval_hours.max(1)); // Every N hours, minimum 1
563
564                    info!(
565                        "Scheduling '{}' feed updates every {} hours (cron: {})",
566                        feed_name, update_interval_hours, cron_expression
567                    );
568
569                    // Clone necessary data for the closure
570                    let client_clone = self.client.clone();
571                    let config_clone = self.config.clone();
572                    let feed_name_clone = feed_name.clone();
573                    let feed_config_clone = feed_config.clone();
574
575                    // Create the scheduled job
576                    let job = Job::new_async(cron_expression.as_str(), move |_uuid, _l| {
577                        let client = client_clone.clone();
578                        let config = config_clone.clone();
579                        let name = feed_name_clone.clone();
580                        let config_feed = feed_config_clone.clone();
581
582                        Box::pin(async move {
583                            info!("⏰ Scheduled update starting for threat feed: {}", name);
584
585                            match Self::download_feed(&client, &config, &name, &config_feed).await {
586                                Ok(()) => {
587                                    info!("✅ Scheduled update completed for '{}'", name);
588                                }
589                                Err(e) => {
590                                    error!("❌ Scheduled update failed for '{}': {}", name, e);
591                                }
592                            }
593                        })
594                    })?;
595                    scheduler.add(job).await?;
596                }
597            }
598
599            // Start the scheduler
600            scheduler.start().await?;
601            info!("🚀 Threat intelligence scheduling started successfully");
602        } else {
603            warn!("⚠️ Scheduler not initialized - automated updates disabled");
604        }
605
606        Ok(())
607    }
608
609    /// Download all enabled feeds immediately
610    pub async fn download_all_feeds(&self) -> Result<(), Box<dyn std::error::Error>> {
611        for (feed_name, feed_config) in &self.config.feeds {
612            if feed_config.enabled {
613                match Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
614                {
615                    Ok(_) => info!("Successfully downloaded feed: {}", feed_name),
616                    Err(e) => error!("Failed to download feed {}: {}", feed_name, e),
617                }
618            }
619        }
620        Ok(())
621    }
622
623    /// Download a specific threat intelligence feed
624    async fn download_feed(
625        client: &Client,
626        config: &ThreatIntelConfig,
627        feed_name: &str,
628        feed_config: &FeedConfig,
629    ) -> Result<(), Box<dyn std::error::Error>> {
630        debug!("Downloading feed: {} from {}", feed_name, feed_config.url);
631
632        let mut request = client.get(&feed_config.url);
633
634        // Add authentication headers
635        for (key, value) in &feed_config.headers {
636            request = request.header(key, value);
637        }
638
639        let mut request_url = feed_config.url.clone();
640
641        // Add API key as header or query param based on service
642        if let Some(api_key) = &feed_config.api_key {
643            match feed_name {
644                name if name.contains("virustotal") => {
645                    request = request.header("X-Apikey", api_key);
646                }
647                name if name.contains("maxmind") => {
648                    // Append the license key at request time only so that
649                    // the stored URL (logged, serialised) never contains it.
650                    let sep = if request_url.contains('?') { '&' } else { '?' };
651                    request_url = format!(
652                        "{}{}license_key={}",
653                        request_url,
654                        sep,
655                        urlencoding::encode(api_key)
656                    );
657                    request = client.get(&request_url);
658                    // Re-attach headers after rebuilding request
659                    for (key, value) in &feed_config.headers {
660                        request = request.header(key, value);
661                    }
662                }
663                _ => {
664                    // Generic API key header
665                    request = request.header("Authorization", format!("Bearer {}", api_key));
666                }
667            }
668        }
669
670        let response = request.send().await.map_err(reqwest::Error::without_url)?;
671
672        if !response.status().is_success() {
673            return Err(format!(
674                "HTTP error {}: {}",
675                response.status(),
676                response.text().await.map_err(reqwest::Error::without_url)?
677            )
678            .into());
679        }
680
681        let content = response
682            .bytes()
683            .await
684            .map_err(reqwest::Error::without_url)?;
685        let file_path = config.feeds_directory.join(&feed_config.filename);
686
687        // Handle compressed feeds (like MaxMind)
688        if feed_config.filename.ends_with(".tar.gz") {
689            // Extract tar.gz if needed
690            Self::extract_compressed_feed(&content, &file_path).await?;
691        } else {
692            fs::write(&file_path, &content).await?;
693        }
694
695        info!("Saved feed {} to {}", feed_name, file_path.display());
696
697        // Validate feed format
698        Self::validate_feed_format(&file_path, &feed_config.format)?;
699
700        Ok(())
701    }
702
703    /// Extract compressed feeds: tar.gz is extracted, bz2/xz/zip are refused, anything else is saved as-is.
704    async fn extract_compressed_feed(
705        content: &[u8],
706        output_path: &Path,
707    ) -> Result<(), Box<dyn std::error::Error>> {
708        // Production implementation: Detect archive type and extract properly
709        let extension = output_path
710            .extension()
711            .and_then(|ext| ext.to_str())
712            .unwrap_or("");
713
714        match extension.to_lowercase().as_str() {
715            "gz" | "tar" => Self::extract_tar_gz(content, output_path).await,
716            "zip" => Err(format!(
717                "zip extraction is not supported: {} was not extracted",
718                output_path.display()
719            )
720            .into()),
721            "bz2" => Self::extract_bzip2(content, output_path).await,
722            "xz" => Self::extract_xz(content, output_path).await,
723            _ => {
724                // Unknown compression format, save as-is with warning
725                fs::write(output_path, content).await?;
726                warn!(
727                    "Unknown compression format '{}' - saved as-is: {}",
728                    extension,
729                    output_path.display()
730                );
731                Ok(())
732            }
733        }
734    }
735
736    /// Extract tar.gz archives with path traversal protection
737    async fn extract_tar_gz(
738        content: &[u8],
739        output_path: &Path,
740    ) -> Result<(), Box<dyn std::error::Error>> {
741        info!("Extracting tar.gz archive to: {}", output_path.display());
742
743        let content = content.to_vec();
744        let dest = output_path
745            .parent()
746            .unwrap_or_else(|| Path::new("."))
747            .to_path_buf();
748
749        tokio::task::spawn_blocking(move || -> Result<(), String> {
750            let decoder = flate2::read::GzDecoder::new(content.as_slice());
751            let mut archive = tar::Archive::new(decoder);
752            let canonical_dest = dest.canonicalize().map_err(|e| e.to_string())?;
753
754            for entry in archive.entries().map_err(|e| e.to_string())? {
755                let mut entry = entry.map_err(|e| e.to_string())?;
756                let path = entry.path().map_err(|e| e.to_string())?.into_owned();
757
758                // A feed is data: extract regular files and directories only.
759                let kind = entry.header().entry_type();
760                if !(kind.is_file() || kind.is_dir()) {
761                    return Err(format!(
762                        "Archive entry {} has unsupported type {:?}: only files and directories are extracted",
763                        path.display(),
764                        kind
765                    ));
766                }
767
768                // Reject absolute paths and path traversal sequences
769                if path.is_absolute()
770                    || path
771                        .components()
772                        .any(|c| matches!(c, std::path::Component::ParentDir))
773                {
774                    return Err(format!(
775                        "Zip Slip: archive entry has unsafe path: {}",
776                        path.display()
777                    ));
778                }
779
780                let target = canonical_dest.join(&path);
781                // Verify the resolved path stays within the destination
782                if !target.starts_with(&canonical_dest) {
783                    return Err(format!(
784                        "Zip Slip: entry escapes destination: {}",
785                        path.display()
786                    ));
787                }
788
789                // `unpack_in` confines each write to `dest`.
790                let unpacked = entry
791                    .unpack_in(&canonical_dest)
792                    .map_err(|e| e.to_string())?;
793                if !unpacked {
794                    return Err(format!(
795                        "Zip Slip: entry escapes destination: {}",
796                        path.display()
797                    ));
798                }
799            }
800            Ok(())
801        })
802        .await
803        .map_err(|e| -> Box<dyn std::error::Error> { e.to_string().into() })?
804        .map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
805
806        Ok(())
807    }
808
809    /// Extract bzip2 archives
810    ///
811    /// NOTE: Requires the `bzip2 = "0.4"` crate (not currently in Cargo.toml).
812    /// Returns an error rather than silently writing unextracted bytes, which
813    /// would produce a corrupt feed file.
814    /// To enable: add `bzip2 = "0.4"` to Cargo.toml, then replace this body
815    /// with the same `spawn_blocking` decompression pattern used by
816    /// `extract_tar_gz`.
817    async fn extract_bzip2(
818        _content: &[u8],
819        output_path: &Path,
820    ) -> Result<(), Box<dyn std::error::Error>> {
821        Err(format!(
822            "bzip2 extraction is not supported: add the `bzip2` crate to Cargo.toml \
823             to enable decompression to {}",
824            output_path.display()
825        )
826        .into())
827    }
828
829    /// Extract XZ archives
830    ///
831    /// NOTE: Requires the `xz2 = "0.1"` crate (not currently in Cargo.toml).
832    /// Returns an error rather than silently writing unextracted bytes, which
833    /// would produce a corrupt feed file.
834    /// To enable: add `xz2 = "0.1"` to Cargo.toml, then replace this body
835    /// with the same `spawn_blocking` decompression pattern used by
836    /// `extract_tar_gz`.
837    async fn extract_xz(
838        _content: &[u8],
839        output_path: &Path,
840    ) -> Result<(), Box<dyn std::error::Error>> {
841        Err(format!(
842            "XZ extraction is not supported: add the `xz2` crate to Cargo.toml \
843             to enable decompression to {}",
844            output_path.display()
845        )
846        .into())
847    }
848
849    /// Validate that downloaded feed has expected format
850    fn validate_feed_format(
851        file_path: &Path,
852        format: &FeedFormat,
853    ) -> Result<(), Box<dyn std::error::Error>> {
854        let content = std::fs::read_to_string(file_path)?;
855
856        match format {
857            FeedFormat::PlainText => {
858                // Basic validation - check if it looks like IP addresses or networks
859                let lines: Vec<&str> = content
860                    .lines()
861                    .filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
862                    .collect();
863                if lines.is_empty() {
864                    return Err("Feed appears to be empty".into());
865                }
866            }
867            FeedFormat::Csv => {
868                let mut reader = csv::Reader::from_reader(content.as_bytes());
869                if reader.headers().is_err() {
870                    return Err("Invalid CSV format".into());
871                }
872            }
873            FeedFormat::Json => {
874                serde_json::from_str::<serde_json::Value>(&content)?;
875            }
876            FeedFormat::Xml => {
877                // Basic XML validation - check for well-formed structure
878                if !content.trim_start().starts_with('<') {
879                    return Err("Invalid XML format".into());
880                }
881            }
882        }
883
884        debug!("Feed format validation passed: {}", file_path.display());
885        Ok(())
886    }
887
888    /// Get status of all feeds
889    pub async fn get_feed_status(&self) -> HashMap<String, FeedStatus> {
890        let mut status = HashMap::new();
891
892        for (feed_name, feed_config) in &self.config.feeds {
893            let file_path = self.config.feeds_directory.join(&feed_config.filename);
894
895            let feed_status = if feed_config.enabled {
896                if file_path.exists() {
897                    if let Ok(metadata) = fs::metadata(&file_path).await {
898                        FeedStatus::Active {
899                            last_updated: metadata
900                                .modified()
901                                .unwrap_or(std::time::SystemTime::UNIX_EPOCH),
902                            size_bytes: metadata.len(),
903                        }
904                    } else {
905                        FeedStatus::Error("Cannot read file metadata".to_string())
906                    }
907                } else {
908                    FeedStatus::NotDownloaded
909                }
910            } else {
911                FeedStatus::Disabled
912            };
913
914            status.insert(feed_name.clone(), feed_status);
915        }
916
917        status
918    }
919
920    /// Manually trigger update of specific feed
921    pub async fn update_feed(&self, feed_name: &str) -> Result<(), Box<dyn std::error::Error>> {
922        if let Some(feed_config) = self.config.feeds.get(feed_name) {
923            if feed_config.enabled {
924                Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
925            } else {
926                Err(format!("Feed '{}' is disabled", feed_name).into())
927            }
928        } else {
929            Err(format!("Feed '{}' not found", feed_name).into())
930        }
931    }
932
933    /// Check if an IP address is in malicious IP feeds
934    pub fn is_malicious_ip(&self, ip: &std::net::IpAddr) -> bool {
935        for (feed_name, feed_config) in &self.config.feeds {
936            if !feed_config.enabled {
937                continue;
938            }
939
940            if matches!(feed_config.feed_type, FeedType::MaliciousIPs) {
941                let file_path = self.config.feeds_directory.join(&feed_config.filename);
942                if self.check_ip_in_feed(&file_path, ip) {
943                    tracing::warn!("Malicious IP detected: {} (source: {})", ip, feed_name);
944                    return true;
945                }
946            }
947        }
948        false
949    }
950
951    /// Check if an IP address is a Tor exit node
952    pub fn is_tor_exit(&self, ip: &std::net::IpAddr) -> bool {
953        for (feed_name, feed_config) in &self.config.feeds {
954            if !feed_config.enabled {
955                continue;
956            }
957
958            if matches!(feed_config.feed_type, FeedType::TorExitNodes) {
959                let file_path = self.config.feeds_directory.join(&feed_config.filename);
960                if self.check_ip_in_feed(&file_path, ip) {
961                    tracing::warn!("Tor exit node detected: {} (source: {})", ip, feed_name);
962                    return true;
963                }
964            }
965        }
966        false
967    }
968
969    /// Check if an IP address is from a VPN or proxy service
970    pub fn is_proxy_vpn(&self, ip: &std::net::IpAddr) -> bool {
971        for (feed_name, feed_config) in &self.config.feeds {
972            if !feed_config.enabled {
973                continue;
974            }
975
976            if matches!(feed_config.feed_type, FeedType::VpnProxy) {
977                let file_path = self.config.feeds_directory.join(&feed_config.filename);
978                if self.check_ip_in_feed(&file_path, ip) {
979                    tracing::info!("VPN/Proxy detected: {} (source: {})", ip, feed_name);
980                    return true;
981                }
982            }
983        }
984        false
985    }
986
987    /// Helper method to check if an IP is present in a feed file
988    fn check_ip_in_feed(&self, file_path: &std::path::Path, ip: &std::net::IpAddr) -> bool {
989        if !file_path.exists() {
990            return false;
991        }
992
993        if let Ok(contents) = std::fs::read_to_string(file_path) {
994            for line in contents.lines() {
995                let line = line.trim();
996                if line.is_empty() || line.starts_with('#') {
997                    continue;
998                }
999
1000                // Check exact IP match
1001                if line == ip.to_string() {
1002                    return true;
1003                }
1004
1005                // Check CIDR network match
1006                if line.contains('/') {
1007                    match ip {
1008                        std::net::IpAddr::V4(ipv4) => {
1009                            if let Ok(network) = line.parse::<ipnetwork::Ipv4Network>()
1010                                && network.contains(*ipv4)
1011                            {
1012                                return true;
1013                            }
1014                        }
1015                        std::net::IpAddr::V6(ipv6) => {
1016                            if let Ok(network) = line.parse::<ipnetwork::Ipv6Network>()
1017                                && network.contains(*ipv6)
1018                            {
1019                                return true;
1020                            }
1021                        }
1022                    }
1023                }
1024            }
1025        }
1026
1027        false
1028    }
1029}
1030
1031/// Status of a threat intelligence feed
1032#[derive(Debug, Clone)]
1033pub enum FeedStatus {
1034    /// Feed is disabled
1035    Disabled,
1036
1037    /// Feed is enabled but not yet downloaded
1038    NotDownloaded,
1039
1040    /// Feed is active and up-to-date
1041    Active {
1042        last_updated: std::time::SystemTime,
1043        size_bytes: u64,
1044    },
1045
1046    /// Feed has an error
1047    Error(String),
1048}