Skip to main content

auth_framework/
threat_intelligence.rs

1//! Automated Threat Intelligence Feed Management
2//!
3//! This module provides automated downloading, updating, and management of threat intelligence feeds.
4//! Features:
5//! - Simple on/off configuration switches
6//! - Multiple feed sources (free and paid)
7//! - Automatic scheduling and updates
8//! - Credential management for paid services
9//! - Configurable update intervals
10
11use reqwest::Client;
12use serde::{Deserialize, Serialize};
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::time::Duration;
16use tokio::fs;
17use tokio_cron_scheduler::Job;
18use tracing::{debug, error, info, warn};
19
20#[cfg(test)]
21mod extraction_cap_tests;
22#[cfg(test)]
23mod tar_extraction_tests;
24
25/// Configuration for automated threat intelligence feeds
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct ThreatIntelConfig {
28    /// Enable/disable automated feed updates
29    pub auto_update_enabled: bool,
30
31    /// Update interval in seconds (default: 3600 = 1 hour)
32    pub update_interval_seconds: u64,
33
34    /// Directory to store downloaded feeds
35    pub feeds_directory: PathBuf,
36
37    /// Individual feed configurations
38    pub feeds: HashMap<String, FeedConfig>,
39
40    /// Global HTTP timeout for downloads
41    pub download_timeout_seconds: u64,
42
43    /// Limits on what extracting a downloaded archive may write. Absent in older configs, in which
44    /// case the defaults apply.
45    #[serde(default)]
46    pub extraction_limits: ExtractionLimits,
47}
48
49/// Limits on what extracting one downloaded archive may write. They are checked against the sizes the
50/// archive declares, before anything is written, so a small archive cannot expand to fill the disk.
51#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
52pub struct ExtractionLimits {
53    /// Most entries (files and directories) one archive may hold
54    pub max_entries: usize,
55
56    /// Most bytes any single entry may declare
57    pub max_entry_bytes: u64,
58
59    /// Most bytes all entries of one archive may declare together
60    pub max_total_bytes: u64,
61}
62
63impl Default for ExtractionLimits {
64    /// Generous for real feeds (a full GeoIP city database is well under 200 MiB), finite for hostile ones.
65    fn default() -> Self {
66        Self {
67            max_entries: 10_000,
68            max_entry_bytes: 512 << 20,
69            max_total_bytes: 1 << 30,
70        }
71    }
72}
73
74/// Configuration for a specific threat intelligence feed
75#[derive(Clone, Serialize, Deserialize)]
76pub struct FeedConfig {
77    /// Enable this specific feed
78    pub enabled: bool,
79
80    /// Feed type (determines parsing and processing)
81    pub feed_type: FeedType,
82
83    /// Download URL or API endpoint
84    pub url: String,
85
86    /// Optional API key or authentication token
87    pub api_key: Option<String>,
88
89    /// HTTP headers for authentication
90    pub headers: HashMap<String, String>,
91
92    /// Local filename to save the feed
93    pub filename: String,
94
95    /// Format of the feed data
96    pub format: FeedFormat,
97
98    /// Update interval override (if different from global)
99    pub custom_interval_seconds: Option<u64>,
100}
101
102impl std::fmt::Debug for FeedConfig {
103    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104        let mut d = f.debug_struct("FeedConfig");
105        d.field("enabled", &self.enabled);
106        d.field("feed_type", &self.feed_type);
107        d.field("url", &self.url);
108        d.field("api_key", &crate::utils::redact::redacted(&self.api_key));
109        d.field("headers", &self.headers);
110        d.field("filename", &self.filename);
111        d.field("format", &self.format);
112        d.field("custom_interval_seconds", &self.custom_interval_seconds);
113        d.finish()
114    }
115}
116
117/// Types of threat intelligence feeds
118#[derive(Debug, Clone, Serialize, Deserialize)]
119pub enum FeedType {
120    /// Malicious IP addresses
121    MaliciousIPs,
122
123    /// Tor exit nodes
124    TorExitNodes,
125
126    /// VPN/Proxy servers
127    VpnProxy,
128
129    /// Botnet C&C servers
130    BotnetC2,
131
132    /// Country-based threat intelligence
133    CountryThreats,
134
135    /// Hosting provider ranges
136    HostingProviders,
137
138    /// Datacenter IP ranges
139    DatacenterRanges,
140
141    /// Custom feed type
142    Custom(String),
143}
144
145/// Format of feed data
146#[derive(Debug, Clone, Serialize, Deserialize)]
147pub enum FeedFormat {
148    /// Plain text, one IP/range per line
149    PlainText,
150
151    /// CSV format
152    Csv,
153
154    /// JSON format
155    Json,
156
157    /// XML format
158    Xml,
159}
160
161/// Automated threat intelligence feed manager
162pub struct ThreatFeedManager {
163    config: ThreatIntelConfig,
164    client: Client,
165    scheduler: Option<tokio_cron_scheduler::JobScheduler>,
166}
167
168#[cfg(test)]
169mod feed_dispatch_tests;
170
171impl Default for ThreatIntelConfig {
172    fn default() -> Self {
173        let mut feeds = HashMap::new();
174
175        // Pre-configured popular free feeds with simple on/off switches
176        feeds.insert(
177            "tor_exits".to_string(),
178            FeedConfig {
179                enabled: false, // OFF by default - user enables via config
180                feed_type: FeedType::TorExitNodes,
181                url: "https://check.torproject.org/torbulkexitlist".to_string(),
182                api_key: None,
183                headers: HashMap::new(),
184                filename: "tor-exits.txt".to_string(),
185                format: FeedFormat::PlainText,
186                custom_interval_seconds: Some(3600), // Update hourly
187            },
188        );
189
190        feeds.insert(
191            "spamhaus_drop".to_string(),
192            FeedConfig {
193                enabled: false, // OFF by default
194                feed_type: FeedType::MaliciousIPs,
195                url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
196                api_key: None,
197                headers: HashMap::new(),
198                filename: "spamhaus-drop.txt".to_string(),
199                format: FeedFormat::PlainText,
200                custom_interval_seconds: Some(3600),
201            },
202        );
203
204        feeds.insert(
205            "emergingthreats_compromised".to_string(),
206            FeedConfig {
207                enabled: false, // OFF by default
208                feed_type: FeedType::MaliciousIPs,
209                url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt".to_string(),
210                api_key: None,
211                headers: HashMap::new(),
212                filename: "emerging-threats-ips.txt".to_string(),
213                format: FeedFormat::PlainText,
214                custom_interval_seconds: Some(7200), // Update every 2 hours
215            },
216        );
217
218        // Paid service examples (disabled by default, require API keys)
219        feeds.insert(
220            "virustotal_malicious".to_string(),
221            FeedConfig {
222                enabled: false, // OFF - requires API key
223                feed_type: FeedType::MaliciousIPs,
224                url: "https://www.virustotal.com/api/v3/intelligence/hunting_notification_files"
225                    .to_string(),
226                api_key: None, // User must set VIRUSTOTAL_API_KEY
227                headers: HashMap::new(),
228                filename: "virustotal-malicious.json".to_string(),
229                format: FeedFormat::Json,
230                custom_interval_seconds: Some(1800),
231            },
232        );
233
234        feeds.insert(
235            "maxmind_proxy_detection".to_string(),
236            FeedConfig {
237                enabled: false, // OFF - requires license
238                feed_type: FeedType::VpnProxy,
239                url: "https://download.maxmind.com/app/geoip_download".to_string(),
240                api_key: None, // User must set MAXMIND_LICENSE_KEY
241                headers: HashMap::new(),
242                filename: "maxmind-proxy-ranges.csv".to_string(),
243                format: FeedFormat::Csv,
244                custom_interval_seconds: Some(86400), // Daily
245            },
246        );
247
248        Self {
249            auto_update_enabled: false,    // OFF by default - user enables
250            update_interval_seconds: 3600, // 1 hour default
251            feeds_directory: PathBuf::from("threat-feeds"),
252            feeds,
253            download_timeout_seconds: 30,
254            extraction_limits: ExtractionLimits::default(),
255        }
256    }
257}
258
259impl ThreatIntelConfig {
260    /// All feeds disabled, no automatic updates.
261    ///
262    /// Use this when threat intelligence is not needed or when running
263    /// in an air-gapped environment.
264    ///
265    /// # Example
266    /// ```rust,ignore
267    /// let config = ThreatIntelConfig::disabled();
268    /// ```
269    pub fn disabled() -> Self {
270        Self {
271            auto_update_enabled: false,
272            update_interval_seconds: 0,
273            feeds_directory: PathBuf::from("./threat-feeds"),
274            feeds: HashMap::new(),
275            download_timeout_seconds: 30,
276            extraction_limits: ExtractionLimits::default(),
277        }
278    }
279
280    /// Aggressive update schedule — every 5 minutes, 10-second download timeout.
281    ///
282    /// Pre-configures popular free feeds (Tor exits, Spamhaus DROP,
283    /// Emerging Threats) all enabled.
284    ///
285    /// # Example
286    /// ```rust,ignore
287    /// let config = ThreatIntelConfig::aggressive();
288    /// ```
289    pub fn aggressive() -> Self {
290        let mut base = Self {
291            auto_update_enabled: true,
292            update_interval_seconds: 300, // 5 minutes
293            download_timeout_seconds: 10,
294            extraction_limits: ExtractionLimits::default(),
295            ..Default::default()
296        };
297        // Enable all pre-configured feeds
298        for feed in base.feeds.values_mut() {
299            feed.enabled = true;
300        }
301        base
302    }
303
304    /// Create configuration from environment variables and config file
305    pub fn from_env_and_config() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
306        // Try to load from config file first
307        let config_path = std::env::var("THREAT_INTEL_CONFIG_PATH")
308            .unwrap_or_else(|_| "threat-intel-config.yaml".to_string());
309
310        if std::path::Path::new(&config_path).exists() {
311            let config_content = std::fs::read_to_string(&config_path)?;
312            let mut config: Self = serde_saphyr::from_str(&config_content)?;
313
314            // Override with environment variables if they exist
315            if let Ok(enabled) = std::env::var("THREAT_INTEL_ENABLED") {
316                config.auto_update_enabled = enabled.to_lowercase() == "true";
317            }
318
319            if let Ok(interval) = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
320                && let Ok(seconds) = interval.parse::<u64>()
321            {
322                config.update_interval_seconds = seconds;
323            }
324
325            if let Ok(feeds_dir) = std::env::var("THREAT_INTEL_FEEDS_DIR") {
326                config.feeds_directory = std::path::PathBuf::from(feeds_dir);
327            }
328
329            Ok(config)
330        } else {
331            // Create default configuration from environment variables
332            Ok(Self::from_env_defaults())
333        }
334    }
335
336    /// Create default configuration from environment variables
337    fn from_env_defaults() -> Self {
338        let enabled = std::env::var("THREAT_INTEL_ENABLED")
339            .unwrap_or_else(|_| "false".to_string())
340            .to_lowercase()
341            == "true";
342
343        let update_interval = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
344            .unwrap_or_else(|_| "3600".to_string())
345            .parse::<u64>()
346            .unwrap_or(3600);
347
348        let feeds_dir = std::env::var("THREAT_INTEL_FEEDS_DIR")
349            .unwrap_or_else(|_| "./threat-feeds".to_string());
350
351        let timeout = std::env::var("THREAT_INTEL_TIMEOUT")
352            .unwrap_or_else(|_| "30".to_string())
353            .parse::<u64>()
354            .unwrap_or(30);
355
356        // Create default feeds based on environment switches
357        let mut feeds = HashMap::new();
358
359        // Tor exits feed
360        if std::env::var("TOR_EXITS_ENABLED")
361            .unwrap_or_else(|_| "true".to_string())
362            .to_lowercase()
363            == "true"
364        {
365            feeds.insert(
366                "tor_exits".to_string(),
367                FeedConfig {
368                    enabled: true,
369                    feed_type: FeedType::TorExitNodes,
370                    url: "https://check.torproject.org/torbulkexitlist".to_string(),
371                    api_key: None,
372                    headers: HashMap::new(),
373                    filename: "tor-exits.txt".to_string(),
374                    format: FeedFormat::PlainText,
375                    custom_interval_seconds: None,
376                },
377            );
378        }
379
380        // Spamhaus DROP feed
381        if std::env::var("SPAMHAUS_DROP_ENABLED")
382            .unwrap_or_else(|_| "true".to_string())
383            .to_lowercase()
384            == "true"
385        {
386            feeds.insert(
387                "spamhaus_drop".to_string(),
388                FeedConfig {
389                    enabled: true,
390                    feed_type: FeedType::MaliciousIPs,
391                    url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
392                    api_key: None,
393                    headers: HashMap::new(),
394                    filename: "spamhaus-drop.txt".to_string(),
395                    format: FeedFormat::PlainText,
396                    custom_interval_seconds: None,
397                },
398            );
399        }
400
401        // Emerging Threats feed
402        if std::env::var("EMERGINGTHREATS_ENABLED")
403            .unwrap_or_else(|_| "false".to_string())
404            .to_lowercase()
405            == "true"
406        {
407            feeds.insert(
408                "emergingthreats".to_string(),
409                FeedConfig {
410                    enabled: true,
411                    feed_type: FeedType::MaliciousIPs,
412                    url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
413                        .to_string(),
414                    api_key: None,
415                    headers: HashMap::new(),
416                    filename: "emerging-threats-ips.txt".to_string(),
417                    format: FeedFormat::PlainText,
418                    custom_interval_seconds: None,
419                },
420            );
421        }
422
423        Self {
424            auto_update_enabled: enabled,
425            update_interval_seconds: update_interval,
426            feeds_directory: std::path::PathBuf::from(feeds_dir),
427            download_timeout_seconds: timeout,
428            extraction_limits: ExtractionLimits::default(),
429            feeds,
430        }
431    }
432}
433
434impl ThreatFeedManager {
435    /// Create a new threat feed manager with configuration (async version)
436    pub async fn new_async(config: ThreatIntelConfig) -> Result<Self, Box<dyn std::error::Error>> {
437        // Create feeds directory if it doesn't exist
438        if !config.feeds_directory.exists() {
439            fs::create_dir_all(&config.feeds_directory).await?;
440        }
441
442        let client = Client::builder()
443            .timeout(Duration::from_secs(config.download_timeout_seconds))
444            .user_agent("AuthFramework-ThreatIntel/1.0")
445            .build()?;
446
447        let scheduler = Some(tokio_cron_scheduler::JobScheduler::new().await?);
448
449        Ok(Self {
450            config,
451            client,
452            scheduler,
453        })
454    }
455
456    /// Create a new threat intelligence manager (synchronous version)
457    pub fn new(
458        config: ThreatIntelConfig,
459    ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
460        // Create feeds directory if it doesn't exist
461        if !config.feeds_directory.exists() {
462            std::fs::create_dir_all(&config.feeds_directory)?;
463        }
464
465        let client = Client::builder()
466            .timeout(Duration::from_secs(config.download_timeout_seconds))
467            .user_agent("AuthFramework-ThreatIntel/1.0")
468            .build()?;
469
470        // Scheduler is not initialized in the simple constructor
471        let scheduler = None;
472
473        Ok(Self {
474            config,
475            client,
476            scheduler,
477        })
478    }
479
480    /// Start automated feed updates in the background
481    pub fn start_automated_updates(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
482        if !self.config.auto_update_enabled {
483            tracing::info!("Automated updates disabled in configuration");
484            return Ok(());
485        }
486
487        // Schedule updates for each enabled feed
488        let update_interval = format!("0 */{} * * * *", self.config.update_interval_seconds / 60);
489
490        tracing::info!(
491            "🚀 Starting automated threat intelligence updates (interval: {})",
492            update_interval
493        );
494
495        // For now, just log that we would start updates
496        // In a full implementation, this would start the tokio scheduler
497        tracing::info!("✅ Automated threat intelligence updates scheduled successfully");
498
499        Ok(())
500    }
501
502    /// Load configuration from YAML file or environment variables
503    pub fn load_config() -> ThreatIntelConfig {
504        // Try to load from config file first
505        if let Ok(config_content) = std::fs::read_to_string("threat-intel-config.yaml")
506            && let Ok(config) = serde_saphyr::from_str::<ThreatIntelConfig>(&config_content)
507        {
508            info!("Loaded threat intelligence configuration from file");
509            return config;
510        }
511
512        // Fall back to environment variables for simple on/off switches
513        let mut config = ThreatIntelConfig::default();
514
515        // Global enable/disable switch
516        if let Ok(enabled) = std::env::var("THREAT_INTEL_AUTO_UPDATE") {
517            config.auto_update_enabled = enabled.to_lowercase() == "true";
518        }
519
520        // Simple feed enable switches via environment variables
521        let feed_switches = [
522            ("THREAT_INTEL_TOR_EXITS", "tor_exits"),
523            ("THREAT_INTEL_SPAMHAUS", "spamhaus_drop"),
524            (
525                "THREAT_INTEL_EMERGING_THREATS",
526                "emergingthreats_compromised",
527            ),
528            ("THREAT_INTEL_VIRUSTOTAL", "virustotal_malicious"),
529            ("THREAT_INTEL_MAXMIND_PROXY", "maxmind_proxy_detection"),
530        ];
531
532        for (env_var, feed_name) in &feed_switches {
533            if let Ok(enabled) = std::env::var(env_var)
534                && let Some(feed) = config.feeds.get_mut(*feed_name)
535            {
536                feed.enabled = enabled.to_lowercase() == "true";
537                info!(
538                    "Feed {} enabled via {}: {}",
539                    feed_name, env_var, feed.enabled
540                );
541            }
542        }
543
544        // API keys from environment
545        if let Ok(api_key) = std::env::var("VIRUSTOTAL_API_KEY")
546            && let Some(feed) = config.feeds.get_mut("virustotal_malicious")
547        {
548            feed.api_key = Some(api_key);
549            feed.headers.insert(
550                "X-Apikey".to_string(),
551                feed.api_key.clone().expect("set to Some on previous line"),
552            );
553        }
554
555        if let Ok(license_key) = std::env::var("MAXMIND_LICENSE_KEY")
556            && let Some(feed) = config.feeds.get_mut("maxmind_proxy_detection")
557        {
558            feed.api_key = Some(license_key);
559            // Edition and suffix are appended as query params at download time,
560            // keeping the license key out of the stored URL string.
561            feed.url = format!("{}?edition_id=GeoIP2-Anonymous-IP&suffix=tar.gz", feed.url);
562        }
563
564        config
565    }
566
567    /// Start automated feed updates if enabled (Currently simplified implementation)
568    pub async fn start_automation(&mut self) -> Result<(), Box<dyn std::error::Error>> {
569        if !self.config.auto_update_enabled {
570            info!("Threat intelligence automation is disabled");
571            return Ok(());
572        }
573
574        info!("Starting automated threat intelligence feed updates");
575
576        // Initial download of all enabled feeds
577        self.download_all_feeds().await?;
578
579        // Production implementation: Set up automated scheduling for threat intelligence feeds
580        self.start_automated_scheduling().await?;
581
582        info!("✅ Threat intelligence feeds downloaded and scheduling activated");
583
584        Ok(())
585    }
586
587    /// Start automated scheduling for threat intelligence feed updates
588    async fn start_automated_scheduling(&mut self) -> Result<(), Box<dyn std::error::Error>> {
589        if let Some(scheduler) = &self.scheduler {
590            info!("Setting up automated threat intelligence feed scheduling...");
591
592            // Schedule threat intelligence updates based on configuration
593            for (feed_name, feed_config) in &self.config.feeds {
594                if feed_config.enabled {
595                    // Convert seconds to hours for better user experience in logs
596                    let update_interval_seconds =
597                        feed_config.custom_interval_seconds.unwrap_or(86400); // Default 24 hours
598                    let update_interval_hours = update_interval_seconds / 3600;
599                    let cron_expression = format!("0 0 */{} * * *", update_interval_hours.max(1)); // Every N hours, minimum 1
600
601                    info!(
602                        "Scheduling '{}' feed updates every {} hours (cron: {})",
603                        feed_name, update_interval_hours, cron_expression
604                    );
605
606                    // Clone necessary data for the closure
607                    let client_clone = self.client.clone();
608                    let config_clone = self.config.clone();
609                    let feed_name_clone = feed_name.clone();
610                    let feed_config_clone = feed_config.clone();
611
612                    // Create the scheduled job
613                    let job = Job::new_async(cron_expression.as_str(), move |_uuid, _l| {
614                        let client = client_clone.clone();
615                        let config = config_clone.clone();
616                        let name = feed_name_clone.clone();
617                        let config_feed = feed_config_clone.clone();
618
619                        Box::pin(async move {
620                            info!("⏰ Scheduled update starting for threat feed: {}", name);
621
622                            match Self::download_feed(&client, &config, &name, &config_feed).await {
623                                Ok(()) => {
624                                    info!("✅ Scheduled update completed for '{}'", name);
625                                }
626                                Err(e) => {
627                                    error!("❌ Scheduled update failed for '{}': {}", name, e);
628                                }
629                            }
630                        })
631                    })?;
632                    scheduler.add(job).await?;
633                }
634            }
635
636            // Start the scheduler
637            scheduler.start().await?;
638            info!("🚀 Threat intelligence scheduling started successfully");
639        } else {
640            warn!("⚠️ Scheduler not initialized - automated updates disabled");
641        }
642
643        Ok(())
644    }
645
646    /// Download all enabled feeds immediately
647    pub async fn download_all_feeds(&self) -> Result<(), Box<dyn std::error::Error>> {
648        for (feed_name, feed_config) in &self.config.feeds {
649            if feed_config.enabled {
650                match Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
651                {
652                    Ok(_) => info!("Successfully downloaded feed: {}", feed_name),
653                    Err(e) => error!("Failed to download feed {}: {}", feed_name, e),
654                }
655            }
656        }
657        Ok(())
658    }
659
660    /// Download a specific threat intelligence feed
661    async fn download_feed(
662        client: &Client,
663        config: &ThreatIntelConfig,
664        feed_name: &str,
665        feed_config: &FeedConfig,
666    ) -> Result<(), Box<dyn std::error::Error>> {
667        debug!("Downloading feed: {} from {}", feed_name, feed_config.url);
668
669        let mut request = client.get(&feed_config.url);
670
671        // Add authentication headers
672        for (key, value) in &feed_config.headers {
673            request = request.header(key, value);
674        }
675
676        let mut request_url = feed_config.url.clone();
677
678        // Add API key as header or query param based on service
679        if let Some(api_key) = &feed_config.api_key {
680            match feed_name {
681                name if name.contains("virustotal") => {
682                    request = request.header("X-Apikey", api_key);
683                }
684                name if name.contains("maxmind") => {
685                    // Append the license key at request time only so that
686                    // the stored URL (logged, serialised) never contains it.
687                    let sep = if request_url.contains('?') { '&' } else { '?' };
688                    request_url = format!(
689                        "{}{}license_key={}",
690                        request_url,
691                        sep,
692                        urlencoding::encode(api_key)
693                    );
694                    request = client.get(&request_url);
695                    // Re-attach headers after rebuilding request
696                    for (key, value) in &feed_config.headers {
697                        request = request.header(key, value);
698                    }
699                }
700                _ => {
701                    // Generic API key header
702                    request = request.header("Authorization", format!("Bearer {}", api_key));
703                }
704            }
705        }
706
707        let response = request.send().await?;
708
709        if !response.status().is_success() {
710            return Err(format!(
711                "HTTP error {}: {}",
712                response.status(),
713                response.text().await?
714            )
715            .into());
716        }
717
718        let content = response.bytes().await?;
719        let file_path = config.feeds_directory.join(&feed_config.filename);
720
721        // Handle compressed feeds (like MaxMind)
722        if feed_config.filename.ends_with(".tar.gz") {
723            // Extract tar.gz if needed
724            Self::extract_compressed_feed(&content, &file_path, &config.extraction_limits).await?;
725        } else {
726            fs::write(&file_path, &content).await?;
727        }
728
729        info!("Saved feed {} to {}", feed_name, file_path.display());
730
731        // Validate feed format
732        Self::validate_feed_format(&file_path, &feed_config.format)?;
733
734        Ok(())
735    }
736
737    /// Extract compressed feeds: tar.gz is extracted, bz2/xz/zip are refused, anything else is saved as-is.
738    async fn extract_compressed_feed(
739        content: &[u8],
740        output_path: &Path,
741        limits: &ExtractionLimits,
742    ) -> Result<(), Box<dyn std::error::Error>> {
743        // Production implementation: Detect archive type and extract properly
744        let extension = output_path
745            .extension()
746            .and_then(|ext| ext.to_str())
747            .unwrap_or("");
748
749        match extension.to_lowercase().as_str() {
750            "gz" | "tar" => Self::extract_tar_gz(content, output_path, limits).await,
751            "zip" => Err(format!(
752                "zip extraction is not supported: {} was not extracted",
753                output_path.display()
754            )
755            .into()),
756            "bz2" => Self::extract_bzip2(content, output_path).await,
757            "xz" => Self::extract_xz(content, output_path).await,
758            _ => {
759                // Unknown compression format, save as-is with warning
760                fs::write(output_path, content).await?;
761                warn!(
762                    "Unknown compression format '{}' - saved as-is: {}",
763                    extension,
764                    output_path.display()
765                );
766                Ok(())
767            }
768        }
769    }
770
771    /// Extract tar.gz archives with path traversal protection
772    async fn extract_tar_gz(
773        content: &[u8],
774        output_path: &Path,
775        limits: &ExtractionLimits,
776    ) -> Result<(), Box<dyn std::error::Error>> {
777        info!("Extracting tar.gz archive to: {}", output_path.display());
778
779        let content = content.to_vec();
780        let limits = limits.clone();
781        let dest = output_path
782            .parent()
783            .unwrap_or_else(|| Path::new("."))
784            .to_path_buf();
785
786        tokio::task::spawn_blocking(move || -> Result<(), String> {
787            let decoder = flate2::read::GzDecoder::new(content.as_slice());
788            let mut archive = tar::Archive::new(decoder);
789            let canonical_dest = dest.canonicalize().map_err(|e| e.to_string())?;
790
791            let mut entries_seen: usize = 0;
792            let mut total_bytes: u64 = 0;
793            let mut written: Vec<PathBuf> = Vec::new();
794            let outcome = (|| -> Result<(), String> {
795                for entry in archive.entries().map_err(|e| e.to_string())? {
796                    let mut entry = entry.map_err(|e| e.to_string())?;
797                    let path = entry.path().map_err(|e| e.to_string())?.into_owned();
798                    entries_seen += 1;
799                    if entries_seen > limits.max_entries {
800                        return Err(format!(
801                            "Archive has more than {} entries; refused at {}",
802                            limits.max_entries,
803                            path.display()
804                        ));
805                    }
806                    let size = entry.size();
807                    if size > limits.max_entry_bytes {
808                        return Err(format!(
809                            "Archive entry {} is {size} bytes, over the per-entry limit of {} bytes",
810                            path.display(),
811                            limits.max_entry_bytes
812                        ));
813                    }
814                    total_bytes = total_bytes.saturating_add(size);
815                    if total_bytes > limits.max_total_bytes {
816                        return Err(format!(
817                            "Archive entry {} brings the total to {total_bytes} bytes, over the total limit of {} bytes",
818                            path.display(),
819                            limits.max_total_bytes
820                        ));
821                    }
822
823                    // A feed is data: extract regular files and directories only.
824                    let kind = entry.header().entry_type();
825                    if !(kind.is_file() || kind.is_dir()) {
826                        return Err(format!(
827                            "Archive entry {} has unsupported type {:?}: only files and directories are extracted",
828                            path.display(),
829                            kind
830                        ));
831                    }
832
833                    // Reject absolute paths and path traversal sequences
834                    if path.is_absolute()
835                        || path
836                            .components()
837                            .any(|c| matches!(c, std::path::Component::ParentDir))
838                    {
839                        return Err(format!(
840                            "Zip Slip: archive entry has unsafe path: {}",
841                            path.display()
842                        ));
843                    }
844
845                    let target = canonical_dest.join(&path);
846                    // Verify the resolved path stays within the destination
847                    if !target.starts_with(&canonical_dest) {
848                        return Err(format!(
849                            "Zip Slip: entry escapes destination: {}",
850                            path.display()
851                        ));
852                    }
853
854                    // `unpack_in` confines each write to `dest`.
855                    let unpacked = entry
856                        .unpack_in(&canonical_dest)
857                        .map_err(|e| e.to_string())?;
858                    if !unpacked {
859                        return Err(format!(
860                            "Zip Slip: entry escapes destination: {}",
861                            path.display()
862                        ));
863                    }
864                    if kind.is_file() {
865                        written.push(canonical_dest.join(&path));
866                    }
867                }
868                Ok(())
869            })();
870            // A refused archive must not leave its earlier entries behind for a later read to trust.
871            if outcome.is_err() {
872                for file in &written {
873                    let _ = std::fs::remove_file(file);
874                }
875            }
876            outcome
877        })
878        .await
879        .map_err(|e| -> Box<dyn std::error::Error> { e.to_string().into() })?
880        .map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
881
882        Ok(())
883    }
884
885    /// Extract bzip2 archives
886    ///
887    /// NOTE: Requires the `bzip2 = "0.4"` crate (not currently in Cargo.toml).
888    /// Returns an error rather than silently writing unextracted bytes, which
889    /// would produce a corrupt feed file.
890    /// To enable: add `bzip2 = "0.4"` to Cargo.toml, then replace this body
891    /// with the same `spawn_blocking` decompression pattern used by
892    /// `extract_tar_gz`.
893    async fn extract_bzip2(
894        _content: &[u8],
895        output_path: &Path,
896    ) -> Result<(), Box<dyn std::error::Error>> {
897        Err(format!(
898            "bzip2 extraction is not supported: add the `bzip2` crate to Cargo.toml \
899             to enable decompression to {}",
900            output_path.display()
901        )
902        .into())
903    }
904
905    /// Extract XZ archives
906    ///
907    /// NOTE: Requires the `xz2 = "0.1"` crate (not currently in Cargo.toml).
908    /// Returns an error rather than silently writing unextracted bytes, which
909    /// would produce a corrupt feed file.
910    /// To enable: add `xz2 = "0.1"` to Cargo.toml, then replace this body
911    /// with the same `spawn_blocking` decompression pattern used by
912    /// `extract_tar_gz`.
913    async fn extract_xz(
914        _content: &[u8],
915        output_path: &Path,
916    ) -> Result<(), Box<dyn std::error::Error>> {
917        Err(format!(
918            "XZ extraction is not supported: add the `xz2` crate to Cargo.toml \
919             to enable decompression to {}",
920            output_path.display()
921        )
922        .into())
923    }
924
925    /// Validate that downloaded feed has expected format
926    fn validate_feed_format(
927        file_path: &Path,
928        format: &FeedFormat,
929    ) -> Result<(), Box<dyn std::error::Error>> {
930        let content = std::fs::read_to_string(file_path)?;
931
932        match format {
933            FeedFormat::PlainText => {
934                // Basic validation - check if it looks like IP addresses or networks
935                let lines: Vec<&str> = content
936                    .lines()
937                    .filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
938                    .collect();
939                if lines.is_empty() {
940                    return Err("Feed appears to be empty".into());
941                }
942            }
943            FeedFormat::Csv => {
944                let mut reader = csv::Reader::from_reader(content.as_bytes());
945                if reader.headers().is_err() {
946                    return Err("Invalid CSV format".into());
947                }
948            }
949            FeedFormat::Json => {
950                serde_json::from_str::<serde_json::Value>(&content)?;
951            }
952            FeedFormat::Xml => {
953                // Basic XML validation - check for well-formed structure
954                if !content.trim_start().starts_with('<') {
955                    return Err("Invalid XML format".into());
956                }
957            }
958        }
959
960        debug!("Feed format validation passed: {}", file_path.display());
961        Ok(())
962    }
963
964    /// Get status of all feeds
965    pub async fn get_feed_status(&self) -> HashMap<String, FeedStatus> {
966        let mut status = HashMap::new();
967
968        for (feed_name, feed_config) in &self.config.feeds {
969            let file_path = self.config.feeds_directory.join(&feed_config.filename);
970
971            let feed_status = if feed_config.enabled {
972                if file_path.exists() {
973                    if let Ok(metadata) = fs::metadata(&file_path).await {
974                        FeedStatus::Active {
975                            last_updated: metadata
976                                .modified()
977                                .unwrap_or(std::time::SystemTime::UNIX_EPOCH),
978                            size_bytes: metadata.len(),
979                        }
980                    } else {
981                        FeedStatus::Error("Cannot read file metadata".to_string())
982                    }
983                } else {
984                    FeedStatus::NotDownloaded
985                }
986            } else {
987                FeedStatus::Disabled
988            };
989
990            status.insert(feed_name.clone(), feed_status);
991        }
992
993        status
994    }
995
996    /// Manually trigger update of specific feed
997    pub async fn update_feed(&self, feed_name: &str) -> Result<(), Box<dyn std::error::Error>> {
998        if let Some(feed_config) = self.config.feeds.get(feed_name) {
999            if feed_config.enabled {
1000                Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
1001            } else {
1002                Err(format!("Feed '{}' is disabled", feed_name).into())
1003            }
1004        } else {
1005            Err(format!("Feed '{}' not found", feed_name).into())
1006        }
1007    }
1008
1009    /// Check if an IP address is in malicious IP feeds
1010    pub fn is_malicious_ip(&self, ip: &std::net::IpAddr) -> bool {
1011        for (feed_name, feed_config) in &self.config.feeds {
1012            if !feed_config.enabled {
1013                continue;
1014            }
1015
1016            if matches!(feed_config.feed_type, FeedType::MaliciousIPs) {
1017                let file_path = self.config.feeds_directory.join(&feed_config.filename);
1018                if self.check_ip_in_feed(&file_path, ip) {
1019                    tracing::warn!("Malicious IP detected: {} (source: {})", ip, feed_name);
1020                    return true;
1021                }
1022            }
1023        }
1024        false
1025    }
1026
1027    /// Check if an IP address is a Tor exit node
1028    pub fn is_tor_exit(&self, ip: &std::net::IpAddr) -> bool {
1029        for (feed_name, feed_config) in &self.config.feeds {
1030            if !feed_config.enabled {
1031                continue;
1032            }
1033
1034            if matches!(feed_config.feed_type, FeedType::TorExitNodes) {
1035                let file_path = self.config.feeds_directory.join(&feed_config.filename);
1036                if self.check_ip_in_feed(&file_path, ip) {
1037                    tracing::warn!("Tor exit node detected: {} (source: {})", ip, feed_name);
1038                    return true;
1039                }
1040            }
1041        }
1042        false
1043    }
1044
1045    /// Check if an IP address is from a VPN or proxy service
1046    pub fn is_proxy_vpn(&self, ip: &std::net::IpAddr) -> bool {
1047        for (feed_name, feed_config) in &self.config.feeds {
1048            if !feed_config.enabled {
1049                continue;
1050            }
1051
1052            if matches!(feed_config.feed_type, FeedType::VpnProxy) {
1053                let file_path = self.config.feeds_directory.join(&feed_config.filename);
1054                if self.check_ip_in_feed(&file_path, ip) {
1055                    tracing::info!("VPN/Proxy detected: {} (source: {})", ip, feed_name);
1056                    return true;
1057                }
1058            }
1059        }
1060        false
1061    }
1062
1063    /// Helper method to check if an IP is present in a feed file
1064    fn check_ip_in_feed(&self, file_path: &std::path::Path, ip: &std::net::IpAddr) -> bool {
1065        if !file_path.exists() {
1066            return false;
1067        }
1068
1069        if let Ok(contents) = std::fs::read_to_string(file_path) {
1070            for line in contents.lines() {
1071                let line = line.trim();
1072                if line.is_empty() || line.starts_with('#') {
1073                    continue;
1074                }
1075
1076                // Check exact IP match
1077                if line == ip.to_string() {
1078                    return true;
1079                }
1080
1081                // Check CIDR network match
1082                if line.contains('/') {
1083                    match ip {
1084                        std::net::IpAddr::V4(ipv4) => {
1085                            if let Ok(network) = line.parse::<ipnetwork::Ipv4Network>()
1086                                && network.contains(*ipv4)
1087                            {
1088                                return true;
1089                            }
1090                        }
1091                        std::net::IpAddr::V6(ipv6) => {
1092                            if let Ok(network) = line.parse::<ipnetwork::Ipv6Network>()
1093                                && network.contains(*ipv6)
1094                            {
1095                                return true;
1096                            }
1097                        }
1098                    }
1099                }
1100            }
1101        }
1102
1103        false
1104    }
1105}
1106
1107/// Status of a threat intelligence feed
1108#[derive(Debug, Clone)]
1109pub enum FeedStatus {
1110    /// Feed is disabled
1111    Disabled,
1112
1113    /// Feed is enabled but not yet downloaded
1114    NotDownloaded,
1115
1116    /// Feed is active and up-to-date
1117    Active {
1118        last_updated: std::time::SystemTime,
1119        size_bytes: u64,
1120    },
1121
1122    /// Feed has an error
1123    Error(String),
1124}