Skip to main content

auth_framework/
threat_intelligence.rs

1//! Automated Threat Intelligence Feed Management
2//!
3//! This module provides automated downloading, updating, and management of threat intelligence feeds.
4//! Features:
5//! - Simple on/off configuration switches
6//! - Multiple feed sources (free and paid)
7//! - Automatic scheduling and updates
8//! - Credential management for paid services
9//! - Configurable update intervals
10
11use reqwest::Client;
12use serde::{Deserialize, Serialize};
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::time::Duration;
16use tokio::fs;
17use tokio_cron_scheduler::Job;
18use tracing::{debug, error, info, warn};
19
20#[cfg(test)]
21mod extraction_cap_tests;
22#[cfg(test)]
23mod tar_extraction_tests;
24
25/// Configuration for automated threat intelligence feeds
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct ThreatIntelConfig {
28    /// Enable/disable automated feed updates
29    pub auto_update_enabled: bool,
30
31    /// Update interval in seconds (default: 3600 = 1 hour)
32    pub update_interval_seconds: u64,
33
34    /// Directory to store downloaded feeds
35    pub feeds_directory: PathBuf,
36
37    /// Individual feed configurations
38    pub feeds: HashMap<String, FeedConfig>,
39
40    /// Global HTTP timeout for downloads
41    pub download_timeout_seconds: u64,
42
43    /// Limits on what extracting a downloaded archive may write. Absent in older configs, in which
44    /// case the defaults apply.
45    #[serde(default)]
46    pub extraction_limits: ExtractionLimits,
47}
48
49/// Limits on what extracting one downloaded archive may write. They are checked against the sizes the
50/// archive declares, before anything is written, so a small archive cannot expand to fill the disk.
51#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
52pub struct ExtractionLimits {
53    /// Most entries (files and directories) one archive may hold
54    pub max_entries: usize,
55
56    /// Most bytes any single entry may declare
57    pub max_entry_bytes: u64,
58
59    /// Most bytes all entries of one archive may declare together
60    pub max_total_bytes: u64,
61}
62
63impl Default for ExtractionLimits {
64    /// Generous for real feeds (a full GeoIP city database is well under 200 MiB), finite for hostile ones.
65    fn default() -> Self {
66        Self {
67            max_entries: 10_000,
68            max_entry_bytes: 512 << 20,
69            max_total_bytes: 1 << 30,
70        }
71    }
72}
73
74/// Configuration for a specific threat intelligence feed
75#[derive(Debug, Clone, Serialize, Deserialize)]
76pub struct FeedConfig {
77    /// Enable this specific feed
78    pub enabled: bool,
79
80    /// Feed type (determines parsing and processing)
81    pub feed_type: FeedType,
82
83    /// Download URL or API endpoint
84    pub url: String,
85
86    /// Optional API key or authentication token
87    pub api_key: Option<String>,
88
89    /// HTTP headers for authentication
90    pub headers: HashMap<String, String>,
91
92    /// Local filename to save the feed
93    pub filename: String,
94
95    /// Format of the feed data
96    pub format: FeedFormat,
97
98    /// Update interval override (if different from global)
99    pub custom_interval_seconds: Option<u64>,
100}
101
102/// Types of threat intelligence feeds
103#[derive(Debug, Clone, Serialize, Deserialize)]
104pub enum FeedType {
105    /// Malicious IP addresses
106    MaliciousIPs,
107
108    /// Tor exit nodes
109    TorExitNodes,
110
111    /// VPN/Proxy servers
112    VpnProxy,
113
114    /// Botnet C&C servers
115    BotnetC2,
116
117    /// Country-based threat intelligence
118    CountryThreats,
119
120    /// Hosting provider ranges
121    HostingProviders,
122
123    /// Datacenter IP ranges
124    DatacenterRanges,
125
126    /// Custom feed type
127    Custom(String),
128}
129
130/// Format of feed data
131#[derive(Debug, Clone, Serialize, Deserialize)]
132pub enum FeedFormat {
133    /// Plain text, one IP/range per line
134    PlainText,
135
136    /// CSV format
137    Csv,
138
139    /// JSON format
140    Json,
141
142    /// XML format
143    Xml,
144}
145
146/// Automated threat intelligence feed manager
147pub struct ThreatFeedManager {
148    config: ThreatIntelConfig,
149    client: Client,
150    scheduler: Option<tokio_cron_scheduler::JobScheduler>,
151}
152
153#[cfg(test)]
154mod feed_dispatch_tests;
155
156impl Default for ThreatIntelConfig {
157    fn default() -> Self {
158        let mut feeds = HashMap::new();
159
160        // Pre-configured popular free feeds with simple on/off switches
161        feeds.insert(
162            "tor_exits".to_string(),
163            FeedConfig {
164                enabled: false, // OFF by default - user enables via config
165                feed_type: FeedType::TorExitNodes,
166                url: "https://check.torproject.org/torbulkexitlist".to_string(),
167                api_key: None,
168                headers: HashMap::new(),
169                filename: "tor-exits.txt".to_string(),
170                format: FeedFormat::PlainText,
171                custom_interval_seconds: Some(3600), // Update hourly
172            },
173        );
174
175        feeds.insert(
176            "spamhaus_drop".to_string(),
177            FeedConfig {
178                enabled: false, // OFF by default
179                feed_type: FeedType::MaliciousIPs,
180                url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
181                api_key: None,
182                headers: HashMap::new(),
183                filename: "spamhaus-drop.txt".to_string(),
184                format: FeedFormat::PlainText,
185                custom_interval_seconds: Some(3600),
186            },
187        );
188
189        feeds.insert(
190            "emergingthreats_compromised".to_string(),
191            FeedConfig {
192                enabled: false, // OFF by default
193                feed_type: FeedType::MaliciousIPs,
194                url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt".to_string(),
195                api_key: None,
196                headers: HashMap::new(),
197                filename: "emerging-threats-ips.txt".to_string(),
198                format: FeedFormat::PlainText,
199                custom_interval_seconds: Some(7200), // Update every 2 hours
200            },
201        );
202
203        // Paid service examples (disabled by default, require API keys)
204        feeds.insert(
205            "virustotal_malicious".to_string(),
206            FeedConfig {
207                enabled: false, // OFF - requires API key
208                feed_type: FeedType::MaliciousIPs,
209                url: "https://www.virustotal.com/api/v3/intelligence/hunting_notification_files"
210                    .to_string(),
211                api_key: None, // User must set VIRUSTOTAL_API_KEY
212                headers: HashMap::new(),
213                filename: "virustotal-malicious.json".to_string(),
214                format: FeedFormat::Json,
215                custom_interval_seconds: Some(1800),
216            },
217        );
218
219        feeds.insert(
220            "maxmind_proxy_detection".to_string(),
221            FeedConfig {
222                enabled: false, // OFF - requires license
223                feed_type: FeedType::VpnProxy,
224                url: "https://download.maxmind.com/app/geoip_download".to_string(),
225                api_key: None, // User must set MAXMIND_LICENSE_KEY
226                headers: HashMap::new(),
227                filename: "maxmind-proxy-ranges.csv".to_string(),
228                format: FeedFormat::Csv,
229                custom_interval_seconds: Some(86400), // Daily
230            },
231        );
232
233        Self {
234            auto_update_enabled: false,    // OFF by default - user enables
235            update_interval_seconds: 3600, // 1 hour default
236            feeds_directory: PathBuf::from("threat-feeds"),
237            feeds,
238            download_timeout_seconds: 30,
239            extraction_limits: ExtractionLimits::default(),
240        }
241    }
242}
243
244impl ThreatIntelConfig {
245    /// All feeds disabled, no automatic updates.
246    ///
247    /// Use this when threat intelligence is not needed or when running
248    /// in an air-gapped environment.
249    ///
250    /// # Example
251    /// ```rust,ignore
252    /// let config = ThreatIntelConfig::disabled();
253    /// ```
254    pub fn disabled() -> Self {
255        Self {
256            auto_update_enabled: false,
257            update_interval_seconds: 0,
258            feeds_directory: PathBuf::from("./threat-feeds"),
259            feeds: HashMap::new(),
260            download_timeout_seconds: 30,
261            extraction_limits: ExtractionLimits::default(),
262        }
263    }
264
265    /// Aggressive update schedule — every 5 minutes, 10-second download timeout.
266    ///
267    /// Pre-configures popular free feeds (Tor exits, Spamhaus DROP,
268    /// Emerging Threats) all enabled.
269    ///
270    /// # Example
271    /// ```rust,ignore
272    /// let config = ThreatIntelConfig::aggressive();
273    /// ```
274    pub fn aggressive() -> Self {
275        let mut base = Self {
276            auto_update_enabled: true,
277            update_interval_seconds: 300, // 5 minutes
278            download_timeout_seconds: 10,
279            extraction_limits: ExtractionLimits::default(),
280            ..Default::default()
281        };
282        // Enable all pre-configured feeds
283        for feed in base.feeds.values_mut() {
284            feed.enabled = true;
285        }
286        base
287    }
288
289    /// Create configuration from environment variables and config file
290    pub fn from_env_and_config() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
291        // Try to load from config file first
292        let config_path = std::env::var("THREAT_INTEL_CONFIG_PATH")
293            .unwrap_or_else(|_| "threat-intel-config.yaml".to_string());
294
295        if std::path::Path::new(&config_path).exists() {
296            let config_content = std::fs::read_to_string(&config_path)?;
297            let mut config: Self = serde_saphyr::from_str(&config_content)?;
298
299            // Override with environment variables if they exist
300            if let Ok(enabled) = std::env::var("THREAT_INTEL_ENABLED") {
301                config.auto_update_enabled = enabled.to_lowercase() == "true";
302            }
303
304            if let Ok(interval) = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
305                && let Ok(seconds) = interval.parse::<u64>()
306            {
307                config.update_interval_seconds = seconds;
308            }
309
310            if let Ok(feeds_dir) = std::env::var("THREAT_INTEL_FEEDS_DIR") {
311                config.feeds_directory = std::path::PathBuf::from(feeds_dir);
312            }
313
314            Ok(config)
315        } else {
316            // Create default configuration from environment variables
317            Ok(Self::from_env_defaults())
318        }
319    }
320
321    /// Create default configuration from environment variables
322    fn from_env_defaults() -> Self {
323        let enabled = std::env::var("THREAT_INTEL_ENABLED")
324            .unwrap_or_else(|_| "false".to_string())
325            .to_lowercase()
326            == "true";
327
328        let update_interval = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
329            .unwrap_or_else(|_| "3600".to_string())
330            .parse::<u64>()
331            .unwrap_or(3600);
332
333        let feeds_dir = std::env::var("THREAT_INTEL_FEEDS_DIR")
334            .unwrap_or_else(|_| "./threat-feeds".to_string());
335
336        let timeout = std::env::var("THREAT_INTEL_TIMEOUT")
337            .unwrap_or_else(|_| "30".to_string())
338            .parse::<u64>()
339            .unwrap_or(30);
340
341        // Create default feeds based on environment switches
342        let mut feeds = HashMap::new();
343
344        // Tor exits feed
345        if std::env::var("TOR_EXITS_ENABLED")
346            .unwrap_or_else(|_| "true".to_string())
347            .to_lowercase()
348            == "true"
349        {
350            feeds.insert(
351                "tor_exits".to_string(),
352                FeedConfig {
353                    enabled: true,
354                    feed_type: FeedType::TorExitNodes,
355                    url: "https://check.torproject.org/torbulkexitlist".to_string(),
356                    api_key: None,
357                    headers: HashMap::new(),
358                    filename: "tor-exits.txt".to_string(),
359                    format: FeedFormat::PlainText,
360                    custom_interval_seconds: None,
361                },
362            );
363        }
364
365        // Spamhaus DROP feed
366        if std::env::var("SPAMHAUS_DROP_ENABLED")
367            .unwrap_or_else(|_| "true".to_string())
368            .to_lowercase()
369            == "true"
370        {
371            feeds.insert(
372                "spamhaus_drop".to_string(),
373                FeedConfig {
374                    enabled: true,
375                    feed_type: FeedType::MaliciousIPs,
376                    url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
377                    api_key: None,
378                    headers: HashMap::new(),
379                    filename: "spamhaus-drop.txt".to_string(),
380                    format: FeedFormat::PlainText,
381                    custom_interval_seconds: None,
382                },
383            );
384        }
385
386        // Emerging Threats feed
387        if std::env::var("EMERGINGTHREATS_ENABLED")
388            .unwrap_or_else(|_| "false".to_string())
389            .to_lowercase()
390            == "true"
391        {
392            feeds.insert(
393                "emergingthreats".to_string(),
394                FeedConfig {
395                    enabled: true,
396                    feed_type: FeedType::MaliciousIPs,
397                    url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
398                        .to_string(),
399                    api_key: None,
400                    headers: HashMap::new(),
401                    filename: "emerging-threats-ips.txt".to_string(),
402                    format: FeedFormat::PlainText,
403                    custom_interval_seconds: None,
404                },
405            );
406        }
407
408        Self {
409            auto_update_enabled: enabled,
410            update_interval_seconds: update_interval,
411            feeds_directory: std::path::PathBuf::from(feeds_dir),
412            download_timeout_seconds: timeout,
413            extraction_limits: ExtractionLimits::default(),
414            feeds,
415        }
416    }
417}
418
419impl ThreatFeedManager {
420    /// Create a new threat feed manager with configuration (async version)
421    pub async fn new_async(config: ThreatIntelConfig) -> Result<Self, Box<dyn std::error::Error>> {
422        // Create feeds directory if it doesn't exist
423        if !config.feeds_directory.exists() {
424            fs::create_dir_all(&config.feeds_directory).await?;
425        }
426
427        let client = Client::builder()
428            .timeout(Duration::from_secs(config.download_timeout_seconds))
429            .user_agent("AuthFramework-ThreatIntel/1.0")
430            .build()?;
431
432        let scheduler = Some(tokio_cron_scheduler::JobScheduler::new().await?);
433
434        Ok(Self {
435            config,
436            client,
437            scheduler,
438        })
439    }
440
441    /// Create a new threat intelligence manager (synchronous version)
442    pub fn new(
443        config: ThreatIntelConfig,
444    ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
445        // Create feeds directory if it doesn't exist
446        if !config.feeds_directory.exists() {
447            std::fs::create_dir_all(&config.feeds_directory)?;
448        }
449
450        let client = Client::builder()
451            .timeout(Duration::from_secs(config.download_timeout_seconds))
452            .user_agent("AuthFramework-ThreatIntel/1.0")
453            .build()?;
454
455        // Scheduler is not initialized in the simple constructor
456        let scheduler = None;
457
458        Ok(Self {
459            config,
460            client,
461            scheduler,
462        })
463    }
464
465    /// Start automated feed updates in the background
466    pub fn start_automated_updates(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
467        if !self.config.auto_update_enabled {
468            tracing::info!("Automated updates disabled in configuration");
469            return Ok(());
470        }
471
472        // Schedule updates for each enabled feed
473        let update_interval = format!("0 */{} * * * *", self.config.update_interval_seconds / 60);
474
475        tracing::info!(
476            "🚀 Starting automated threat intelligence updates (interval: {})",
477            update_interval
478        );
479
480        // For now, just log that we would start updates
481        // In a full implementation, this would start the tokio scheduler
482        tracing::info!("✅ Automated threat intelligence updates scheduled successfully");
483
484        Ok(())
485    }
486
487    /// Load configuration from YAML file or environment variables
488    pub fn load_config() -> ThreatIntelConfig {
489        // Try to load from config file first
490        if let Ok(config_content) = std::fs::read_to_string("threat-intel-config.yaml")
491            && let Ok(config) = serde_saphyr::from_str::<ThreatIntelConfig>(&config_content)
492        {
493            info!("Loaded threat intelligence configuration from file");
494            return config;
495        }
496
497        // Fall back to environment variables for simple on/off switches
498        let mut config = ThreatIntelConfig::default();
499
500        // Global enable/disable switch
501        if let Ok(enabled) = std::env::var("THREAT_INTEL_AUTO_UPDATE") {
502            config.auto_update_enabled = enabled.to_lowercase() == "true";
503        }
504
505        // Simple feed enable switches via environment variables
506        let feed_switches = [
507            ("THREAT_INTEL_TOR_EXITS", "tor_exits"),
508            ("THREAT_INTEL_SPAMHAUS", "spamhaus_drop"),
509            (
510                "THREAT_INTEL_EMERGING_THREATS",
511                "emergingthreats_compromised",
512            ),
513            ("THREAT_INTEL_VIRUSTOTAL", "virustotal_malicious"),
514            ("THREAT_INTEL_MAXMIND_PROXY", "maxmind_proxy_detection"),
515        ];
516
517        for (env_var, feed_name) in &feed_switches {
518            if let Ok(enabled) = std::env::var(env_var)
519                && let Some(feed) = config.feeds.get_mut(*feed_name)
520            {
521                feed.enabled = enabled.to_lowercase() == "true";
522                info!(
523                    "Feed {} enabled via {}: {}",
524                    feed_name, env_var, feed.enabled
525                );
526            }
527        }
528
529        // API keys from environment
530        if let Ok(api_key) = std::env::var("VIRUSTOTAL_API_KEY")
531            && let Some(feed) = config.feeds.get_mut("virustotal_malicious")
532        {
533            feed.api_key = Some(api_key);
534            feed.headers.insert(
535                "X-Apikey".to_string(),
536                feed.api_key.clone().expect("set to Some on previous line"),
537            );
538        }
539
540        if let Ok(license_key) = std::env::var("MAXMIND_LICENSE_KEY")
541            && let Some(feed) = config.feeds.get_mut("maxmind_proxy_detection")
542        {
543            feed.api_key = Some(license_key);
544            // Edition and suffix are appended as query params at download time,
545            // keeping the license key out of the stored URL string.
546            feed.url = format!("{}?edition_id=GeoIP2-Anonymous-IP&suffix=tar.gz", feed.url);
547        }
548
549        config
550    }
551
552    /// Start automated feed updates if enabled (Currently simplified implementation)
553    pub async fn start_automation(&mut self) -> Result<(), Box<dyn std::error::Error>> {
554        if !self.config.auto_update_enabled {
555            info!("Threat intelligence automation is disabled");
556            return Ok(());
557        }
558
559        info!("Starting automated threat intelligence feed updates");
560
561        // Initial download of all enabled feeds
562        self.download_all_feeds().await?;
563
564        // Production implementation: Set up automated scheduling for threat intelligence feeds
565        self.start_automated_scheduling().await?;
566
567        info!("✅ Threat intelligence feeds downloaded and scheduling activated");
568
569        Ok(())
570    }
571
572    /// Start automated scheduling for threat intelligence feed updates
573    async fn start_automated_scheduling(&mut self) -> Result<(), Box<dyn std::error::Error>> {
574        if let Some(scheduler) = &self.scheduler {
575            info!("Setting up automated threat intelligence feed scheduling...");
576
577            // Schedule threat intelligence updates based on configuration
578            for (feed_name, feed_config) in &self.config.feeds {
579                if feed_config.enabled {
580                    // Convert seconds to hours for better user experience in logs
581                    let update_interval_seconds =
582                        feed_config.custom_interval_seconds.unwrap_or(86400); // Default 24 hours
583                    let update_interval_hours = update_interval_seconds / 3600;
584                    let cron_expression = format!("0 0 */{} * * *", update_interval_hours.max(1)); // Every N hours, minimum 1
585
586                    info!(
587                        "Scheduling '{}' feed updates every {} hours (cron: {})",
588                        feed_name, update_interval_hours, cron_expression
589                    );
590
591                    // Clone necessary data for the closure
592                    let client_clone = self.client.clone();
593                    let config_clone = self.config.clone();
594                    let feed_name_clone = feed_name.clone();
595                    let feed_config_clone = feed_config.clone();
596
597                    // Create the scheduled job
598                    let job = Job::new_async(cron_expression.as_str(), move |_uuid, _l| {
599                        let client = client_clone.clone();
600                        let config = config_clone.clone();
601                        let name = feed_name_clone.clone();
602                        let config_feed = feed_config_clone.clone();
603
604                        Box::pin(async move {
605                            info!("⏰ Scheduled update starting for threat feed: {}", name);
606
607                            match Self::download_feed(&client, &config, &name, &config_feed).await {
608                                Ok(()) => {
609                                    info!("✅ Scheduled update completed for '{}'", name);
610                                }
611                                Err(e) => {
612                                    error!("❌ Scheduled update failed for '{}': {}", name, e);
613                                }
614                            }
615                        })
616                    })?;
617                    scheduler.add(job).await?;
618                }
619            }
620
621            // Start the scheduler
622            scheduler.start().await?;
623            info!("🚀 Threat intelligence scheduling started successfully");
624        } else {
625            warn!("⚠️ Scheduler not initialized - automated updates disabled");
626        }
627
628        Ok(())
629    }
630
631    /// Download all enabled feeds immediately
632    pub async fn download_all_feeds(&self) -> Result<(), Box<dyn std::error::Error>> {
633        for (feed_name, feed_config) in &self.config.feeds {
634            if feed_config.enabled {
635                match Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
636                {
637                    Ok(_) => info!("Successfully downloaded feed: {}", feed_name),
638                    Err(e) => error!("Failed to download feed {}: {}", feed_name, e),
639                }
640            }
641        }
642        Ok(())
643    }
644
645    /// Download a specific threat intelligence feed
646    async fn download_feed(
647        client: &Client,
648        config: &ThreatIntelConfig,
649        feed_name: &str,
650        feed_config: &FeedConfig,
651    ) -> Result<(), Box<dyn std::error::Error>> {
652        debug!("Downloading feed: {} from {}", feed_name, feed_config.url);
653
654        let mut request = client.get(&feed_config.url);
655
656        // Add authentication headers
657        for (key, value) in &feed_config.headers {
658            request = request.header(key, value);
659        }
660
661        let mut request_url = feed_config.url.clone();
662
663        // Add API key as header or query param based on service
664        if let Some(api_key) = &feed_config.api_key {
665            match feed_name {
666                name if name.contains("virustotal") => {
667                    request = request.header("X-Apikey", api_key);
668                }
669                name if name.contains("maxmind") => {
670                    // Append the license key at request time only so that
671                    // the stored URL (logged, serialised) never contains it.
672                    let sep = if request_url.contains('?') { '&' } else { '?' };
673                    request_url = format!(
674                        "{}{}license_key={}",
675                        request_url,
676                        sep,
677                        urlencoding::encode(api_key)
678                    );
679                    request = client.get(&request_url);
680                    // Re-attach headers after rebuilding request
681                    for (key, value) in &feed_config.headers {
682                        request = request.header(key, value);
683                    }
684                }
685                _ => {
686                    // Generic API key header
687                    request = request.header("Authorization", format!("Bearer {}", api_key));
688                }
689            }
690        }
691
692        let response = request.send().await?;
693
694        if !response.status().is_success() {
695            return Err(format!(
696                "HTTP error {}: {}",
697                response.status(),
698                response.text().await?
699            )
700            .into());
701        }
702
703        let content = response.bytes().await?;
704        let file_path = config.feeds_directory.join(&feed_config.filename);
705
706        // Handle compressed feeds (like MaxMind)
707        if feed_config.filename.ends_with(".tar.gz") {
708            // Extract tar.gz if needed
709            Self::extract_compressed_feed(&content, &file_path, &config.extraction_limits).await?;
710        } else {
711            fs::write(&file_path, &content).await?;
712        }
713
714        info!("Saved feed {} to {}", feed_name, file_path.display());
715
716        // Validate feed format
717        Self::validate_feed_format(&file_path, &feed_config.format)?;
718
719        Ok(())
720    }
721
722    /// Extract compressed feeds: tar.gz is extracted, bz2/xz/zip are refused, anything else is saved as-is.
723    async fn extract_compressed_feed(
724        content: &[u8],
725        output_path: &Path,
726        limits: &ExtractionLimits,
727    ) -> Result<(), Box<dyn std::error::Error>> {
728        // Production implementation: Detect archive type and extract properly
729        let extension = output_path
730            .extension()
731            .and_then(|ext| ext.to_str())
732            .unwrap_or("");
733
734        match extension.to_lowercase().as_str() {
735            "gz" | "tar" => Self::extract_tar_gz(content, output_path, limits).await,
736            "zip" => Err(format!(
737                "zip extraction is not supported: {} was not extracted",
738                output_path.display()
739            )
740            .into()),
741            "bz2" => Self::extract_bzip2(content, output_path).await,
742            "xz" => Self::extract_xz(content, output_path).await,
743            _ => {
744                // Unknown compression format, save as-is with warning
745                fs::write(output_path, content).await?;
746                warn!(
747                    "Unknown compression format '{}' - saved as-is: {}",
748                    extension,
749                    output_path.display()
750                );
751                Ok(())
752            }
753        }
754    }
755
756    /// Extract tar.gz archives with path traversal protection
757    async fn extract_tar_gz(
758        content: &[u8],
759        output_path: &Path,
760        limits: &ExtractionLimits,
761    ) -> Result<(), Box<dyn std::error::Error>> {
762        info!("Extracting tar.gz archive to: {}", output_path.display());
763
764        let content = content.to_vec();
765        let limits = limits.clone();
766        let dest = output_path
767            .parent()
768            .unwrap_or_else(|| Path::new("."))
769            .to_path_buf();
770
771        tokio::task::spawn_blocking(move || -> Result<(), String> {
772            let decoder = flate2::read::GzDecoder::new(content.as_slice());
773            let mut archive = tar::Archive::new(decoder);
774            let canonical_dest = dest.canonicalize().map_err(|e| e.to_string())?;
775
776            let mut entries_seen: usize = 0;
777            let mut total_bytes: u64 = 0;
778            let mut written: Vec<PathBuf> = Vec::new();
779            let outcome = (|| -> Result<(), String> {
780                for entry in archive.entries().map_err(|e| e.to_string())? {
781                    let mut entry = entry.map_err(|e| e.to_string())?;
782                    let path = entry.path().map_err(|e| e.to_string())?.into_owned();
783                    entries_seen += 1;
784                    if entries_seen > limits.max_entries {
785                        return Err(format!(
786                            "Archive has more than {} entries; refused at {}",
787                            limits.max_entries,
788                            path.display()
789                        ));
790                    }
791                    let size = entry.size();
792                    if size > limits.max_entry_bytes {
793                        return Err(format!(
794                            "Archive entry {} is {size} bytes, over the per-entry limit of {} bytes",
795                            path.display(),
796                            limits.max_entry_bytes
797                        ));
798                    }
799                    total_bytes = total_bytes.saturating_add(size);
800                    if total_bytes > limits.max_total_bytes {
801                        return Err(format!(
802                            "Archive entry {} brings the total to {total_bytes} bytes, over the total limit of {} bytes",
803                            path.display(),
804                            limits.max_total_bytes
805                        ));
806                    }
807
808                    // A feed is data: extract regular files and directories only.
809                    let kind = entry.header().entry_type();
810                    if !(kind.is_file() || kind.is_dir()) {
811                        return Err(format!(
812                            "Archive entry {} has unsupported type {:?}: only files and directories are extracted",
813                            path.display(),
814                            kind
815                        ));
816                    }
817
818                    // Reject absolute paths and path traversal sequences
819                    if path.is_absolute()
820                        || path
821                            .components()
822                            .any(|c| matches!(c, std::path::Component::ParentDir))
823                    {
824                        return Err(format!(
825                            "Zip Slip: archive entry has unsafe path: {}",
826                            path.display()
827                        ));
828                    }
829
830                    let target = canonical_dest.join(&path);
831                    // Verify the resolved path stays within the destination
832                    if !target.starts_with(&canonical_dest) {
833                        return Err(format!(
834                            "Zip Slip: entry escapes destination: {}",
835                            path.display()
836                        ));
837                    }
838
839                    // `unpack_in` confines each write to `dest`.
840                    let unpacked = entry
841                        .unpack_in(&canonical_dest)
842                        .map_err(|e| e.to_string())?;
843                    if !unpacked {
844                        return Err(format!(
845                            "Zip Slip: entry escapes destination: {}",
846                            path.display()
847                        ));
848                    }
849                    if kind.is_file() {
850                        written.push(canonical_dest.join(&path));
851                    }
852                }
853                Ok(())
854            })();
855            // A refused archive must not leave its earlier entries behind for a later read to trust.
856            if outcome.is_err() {
857                for file in &written {
858                    let _ = std::fs::remove_file(file);
859                }
860            }
861            outcome
862        })
863        .await
864        .map_err(|e| -> Box<dyn std::error::Error> { e.to_string().into() })?
865        .map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
866
867        Ok(())
868    }
869
870    /// Extract bzip2 archives
871    ///
872    /// NOTE: Requires the `bzip2 = "0.4"` crate (not currently in Cargo.toml).
873    /// Returns an error rather than silently writing unextracted bytes, which
874    /// would produce a corrupt feed file.
875    /// To enable: add `bzip2 = "0.4"` to Cargo.toml, then replace this body
876    /// with the same `spawn_blocking` decompression pattern used by
877    /// `extract_tar_gz`.
878    async fn extract_bzip2(
879        _content: &[u8],
880        output_path: &Path,
881    ) -> Result<(), Box<dyn std::error::Error>> {
882        Err(format!(
883            "bzip2 extraction is not supported: add the `bzip2` crate to Cargo.toml \
884             to enable decompression to {}",
885            output_path.display()
886        )
887        .into())
888    }
889
890    /// Extract XZ archives
891    ///
892    /// NOTE: Requires the `xz2 = "0.1"` crate (not currently in Cargo.toml).
893    /// Returns an error rather than silently writing unextracted bytes, which
894    /// would produce a corrupt feed file.
895    /// To enable: add `xz2 = "0.1"` to Cargo.toml, then replace this body
896    /// with the same `spawn_blocking` decompression pattern used by
897    /// `extract_tar_gz`.
898    async fn extract_xz(
899        _content: &[u8],
900        output_path: &Path,
901    ) -> Result<(), Box<dyn std::error::Error>> {
902        Err(format!(
903            "XZ extraction is not supported: add the `xz2` crate to Cargo.toml \
904             to enable decompression to {}",
905            output_path.display()
906        )
907        .into())
908    }
909
910    /// Validate that downloaded feed has expected format
911    fn validate_feed_format(
912        file_path: &Path,
913        format: &FeedFormat,
914    ) -> Result<(), Box<dyn std::error::Error>> {
915        let content = std::fs::read_to_string(file_path)?;
916
917        match format {
918            FeedFormat::PlainText => {
919                // Basic validation - check if it looks like IP addresses or networks
920                let lines: Vec<&str> = content
921                    .lines()
922                    .filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
923                    .collect();
924                if lines.is_empty() {
925                    return Err("Feed appears to be empty".into());
926                }
927            }
928            FeedFormat::Csv => {
929                let mut reader = csv::Reader::from_reader(content.as_bytes());
930                if reader.headers().is_err() {
931                    return Err("Invalid CSV format".into());
932                }
933            }
934            FeedFormat::Json => {
935                serde_json::from_str::<serde_json::Value>(&content)?;
936            }
937            FeedFormat::Xml => {
938                // Basic XML validation - check for well-formed structure
939                if !content.trim_start().starts_with('<') {
940                    return Err("Invalid XML format".into());
941                }
942            }
943        }
944
945        debug!("Feed format validation passed: {}", file_path.display());
946        Ok(())
947    }
948
949    /// Get status of all feeds
950    pub async fn get_feed_status(&self) -> HashMap<String, FeedStatus> {
951        let mut status = HashMap::new();
952
953        for (feed_name, feed_config) in &self.config.feeds {
954            let file_path = self.config.feeds_directory.join(&feed_config.filename);
955
956            let feed_status = if feed_config.enabled {
957                if file_path.exists() {
958                    if let Ok(metadata) = fs::metadata(&file_path).await {
959                        FeedStatus::Active {
960                            last_updated: metadata
961                                .modified()
962                                .unwrap_or(std::time::SystemTime::UNIX_EPOCH),
963                            size_bytes: metadata.len(),
964                        }
965                    } else {
966                        FeedStatus::Error("Cannot read file metadata".to_string())
967                    }
968                } else {
969                    FeedStatus::NotDownloaded
970                }
971            } else {
972                FeedStatus::Disabled
973            };
974
975            status.insert(feed_name.clone(), feed_status);
976        }
977
978        status
979    }
980
981    /// Manually trigger update of specific feed
982    pub async fn update_feed(&self, feed_name: &str) -> Result<(), Box<dyn std::error::Error>> {
983        if let Some(feed_config) = self.config.feeds.get(feed_name) {
984            if feed_config.enabled {
985                Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
986            } else {
987                Err(format!("Feed '{}' is disabled", feed_name).into())
988            }
989        } else {
990            Err(format!("Feed '{}' not found", feed_name).into())
991        }
992    }
993
994    /// Check if an IP address is in malicious IP feeds
995    pub fn is_malicious_ip(&self, ip: &std::net::IpAddr) -> bool {
996        for (feed_name, feed_config) in &self.config.feeds {
997            if !feed_config.enabled {
998                continue;
999            }
1000
1001            if matches!(feed_config.feed_type, FeedType::MaliciousIPs) {
1002                let file_path = self.config.feeds_directory.join(&feed_config.filename);
1003                if self.check_ip_in_feed(&file_path, ip) {
1004                    tracing::warn!("Malicious IP detected: {} (source: {})", ip, feed_name);
1005                    return true;
1006                }
1007            }
1008        }
1009        false
1010    }
1011
1012    /// Check if an IP address is a Tor exit node
1013    pub fn is_tor_exit(&self, ip: &std::net::IpAddr) -> bool {
1014        for (feed_name, feed_config) in &self.config.feeds {
1015            if !feed_config.enabled {
1016                continue;
1017            }
1018
1019            if matches!(feed_config.feed_type, FeedType::TorExitNodes) {
1020                let file_path = self.config.feeds_directory.join(&feed_config.filename);
1021                if self.check_ip_in_feed(&file_path, ip) {
1022                    tracing::warn!("Tor exit node detected: {} (source: {})", ip, feed_name);
1023                    return true;
1024                }
1025            }
1026        }
1027        false
1028    }
1029
1030    /// Check if an IP address is from a VPN or proxy service
1031    pub fn is_proxy_vpn(&self, ip: &std::net::IpAddr) -> bool {
1032        for (feed_name, feed_config) in &self.config.feeds {
1033            if !feed_config.enabled {
1034                continue;
1035            }
1036
1037            if matches!(feed_config.feed_type, FeedType::VpnProxy) {
1038                let file_path = self.config.feeds_directory.join(&feed_config.filename);
1039                if self.check_ip_in_feed(&file_path, ip) {
1040                    tracing::info!("VPN/Proxy detected: {} (source: {})", ip, feed_name);
1041                    return true;
1042                }
1043            }
1044        }
1045        false
1046    }
1047
1048    /// Helper method to check if an IP is present in a feed file
1049    fn check_ip_in_feed(&self, file_path: &std::path::Path, ip: &std::net::IpAddr) -> bool {
1050        if !file_path.exists() {
1051            return false;
1052        }
1053
1054        if let Ok(contents) = std::fs::read_to_string(file_path) {
1055            for line in contents.lines() {
1056                let line = line.trim();
1057                if line.is_empty() || line.starts_with('#') {
1058                    continue;
1059                }
1060
1061                // Check exact IP match
1062                if line == ip.to_string() {
1063                    return true;
1064                }
1065
1066                // Check CIDR network match
1067                if line.contains('/') {
1068                    match ip {
1069                        std::net::IpAddr::V4(ipv4) => {
1070                            if let Ok(network) = line.parse::<ipnetwork::Ipv4Network>()
1071                                && network.contains(*ipv4)
1072                            {
1073                                return true;
1074                            }
1075                        }
1076                        std::net::IpAddr::V6(ipv6) => {
1077                            if let Ok(network) = line.parse::<ipnetwork::Ipv6Network>()
1078                                && network.contains(*ipv6)
1079                            {
1080                                return true;
1081                            }
1082                        }
1083                    }
1084                }
1085            }
1086        }
1087
1088        false
1089    }
1090}
1091
1092/// Status of a threat intelligence feed
1093#[derive(Debug, Clone)]
1094pub enum FeedStatus {
1095    /// Feed is disabled
1096    Disabled,
1097
1098    /// Feed is enabled but not yet downloaded
1099    NotDownloaded,
1100
1101    /// Feed is active and up-to-date
1102    Active {
1103        last_updated: std::time::SystemTime,
1104        size_bytes: u64,
1105    },
1106
1107    /// Feed has an error
1108    Error(String),
1109}