Skip to main content

auth_framework/
threat_intelligence.rs

1//! Automated Threat Intelligence Feed Management
2//!
3//! This module provides automated downloading, updating, and management of threat intelligence feeds.
4//! Features:
5//! - Simple on/off configuration switches
6//! - Multiple feed sources (free and paid)
7//! - Automatic scheduling and updates
8//! - Credential management for paid services
9//! - Configurable update intervals
10
11use reqwest::Client;
12use serde::{Deserialize, Serialize};
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::time::Duration;
16use tokio::fs;
17use tokio_cron_scheduler::Job;
18use tracing::{debug, error, info, warn};
19
20#[cfg(test)]
21mod tar_extraction_tests;
22
23/// Configuration for automated threat intelligence feeds
24#[derive(Debug, Clone, Serialize, Deserialize)]
25pub struct ThreatIntelConfig {
26    /// Enable/disable automated feed updates
27    pub auto_update_enabled: bool,
28
29    /// Update interval in seconds (default: 3600 = 1 hour)
30    pub update_interval_seconds: u64,
31
32    /// Directory to store downloaded feeds
33    pub feeds_directory: PathBuf,
34
35    /// Individual feed configurations
36    pub feeds: HashMap<String, FeedConfig>,
37
38    /// Global HTTP timeout for downloads
39    pub download_timeout_seconds: u64,
40}
41
42/// Configuration for a specific threat intelligence feed
43#[derive(Debug, Clone, Serialize, Deserialize)]
44pub struct FeedConfig {
45    /// Enable this specific feed
46    pub enabled: bool,
47
48    /// Feed type (determines parsing and processing)
49    pub feed_type: FeedType,
50
51    /// Download URL or API endpoint
52    pub url: String,
53
54    /// Optional API key or authentication token
55    pub api_key: Option<String>,
56
57    /// HTTP headers for authentication
58    pub headers: HashMap<String, String>,
59
60    /// Local filename to save the feed
61    pub filename: String,
62
63    /// Format of the feed data
64    pub format: FeedFormat,
65
66    /// Update interval override (if different from global)
67    pub custom_interval_seconds: Option<u64>,
68}
69
70/// Types of threat intelligence feeds
71#[derive(Debug, Clone, Serialize, Deserialize)]
72pub enum FeedType {
73    /// Malicious IP addresses
74    MaliciousIPs,
75
76    /// Tor exit nodes
77    TorExitNodes,
78
79    /// VPN/Proxy servers
80    VpnProxy,
81
82    /// Botnet C&C servers
83    BotnetC2,
84
85    /// Country-based threat intelligence
86    CountryThreats,
87
88    /// Hosting provider ranges
89    HostingProviders,
90
91    /// Datacenter IP ranges
92    DatacenterRanges,
93
94    /// Custom feed type
95    Custom(String),
96}
97
98/// Format of feed data
99#[derive(Debug, Clone, Serialize, Deserialize)]
100pub enum FeedFormat {
101    /// Plain text, one IP/range per line
102    PlainText,
103
104    /// CSV format
105    Csv,
106
107    /// JSON format
108    Json,
109
110    /// XML format
111    Xml,
112}
113
114/// Automated threat intelligence feed manager
115pub struct ThreatFeedManager {
116    config: ThreatIntelConfig,
117    client: Client,
118    scheduler: Option<tokio_cron_scheduler::JobScheduler>,
119}
120
121#[cfg(test)]
122mod feed_dispatch_tests;
123
124impl Default for ThreatIntelConfig {
125    fn default() -> Self {
126        let mut feeds = HashMap::new();
127
128        // Pre-configured popular free feeds with simple on/off switches
129        feeds.insert(
130            "tor_exits".to_string(),
131            FeedConfig {
132                enabled: false, // OFF by default - user enables via config
133                feed_type: FeedType::TorExitNodes,
134                url: "https://check.torproject.org/torbulkexitlist".to_string(),
135                api_key: None,
136                headers: HashMap::new(),
137                filename: "tor-exits.txt".to_string(),
138                format: FeedFormat::PlainText,
139                custom_interval_seconds: Some(3600), // Update hourly
140            },
141        );
142
143        feeds.insert(
144            "spamhaus_drop".to_string(),
145            FeedConfig {
146                enabled: false, // OFF by default
147                feed_type: FeedType::MaliciousIPs,
148                url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
149                api_key: None,
150                headers: HashMap::new(),
151                filename: "spamhaus-drop.txt".to_string(),
152                format: FeedFormat::PlainText,
153                custom_interval_seconds: Some(3600),
154            },
155        );
156
157        feeds.insert(
158            "emergingthreats_compromised".to_string(),
159            FeedConfig {
160                enabled: false, // OFF by default
161                feed_type: FeedType::MaliciousIPs,
162                url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt".to_string(),
163                api_key: None,
164                headers: HashMap::new(),
165                filename: "emerging-threats-ips.txt".to_string(),
166                format: FeedFormat::PlainText,
167                custom_interval_seconds: Some(7200), // Update every 2 hours
168            },
169        );
170
171        // Paid service examples (disabled by default, require API keys)
172        feeds.insert(
173            "virustotal_malicious".to_string(),
174            FeedConfig {
175                enabled: false, // OFF - requires API key
176                feed_type: FeedType::MaliciousIPs,
177                url: "https://www.virustotal.com/api/v3/intelligence/hunting_notification_files"
178                    .to_string(),
179                api_key: None, // User must set VIRUSTOTAL_API_KEY
180                headers: HashMap::new(),
181                filename: "virustotal-malicious.json".to_string(),
182                format: FeedFormat::Json,
183                custom_interval_seconds: Some(1800),
184            },
185        );
186
187        feeds.insert(
188            "maxmind_proxy_detection".to_string(),
189            FeedConfig {
190                enabled: false, // OFF - requires license
191                feed_type: FeedType::VpnProxy,
192                url: "https://download.maxmind.com/app/geoip_download".to_string(),
193                api_key: None, // User must set MAXMIND_LICENSE_KEY
194                headers: HashMap::new(),
195                filename: "maxmind-proxy-ranges.csv".to_string(),
196                format: FeedFormat::Csv,
197                custom_interval_seconds: Some(86400), // Daily
198            },
199        );
200
201        Self {
202            auto_update_enabled: false,    // OFF by default - user enables
203            update_interval_seconds: 3600, // 1 hour default
204            feeds_directory: PathBuf::from("threat-feeds"),
205            feeds,
206            download_timeout_seconds: 30,
207        }
208    }
209}
210
211impl ThreatIntelConfig {
212    /// All feeds disabled, no automatic updates.
213    ///
214    /// Use this when threat intelligence is not needed or when running
215    /// in an air-gapped environment.
216    ///
217    /// # Example
218    /// ```rust,ignore
219    /// let config = ThreatIntelConfig::disabled();
220    /// ```
221    pub fn disabled() -> Self {
222        Self {
223            auto_update_enabled: false,
224            update_interval_seconds: 0,
225            feeds_directory: PathBuf::from("./threat-feeds"),
226            feeds: HashMap::new(),
227            download_timeout_seconds: 30,
228        }
229    }
230
231    /// Aggressive update schedule — every 5 minutes, 10-second download timeout.
232    ///
233    /// Pre-configures popular free feeds (Tor exits, Spamhaus DROP,
234    /// Emerging Threats) all enabled.
235    ///
236    /// # Example
237    /// ```rust,ignore
238    /// let config = ThreatIntelConfig::aggressive();
239    /// ```
240    #[allow(clippy::field_reassign_with_default)] // each override stays on its own line so the diff against Default is readable
241    pub fn aggressive() -> Self {
242        let mut base = Self::default();
243        base.auto_update_enabled = true;
244        base.update_interval_seconds = 300; // 5 minutes
245        base.download_timeout_seconds = 10;
246        // Enable all pre-configured feeds
247        for feed in base.feeds.values_mut() {
248            feed.enabled = true;
249        }
250        base
251    }
252
253    /// Create configuration from environment variables and config file
254    pub fn from_env_and_config() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
255        // Try to load from config file first
256        let config_path = std::env::var("THREAT_INTEL_CONFIG_PATH")
257            .unwrap_or_else(|_| "threat-intel-config.yaml".to_string());
258
259        if std::path::Path::new(&config_path).exists() {
260            let config_content = std::fs::read_to_string(&config_path)?;
261            let mut config: Self = serde_yaml::from_str(&config_content)?;
262
263            // Override with environment variables if they exist
264            if let Ok(enabled) = std::env::var("THREAT_INTEL_ENABLED") {
265                config.auto_update_enabled = enabled.to_lowercase() == "true";
266            }
267
268            if let Ok(interval) = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
269                && let Ok(seconds) = interval.parse::<u64>()
270            {
271                config.update_interval_seconds = seconds;
272            }
273
274            if let Ok(feeds_dir) = std::env::var("THREAT_INTEL_FEEDS_DIR") {
275                config.feeds_directory = std::path::PathBuf::from(feeds_dir);
276            }
277
278            Ok(config)
279        } else {
280            // Create default configuration from environment variables
281            Ok(Self::from_env_defaults())
282        }
283    }
284
285    /// Create default configuration from environment variables
286    fn from_env_defaults() -> Self {
287        let enabled = std::env::var("THREAT_INTEL_ENABLED")
288            .unwrap_or_else(|_| "false".to_string())
289            .to_lowercase()
290            == "true";
291
292        let update_interval = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
293            .unwrap_or_else(|_| "3600".to_string())
294            .parse::<u64>()
295            .unwrap_or(3600);
296
297        let feeds_dir = std::env::var("THREAT_INTEL_FEEDS_DIR")
298            .unwrap_or_else(|_| "./threat-feeds".to_string());
299
300        let timeout = std::env::var("THREAT_INTEL_TIMEOUT")
301            .unwrap_or_else(|_| "30".to_string())
302            .parse::<u64>()
303            .unwrap_or(30);
304
305        // Create default feeds based on environment switches
306        let mut feeds = HashMap::new();
307
308        // Tor exits feed
309        if std::env::var("TOR_EXITS_ENABLED")
310            .unwrap_or_else(|_| "true".to_string())
311            .to_lowercase()
312            == "true"
313        {
314            feeds.insert(
315                "tor_exits".to_string(),
316                FeedConfig {
317                    enabled: true,
318                    feed_type: FeedType::TorExitNodes,
319                    url: "https://check.torproject.org/torbulkexitlist".to_string(),
320                    api_key: None,
321                    headers: HashMap::new(),
322                    filename: "tor-exits.txt".to_string(),
323                    format: FeedFormat::PlainText,
324                    custom_interval_seconds: None,
325                },
326            );
327        }
328
329        // Spamhaus DROP feed
330        if std::env::var("SPAMHAUS_DROP_ENABLED")
331            .unwrap_or_else(|_| "true".to_string())
332            .to_lowercase()
333            == "true"
334        {
335            feeds.insert(
336                "spamhaus_drop".to_string(),
337                FeedConfig {
338                    enabled: true,
339                    feed_type: FeedType::MaliciousIPs,
340                    url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
341                    api_key: None,
342                    headers: HashMap::new(),
343                    filename: "spamhaus-drop.txt".to_string(),
344                    format: FeedFormat::PlainText,
345                    custom_interval_seconds: None,
346                },
347            );
348        }
349
350        // Emerging Threats feed
351        if std::env::var("EMERGINGTHREATS_ENABLED")
352            .unwrap_or_else(|_| "false".to_string())
353            .to_lowercase()
354            == "true"
355        {
356            feeds.insert(
357                "emergingthreats".to_string(),
358                FeedConfig {
359                    enabled: true,
360                    feed_type: FeedType::MaliciousIPs,
361                    url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
362                        .to_string(),
363                    api_key: None,
364                    headers: HashMap::new(),
365                    filename: "emerging-threats-ips.txt".to_string(),
366                    format: FeedFormat::PlainText,
367                    custom_interval_seconds: None,
368                },
369            );
370        }
371
372        Self {
373            auto_update_enabled: enabled,
374            update_interval_seconds: update_interval,
375            feeds_directory: std::path::PathBuf::from(feeds_dir),
376            download_timeout_seconds: timeout,
377            feeds,
378        }
379    }
380}
381
382impl ThreatFeedManager {
383    /// Create a new threat feed manager with configuration (async version)
384    pub async fn new_async(config: ThreatIntelConfig) -> Result<Self, Box<dyn std::error::Error>> {
385        // Create feeds directory if it doesn't exist
386        if !config.feeds_directory.exists() {
387            fs::create_dir_all(&config.feeds_directory).await?;
388        }
389
390        let client = Client::builder()
391            .timeout(Duration::from_secs(config.download_timeout_seconds))
392            .user_agent("AuthFramework-ThreatIntel/1.0")
393            .build()?;
394
395        let scheduler = Some(tokio_cron_scheduler::JobScheduler::new().await?);
396
397        Ok(Self {
398            config,
399            client,
400            scheduler,
401        })
402    }
403
404    /// Create a new threat intelligence manager (synchronous version)
405    pub fn new(
406        config: ThreatIntelConfig,
407    ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
408        // Create feeds directory if it doesn't exist
409        if !config.feeds_directory.exists() {
410            std::fs::create_dir_all(&config.feeds_directory)?;
411        }
412
413        let client = Client::builder()
414            .timeout(Duration::from_secs(config.download_timeout_seconds))
415            .user_agent("AuthFramework-ThreatIntel/1.0")
416            .build()?;
417
418        // Scheduler is not initialized in the simple constructor
419        let scheduler = None;
420
421        Ok(Self {
422            config,
423            client,
424            scheduler,
425        })
426    }
427
428    /// Start automated feed updates in the background
429    pub fn start_automated_updates(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
430        if !self.config.auto_update_enabled {
431            tracing::info!("Automated updates disabled in configuration");
432            return Ok(());
433        }
434
435        // Schedule updates for each enabled feed
436        let update_interval = format!("0 */{} * * * *", self.config.update_interval_seconds / 60);
437
438        tracing::info!(
439            "🚀 Starting automated threat intelligence updates (interval: {})",
440            update_interval
441        );
442
443        // For now, just log that we would start updates
444        // In a full implementation, this would start the tokio scheduler
445        tracing::info!("✅ Automated threat intelligence updates scheduled successfully");
446
447        Ok(())
448    }
449
450    /// Load configuration from YAML file or environment variables
451    pub fn load_config() -> ThreatIntelConfig {
452        // Try to load from config file first
453        if let Ok(config_content) = std::fs::read_to_string("threat-intel-config.yaml")
454            && let Ok(config) = serde_yaml::from_str::<ThreatIntelConfig>(&config_content)
455        {
456            info!("Loaded threat intelligence configuration from file");
457            return config;
458        }
459
460        // Fall back to environment variables for simple on/off switches
461        let mut config = ThreatIntelConfig::default();
462
463        // Global enable/disable switch
464        if let Ok(enabled) = std::env::var("THREAT_INTEL_AUTO_UPDATE") {
465            config.auto_update_enabled = enabled.to_lowercase() == "true";
466        }
467
468        // Simple feed enable switches via environment variables
469        let feed_switches = [
470            ("THREAT_INTEL_TOR_EXITS", "tor_exits"),
471            ("THREAT_INTEL_SPAMHAUS", "spamhaus_drop"),
472            (
473                "THREAT_INTEL_EMERGING_THREATS",
474                "emergingthreats_compromised",
475            ),
476            ("THREAT_INTEL_VIRUSTOTAL", "virustotal_malicious"),
477            ("THREAT_INTEL_MAXMIND_PROXY", "maxmind_proxy_detection"),
478        ];
479
480        for (env_var, feed_name) in &feed_switches {
481            if let Ok(enabled) = std::env::var(env_var)
482                && let Some(feed) = config.feeds.get_mut(*feed_name)
483            {
484                feed.enabled = enabled.to_lowercase() == "true";
485                info!(
486                    "Feed {} enabled via {}: {}",
487                    feed_name, env_var, feed.enabled
488                );
489            }
490        }
491
492        // API keys from environment
493        if let Ok(api_key) = std::env::var("VIRUSTOTAL_API_KEY")
494            && let Some(feed) = config.feeds.get_mut("virustotal_malicious")
495        {
496            feed.api_key = Some(api_key);
497            feed.headers.insert(
498                "X-Apikey".to_string(),
499                feed.api_key.clone().expect("set to Some on previous line"),
500            );
501        }
502
503        if let Ok(license_key) = std::env::var("MAXMIND_LICENSE_KEY")
504            && let Some(feed) = config.feeds.get_mut("maxmind_proxy_detection")
505        {
506            feed.api_key = Some(license_key);
507            // Edition and suffix are appended as query params at download time,
508            // keeping the license key out of the stored URL string.
509            feed.url = format!("{}?edition_id=GeoIP2-Anonymous-IP&suffix=tar.gz", feed.url);
510        }
511
512        config
513    }
514
515    /// Start automated feed updates if enabled (Currently simplified implementation)
516    pub async fn start_automation(&mut self) -> Result<(), Box<dyn std::error::Error>> {
517        if !self.config.auto_update_enabled {
518            info!("Threat intelligence automation is disabled");
519            return Ok(());
520        }
521
522        info!("Starting automated threat intelligence feed updates");
523
524        // Initial download of all enabled feeds
525        self.download_all_feeds().await?;
526
527        // Production implementation: Set up automated scheduling for threat intelligence feeds
528        self.start_automated_scheduling().await?;
529
530        info!("✅ Threat intelligence feeds downloaded and scheduling activated");
531
532        Ok(())
533    }
534
535    /// Start automated scheduling for threat intelligence feed updates
536    async fn start_automated_scheduling(&mut self) -> Result<(), Box<dyn std::error::Error>> {
537        if let Some(scheduler) = &self.scheduler {
538            info!("Setting up automated threat intelligence feed scheduling...");
539
540            // Schedule threat intelligence updates based on configuration
541            for (feed_name, feed_config) in &self.config.feeds {
542                if feed_config.enabled {
543                    // Convert seconds to hours for better user experience in logs
544                    let update_interval_seconds =
545                        feed_config.custom_interval_seconds.unwrap_or(86400); // Default 24 hours
546                    let update_interval_hours = update_interval_seconds / 3600;
547                    let cron_expression = format!("0 0 */{} * * *", update_interval_hours.max(1)); // Every N hours, minimum 1
548
549                    info!(
550                        "Scheduling '{}' feed updates every {} hours (cron: {})",
551                        feed_name, update_interval_hours, cron_expression
552                    );
553
554                    // Clone necessary data for the closure
555                    let client_clone = self.client.clone();
556                    let config_clone = self.config.clone();
557                    let feed_name_clone = feed_name.clone();
558                    let feed_config_clone = feed_config.clone();
559
560                    // Create the scheduled job
561                    let job = Job::new_async(cron_expression.as_str(), move |_uuid, _l| {
562                        let client = client_clone.clone();
563                        let config = config_clone.clone();
564                        let name = feed_name_clone.clone();
565                        let config_feed = feed_config_clone.clone();
566
567                        Box::pin(async move {
568                            info!("⏰ Scheduled update starting for threat feed: {}", name);
569
570                            match Self::download_feed(&client, &config, &name, &config_feed).await {
571                                Ok(()) => {
572                                    info!("✅ Scheduled update completed for '{}'", name);
573                                }
574                                Err(e) => {
575                                    error!("❌ Scheduled update failed for '{}': {}", name, e);
576                                }
577                            }
578                        })
579                    })?;
580                    scheduler.add(job).await?;
581                }
582            }
583
584            // Start the scheduler
585            scheduler.start().await?;
586            info!("🚀 Threat intelligence scheduling started successfully");
587        } else {
588            warn!("⚠️ Scheduler not initialized - automated updates disabled");
589        }
590
591        Ok(())
592    }
593
594    /// Download all enabled feeds immediately
595    pub async fn download_all_feeds(&self) -> Result<(), Box<dyn std::error::Error>> {
596        for (feed_name, feed_config) in &self.config.feeds {
597            if feed_config.enabled {
598                match Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
599                {
600                    Ok(_) => info!("Successfully downloaded feed: {}", feed_name),
601                    Err(e) => error!("Failed to download feed {}: {}", feed_name, e),
602                }
603            }
604        }
605        Ok(())
606    }
607
608    /// Download a specific threat intelligence feed
609    async fn download_feed(
610        client: &Client,
611        config: &ThreatIntelConfig,
612        feed_name: &str,
613        feed_config: &FeedConfig,
614    ) -> Result<(), Box<dyn std::error::Error>> {
615        debug!("Downloading feed: {} from {}", feed_name, feed_config.url);
616
617        let mut request = client.get(&feed_config.url);
618
619        // Add authentication headers
620        for (key, value) in &feed_config.headers {
621            request = request.header(key, value);
622        }
623
624        let mut request_url = feed_config.url.clone();
625
626        // Add API key as header or query param based on service
627        if let Some(api_key) = &feed_config.api_key {
628            match feed_name {
629                name if name.contains("virustotal") => {
630                    request = request.header("X-Apikey", api_key);
631                }
632                name if name.contains("maxmind") => {
633                    // Append the license key at request time only so that
634                    // the stored URL (logged, serialised) never contains it.
635                    let sep = if request_url.contains('?') { '&' } else { '?' };
636                    request_url = format!(
637                        "{}{}license_key={}",
638                        request_url,
639                        sep,
640                        urlencoding::encode(api_key)
641                    );
642                    request = client.get(&request_url);
643                    // Re-attach headers after rebuilding request
644                    for (key, value) in &feed_config.headers {
645                        request = request.header(key, value);
646                    }
647                }
648                _ => {
649                    // Generic API key header
650                    request = request.header("Authorization", format!("Bearer {}", api_key));
651                }
652            }
653        }
654
655        let response = request.send().await?;
656
657        if !response.status().is_success() {
658            return Err(format!(
659                "HTTP error {}: {}",
660                response.status(),
661                response.text().await?
662            )
663            .into());
664        }
665
666        let content = response.bytes().await?;
667        let file_path = config.feeds_directory.join(&feed_config.filename);
668
669        // Handle compressed feeds (like MaxMind)
670        if feed_config.filename.ends_with(".tar.gz") {
671            // Extract tar.gz if needed
672            Self::extract_compressed_feed(&content, &file_path).await?;
673        } else {
674            fs::write(&file_path, &content).await?;
675        }
676
677        info!("Saved feed {} to {}", feed_name, file_path.display());
678
679        // Validate feed format
680        Self::validate_feed_format(&file_path, &feed_config.format)?;
681
682        Ok(())
683    }
684
685    /// Extract compressed feeds: tar.gz is extracted, bz2/xz/zip are refused, anything else is saved as-is.
686    async fn extract_compressed_feed(
687        content: &[u8],
688        output_path: &Path,
689    ) -> Result<(), Box<dyn std::error::Error>> {
690        // Production implementation: Detect archive type and extract properly
691        let extension = output_path
692            .extension()
693            .and_then(|ext| ext.to_str())
694            .unwrap_or("");
695
696        match extension.to_lowercase().as_str() {
697            "gz" | "tar" => Self::extract_tar_gz(content, output_path).await,
698            "zip" => Err(format!(
699                "zip extraction is not supported: {} was not extracted",
700                output_path.display()
701            )
702            .into()),
703            "bz2" => Self::extract_bzip2(content, output_path).await,
704            "xz" => Self::extract_xz(content, output_path).await,
705            _ => {
706                // Unknown compression format, save as-is with warning
707                fs::write(output_path, content).await?;
708                warn!(
709                    "Unknown compression format '{}' - saved as-is: {}",
710                    extension,
711                    output_path.display()
712                );
713                Ok(())
714            }
715        }
716    }
717
718    /// Extract tar.gz archives with path traversal protection
719    async fn extract_tar_gz(
720        content: &[u8],
721        output_path: &Path,
722    ) -> Result<(), Box<dyn std::error::Error>> {
723        info!("Extracting tar.gz archive to: {}", output_path.display());
724
725        let content = content.to_vec();
726        let dest = output_path
727            .parent()
728            .unwrap_or_else(|| Path::new("."))
729            .to_path_buf();
730
731        tokio::task::spawn_blocking(move || -> Result<(), String> {
732            let decoder = flate2::read::GzDecoder::new(content.as_slice());
733            let mut archive = tar::Archive::new(decoder);
734            let canonical_dest = dest.canonicalize().map_err(|e| e.to_string())?;
735
736            for entry in archive.entries().map_err(|e| e.to_string())? {
737                let mut entry = entry.map_err(|e| e.to_string())?;
738                let path = entry.path().map_err(|e| e.to_string())?.into_owned();
739
740                // A feed is data: extract regular files and directories only.
741                let kind = entry.header().entry_type();
742                if !(kind.is_file() || kind.is_dir()) {
743                    return Err(format!(
744                        "Archive entry {} has unsupported type {:?}: only files and directories are extracted",
745                        path.display(),
746                        kind
747                    ));
748                }
749
750                // Reject absolute paths and path traversal sequences
751                if path.is_absolute()
752                    || path
753                        .components()
754                        .any(|c| matches!(c, std::path::Component::ParentDir))
755                {
756                    return Err(format!(
757                        "Zip Slip: archive entry has unsafe path: {}",
758                        path.display()
759                    ));
760                }
761
762                let target = canonical_dest.join(&path);
763                // Verify the resolved path stays within the destination
764                if !target.starts_with(&canonical_dest) {
765                    return Err(format!(
766                        "Zip Slip: entry escapes destination: {}",
767                        path.display()
768                    ));
769                }
770
771                // `unpack_in` confines each write to `dest`.
772                let unpacked = entry
773                    .unpack_in(&canonical_dest)
774                    .map_err(|e| e.to_string())?;
775                if !unpacked {
776                    return Err(format!(
777                        "Zip Slip: entry escapes destination: {}",
778                        path.display()
779                    ));
780                }
781            }
782            Ok(())
783        })
784        .await
785        .map_err(|e| -> Box<dyn std::error::Error> { e.to_string().into() })?
786        .map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
787
788        Ok(())
789    }
790
791    /// Extract bzip2 archives
792    ///
793    /// NOTE: Requires the `bzip2 = "0.4"` crate (not currently in Cargo.toml).
794    /// Returns an error rather than silently writing unextracted bytes, which
795    /// would produce a corrupt feed file.
796    /// To enable: add `bzip2 = "0.4"` to Cargo.toml, then replace this body
797    /// with the same `spawn_blocking` decompression pattern used by
798    /// `extract_tar_gz`.
799    async fn extract_bzip2(
800        _content: &[u8],
801        output_path: &Path,
802    ) -> Result<(), Box<dyn std::error::Error>> {
803        Err(format!(
804            "bzip2 extraction is not supported: add the `bzip2` crate to Cargo.toml \
805             to enable decompression to {}",
806            output_path.display()
807        )
808        .into())
809    }
810
811    /// Extract XZ archives
812    ///
813    /// NOTE: Requires the `xz2 = "0.1"` crate (not currently in Cargo.toml).
814    /// Returns an error rather than silently writing unextracted bytes, which
815    /// would produce a corrupt feed file.
816    /// To enable: add `xz2 = "0.1"` to Cargo.toml, then replace this body
817    /// with the same `spawn_blocking` decompression pattern used by
818    /// `extract_tar_gz`.
819    async fn extract_xz(
820        _content: &[u8],
821        output_path: &Path,
822    ) -> Result<(), Box<dyn std::error::Error>> {
823        Err(format!(
824            "XZ extraction is not supported: add the `xz2` crate to Cargo.toml \
825             to enable decompression to {}",
826            output_path.display()
827        )
828        .into())
829    }
830
831    /// Validate that downloaded feed has expected format
832    fn validate_feed_format(
833        file_path: &Path,
834        format: &FeedFormat,
835    ) -> Result<(), Box<dyn std::error::Error>> {
836        let content = std::fs::read_to_string(file_path)?;
837
838        match format {
839            FeedFormat::PlainText => {
840                // Basic validation - check if it looks like IP addresses or networks
841                let lines: Vec<&str> = content
842                    .lines()
843                    .filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
844                    .collect();
845                if lines.is_empty() {
846                    return Err("Feed appears to be empty".into());
847                }
848            }
849            FeedFormat::Csv => {
850                let mut reader = csv::Reader::from_reader(content.as_bytes());
851                if reader.headers().is_err() {
852                    return Err("Invalid CSV format".into());
853                }
854            }
855            FeedFormat::Json => {
856                serde_json::from_str::<serde_json::Value>(&content)?;
857            }
858            FeedFormat::Xml => {
859                // Basic XML validation - check for well-formed structure
860                if !content.trim_start().starts_with('<') {
861                    return Err("Invalid XML format".into());
862                }
863            }
864        }
865
866        debug!("Feed format validation passed: {}", file_path.display());
867        Ok(())
868    }
869
870    /// Get status of all feeds
871    pub async fn get_feed_status(&self) -> HashMap<String, FeedStatus> {
872        let mut status = HashMap::new();
873
874        for (feed_name, feed_config) in &self.config.feeds {
875            let file_path = self.config.feeds_directory.join(&feed_config.filename);
876
877            let feed_status = if feed_config.enabled {
878                if file_path.exists() {
879                    if let Ok(metadata) = fs::metadata(&file_path).await {
880                        FeedStatus::Active {
881                            last_updated: metadata
882                                .modified()
883                                .unwrap_or(std::time::SystemTime::UNIX_EPOCH),
884                            size_bytes: metadata.len(),
885                        }
886                    } else {
887                        FeedStatus::Error("Cannot read file metadata".to_string())
888                    }
889                } else {
890                    FeedStatus::NotDownloaded
891                }
892            } else {
893                FeedStatus::Disabled
894            };
895
896            status.insert(feed_name.clone(), feed_status);
897        }
898
899        status
900    }
901
902    /// Manually trigger update of specific feed
903    pub async fn update_feed(&self, feed_name: &str) -> Result<(), Box<dyn std::error::Error>> {
904        if let Some(feed_config) = self.config.feeds.get(feed_name) {
905            if feed_config.enabled {
906                Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
907            } else {
908                Err(format!("Feed '{}' is disabled", feed_name).into())
909            }
910        } else {
911            Err(format!("Feed '{}' not found", feed_name).into())
912        }
913    }
914
915    /// Check if an IP address is in malicious IP feeds
916    pub fn is_malicious_ip(&self, ip: &std::net::IpAddr) -> bool {
917        for (feed_name, feed_config) in &self.config.feeds {
918            if !feed_config.enabled {
919                continue;
920            }
921
922            if matches!(feed_config.feed_type, FeedType::MaliciousIPs) {
923                let file_path = self.config.feeds_directory.join(&feed_config.filename);
924                if self.check_ip_in_feed(&file_path, ip) {
925                    tracing::warn!("Malicious IP detected: {} (source: {})", ip, feed_name);
926                    return true;
927                }
928            }
929        }
930        false
931    }
932
933    /// Check if an IP address is a Tor exit node
934    pub fn is_tor_exit(&self, ip: &std::net::IpAddr) -> bool {
935        for (feed_name, feed_config) in &self.config.feeds {
936            if !feed_config.enabled {
937                continue;
938            }
939
940            if matches!(feed_config.feed_type, FeedType::TorExitNodes) {
941                let file_path = self.config.feeds_directory.join(&feed_config.filename);
942                if self.check_ip_in_feed(&file_path, ip) {
943                    tracing::warn!("Tor exit node detected: {} (source: {})", ip, feed_name);
944                    return true;
945                }
946            }
947        }
948        false
949    }
950
951    /// Check if an IP address is from a VPN or proxy service
952    pub fn is_proxy_vpn(&self, ip: &std::net::IpAddr) -> bool {
953        for (feed_name, feed_config) in &self.config.feeds {
954            if !feed_config.enabled {
955                continue;
956            }
957
958            if matches!(feed_config.feed_type, FeedType::VpnProxy) {
959                let file_path = self.config.feeds_directory.join(&feed_config.filename);
960                if self.check_ip_in_feed(&file_path, ip) {
961                    tracing::info!("VPN/Proxy detected: {} (source: {})", ip, feed_name);
962                    return true;
963                }
964            }
965        }
966        false
967    }
968
969    /// Helper method to check if an IP is present in a feed file
970    fn check_ip_in_feed(&self, file_path: &std::path::Path, ip: &std::net::IpAddr) -> bool {
971        if !file_path.exists() {
972            return false;
973        }
974
975        if let Ok(contents) = std::fs::read_to_string(file_path) {
976            for line in contents.lines() {
977                let line = line.trim();
978                if line.is_empty() || line.starts_with('#') {
979                    continue;
980                }
981
982                // Check exact IP match
983                if line == ip.to_string() {
984                    return true;
985                }
986
987                // Check CIDR network match
988                if line.contains('/') {
989                    match ip {
990                        std::net::IpAddr::V4(ipv4) => {
991                            if let Ok(network) = line.parse::<ipnetwork::Ipv4Network>()
992                                && network.contains(*ipv4)
993                            {
994                                return true;
995                            }
996                        }
997                        std::net::IpAddr::V6(ipv6) => {
998                            if let Ok(network) = line.parse::<ipnetwork::Ipv6Network>()
999                                && network.contains(*ipv6)
1000                            {
1001                                return true;
1002                            }
1003                        }
1004                    }
1005                }
1006            }
1007        }
1008
1009        false
1010    }
1011}
1012
1013/// Status of a threat intelligence feed
1014#[derive(Debug, Clone)]
1015pub enum FeedStatus {
1016    /// Feed is disabled
1017    Disabled,
1018
1019    /// Feed is enabled but not yet downloaded
1020    NotDownloaded,
1021
1022    /// Feed is active and up-to-date
1023    Active {
1024        last_updated: std::time::SystemTime,
1025        size_bytes: u64,
1026    },
1027
1028    /// Feed has an error
1029    Error(String),
1030}