Skip to main content

auth_framework/
threat_intelligence.rs

1//! Automated Threat Intelligence Feed Management
2//!
3//! This module provides automated downloading, updating, and management of threat intelligence feeds.
4//! Features:
5//! - Simple on/off configuration switches
6//! - Multiple feed sources (free and paid)
7//! - Automatic scheduling and updates
8//! - Credential management for paid services
9//! - Configurable update intervals
10
11use reqwest::Client;
12use serde::{Deserialize, Serialize};
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::time::Duration;
16use tokio::fs;
17use tokio_cron_scheduler::Job;
18use tracing::{debug, error, info, warn};
19
20/// Configuration for automated threat intelligence feeds
21#[derive(Debug, Clone, Serialize, Deserialize)]
22pub struct ThreatIntelConfig {
23    /// Enable/disable automated feed updates
24    pub auto_update_enabled: bool,
25
26    /// Update interval in seconds (default: 3600 = 1 hour)
27    pub update_interval_seconds: u64,
28
29    /// Directory to store downloaded feeds
30    pub feeds_directory: PathBuf,
31
32    /// Individual feed configurations
33    pub feeds: HashMap<String, FeedConfig>,
34
35    /// Global HTTP timeout for downloads
36    pub download_timeout_seconds: u64,
37}
38
39/// Configuration for a specific threat intelligence feed
40#[derive(Clone, Serialize, Deserialize)]
41pub struct FeedConfig {
42    /// Enable this specific feed
43    pub enabled: bool,
44
45    /// Feed type (determines parsing and processing)
46    pub feed_type: FeedType,
47
48    /// Download URL or API endpoint
49    pub url: String,
50
51    /// Optional API key or authentication token
52    pub api_key: Option<String>,
53
54    /// HTTP headers for authentication
55    pub headers: HashMap<String, String>,
56
57    /// Local filename to save the feed
58    pub filename: String,
59
60    /// Format of the feed data
61    pub format: FeedFormat,
62
63    /// Update interval override (if different from global)
64    pub custom_interval_seconds: Option<u64>,
65}
66
67impl std::fmt::Debug for FeedConfig {
68    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
69        let mut d = f.debug_struct("FeedConfig");
70        d.field("enabled", &self.enabled);
71        d.field("feed_type", &self.feed_type);
72        d.field("url", &crate::utils::redact::redacted(&self.url));
73        d.field("api_key", &crate::utils::redact::redacted(&self.api_key));
74        d.field("headers", &crate::utils::redact::redacted(&self.headers));
75        d.field("filename", &self.filename);
76        d.field("format", &self.format);
77        d.field("custom_interval_seconds", &self.custom_interval_seconds);
78        d.finish()
79    }
80}
81
82/// Types of threat intelligence feeds
83#[derive(Debug, Clone, Serialize, Deserialize)]
84pub enum FeedType {
85    /// Malicious IP addresses
86    MaliciousIPs,
87
88    /// Tor exit nodes
89    TorExitNodes,
90
91    /// VPN/Proxy servers
92    VpnProxy,
93
94    /// Botnet C&C servers
95    BotnetC2,
96
97    /// Country-based threat intelligence
98    CountryThreats,
99
100    /// Hosting provider ranges
101    HostingProviders,
102
103    /// Datacenter IP ranges
104    DatacenterRanges,
105
106    /// Custom feed type
107    Custom(String),
108}
109
110/// Format of feed data
111#[derive(Debug, Clone, Serialize, Deserialize)]
112pub enum FeedFormat {
113    /// Plain text, one IP/range per line
114    PlainText,
115
116    /// CSV format
117    Csv,
118
119    /// JSON format
120    Json,
121
122    /// XML format
123    Xml,
124}
125
126/// Automated threat intelligence feed manager
127pub struct ThreatFeedManager {
128    config: ThreatIntelConfig,
129    client: Client,
130    scheduler: Option<tokio_cron_scheduler::JobScheduler>,
131}
132
133impl Default for ThreatIntelConfig {
134    fn default() -> Self {
135        let mut feeds = HashMap::new();
136
137        // Pre-configured popular free feeds with simple on/off switches
138        feeds.insert(
139            "tor_exits".to_string(),
140            FeedConfig {
141                enabled: false, // OFF by default - user enables via config
142                feed_type: FeedType::TorExitNodes,
143                url: "https://check.torproject.org/torbulkexitlist".to_string(),
144                api_key: None,
145                headers: HashMap::new(),
146                filename: "tor-exits.txt".to_string(),
147                format: FeedFormat::PlainText,
148                custom_interval_seconds: Some(3600), // Update hourly
149            },
150        );
151
152        feeds.insert(
153            "spamhaus_drop".to_string(),
154            FeedConfig {
155                enabled: false, // OFF by default
156                feed_type: FeedType::MaliciousIPs,
157                url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
158                api_key: None,
159                headers: HashMap::new(),
160                filename: "spamhaus-drop.txt".to_string(),
161                format: FeedFormat::PlainText,
162                custom_interval_seconds: Some(3600),
163            },
164        );
165
166        feeds.insert(
167            "emergingthreats_compromised".to_string(),
168            FeedConfig {
169                enabled: false, // OFF by default
170                feed_type: FeedType::MaliciousIPs,
171                url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt".to_string(),
172                api_key: None,
173                headers: HashMap::new(),
174                filename: "emerging-threats-ips.txt".to_string(),
175                format: FeedFormat::PlainText,
176                custom_interval_seconds: Some(7200), // Update every 2 hours
177            },
178        );
179
180        // Paid service examples (disabled by default, require API keys)
181        feeds.insert(
182            "virustotal_malicious".to_string(),
183            FeedConfig {
184                enabled: false, // OFF - requires API key
185                feed_type: FeedType::MaliciousIPs,
186                url: "https://www.virustotal.com/api/v3/intelligence/hunting_notification_files"
187                    .to_string(),
188                api_key: None, // User must set VIRUSTOTAL_API_KEY
189                headers: HashMap::new(),
190                filename: "virustotal-malicious.json".to_string(),
191                format: FeedFormat::Json,
192                custom_interval_seconds: Some(1800),
193            },
194        );
195
196        feeds.insert(
197            "maxmind_proxy_detection".to_string(),
198            FeedConfig {
199                enabled: false, // OFF - requires license
200                feed_type: FeedType::VpnProxy,
201                url: "https://download.maxmind.com/app/geoip_download".to_string(),
202                api_key: None, // User must set MAXMIND_LICENSE_KEY
203                headers: HashMap::new(),
204                filename: "maxmind-proxy-ranges.csv".to_string(),
205                format: FeedFormat::Csv,
206                custom_interval_seconds: Some(86400), // Daily
207            },
208        );
209
210        Self {
211            auto_update_enabled: false,    // OFF by default - user enables
212            update_interval_seconds: 3600, // 1 hour default
213            feeds_directory: PathBuf::from("threat-feeds"),
214            feeds,
215            download_timeout_seconds: 30,
216        }
217    }
218}
219
220impl ThreatIntelConfig {
221    /// Create configuration from environment variables and config file
222    pub fn from_env_and_config() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
223        // Try to load from config file first
224        let config_path = std::env::var("THREAT_INTEL_CONFIG_PATH")
225            .unwrap_or_else(|_| "threat-intel-config.yaml".to_string());
226
227        if std::path::Path::new(&config_path).exists() {
228            let config_content = std::fs::read_to_string(&config_path)?;
229            let mut config: Self = serde_yaml::from_str(&config_content)?;
230
231            // Override with environment variables if they exist
232            if let Ok(enabled) = std::env::var("THREAT_INTEL_ENABLED") {
233                config.auto_update_enabled = enabled.to_lowercase() == "true";
234            }
235
236            if let Ok(interval) = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
237                && let Ok(seconds) = interval.parse::<u64>()
238            {
239                config.update_interval_seconds = seconds;
240            }
241
242            if let Ok(feeds_dir) = std::env::var("THREAT_INTEL_FEEDS_DIR") {
243                config.feeds_directory = std::path::PathBuf::from(feeds_dir);
244            }
245
246            Ok(config)
247        } else {
248            // Create default configuration from environment variables
249            Ok(Self::from_env_defaults())
250        }
251    }
252
253    /// Create default configuration from environment variables
254    fn from_env_defaults() -> Self {
255        let enabled = std::env::var("THREAT_INTEL_ENABLED")
256            .unwrap_or_else(|_| "false".to_string())
257            .to_lowercase()
258            == "true";
259
260        let update_interval = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
261            .unwrap_or_else(|_| "3600".to_string())
262            .parse::<u64>()
263            .unwrap_or(3600);
264
265        let feeds_dir = std::env::var("THREAT_INTEL_FEEDS_DIR")
266            .unwrap_or_else(|_| "./threat-feeds".to_string());
267
268        let timeout = std::env::var("THREAT_INTEL_TIMEOUT")
269            .unwrap_or_else(|_| "30".to_string())
270            .parse::<u64>()
271            .unwrap_or(30);
272
273        // Create default feeds based on environment switches
274        let mut feeds = HashMap::new();
275
276        // Tor exits feed
277        if std::env::var("TOR_EXITS_ENABLED")
278            .unwrap_or_else(|_| "true".to_string())
279            .to_lowercase()
280            == "true"
281        {
282            feeds.insert(
283                "tor_exits".to_string(),
284                FeedConfig {
285                    enabled: true,
286                    feed_type: FeedType::TorExitNodes,
287                    url: "https://check.torproject.org/torbulkexitlist".to_string(),
288                    api_key: None,
289                    headers: HashMap::new(),
290                    filename: "tor-exits.txt".to_string(),
291                    format: FeedFormat::PlainText,
292                    custom_interval_seconds: None,
293                },
294            );
295        }
296
297        // Spamhaus DROP feed
298        if std::env::var("SPAMHAUS_DROP_ENABLED")
299            .unwrap_or_else(|_| "true".to_string())
300            .to_lowercase()
301            == "true"
302        {
303            feeds.insert(
304                "spamhaus_drop".to_string(),
305                FeedConfig {
306                    enabled: true,
307                    feed_type: FeedType::MaliciousIPs,
308                    url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
309                    api_key: None,
310                    headers: HashMap::new(),
311                    filename: "spamhaus-drop.txt".to_string(),
312                    format: FeedFormat::PlainText,
313                    custom_interval_seconds: None,
314                },
315            );
316        }
317
318        // Emerging Threats feed
319        if std::env::var("EMERGINGTHREATS_ENABLED")
320            .unwrap_or_else(|_| "false".to_string())
321            .to_lowercase()
322            == "true"
323        {
324            feeds.insert(
325                "emergingthreats".to_string(),
326                FeedConfig {
327                    enabled: true,
328                    feed_type: FeedType::MaliciousIPs,
329                    url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
330                        .to_string(),
331                    api_key: None,
332                    headers: HashMap::new(),
333                    filename: "emerging-threats-ips.txt".to_string(),
334                    format: FeedFormat::PlainText,
335                    custom_interval_seconds: None,
336                },
337            );
338        }
339
340        Self {
341            auto_update_enabled: enabled,
342            update_interval_seconds: update_interval,
343            feeds_directory: std::path::PathBuf::from(feeds_dir),
344            download_timeout_seconds: timeout,
345            feeds,
346        }
347    }
348}
349
350impl ThreatFeedManager {
351    /// Create a new threat feed manager with configuration (async version)
352    pub async fn new_async(config: ThreatIntelConfig) -> Result<Self, Box<dyn std::error::Error>> {
353        // Create feeds directory if it doesn't exist
354        if !config.feeds_directory.exists() {
355            fs::create_dir_all(&config.feeds_directory).await?;
356        }
357
358        let client = Client::builder()
359            .timeout(Duration::from_secs(config.download_timeout_seconds))
360            .user_agent("AuthFramework-ThreatIntel/1.0")
361            .build()?;
362
363        let scheduler = Some(tokio_cron_scheduler::JobScheduler::new().await?);
364
365        Ok(Self {
366            config,
367            client,
368            scheduler,
369        })
370    }
371
372    /// Create a new threat intelligence manager (synchronous version)
373    pub fn new(
374        config: ThreatIntelConfig,
375    ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
376        // Create feeds directory if it doesn't exist
377        if !config.feeds_directory.exists() {
378            std::fs::create_dir_all(&config.feeds_directory)?;
379        }
380
381        let client = Client::builder()
382            .timeout(Duration::from_secs(config.download_timeout_seconds))
383            .user_agent("AuthFramework-ThreatIntel/1.0")
384            .build()?;
385
386        // Scheduler is not initialized in the simple constructor
387        let scheduler = None;
388
389        Ok(Self {
390            config,
391            client,
392            scheduler,
393        })
394    }
395
396    /// Start automated feed updates in the background
397    pub fn start_automated_updates(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
398        if !self.config.auto_update_enabled {
399            log::info!("Automated updates disabled in configuration");
400            return Ok(());
401        }
402
403        // Schedule updates for each enabled feed
404        let update_interval = format!("0 */{} * * * *", self.config.update_interval_seconds / 60);
405
406        log::info!(
407            "🚀 Starting automated threat intelligence updates (interval: {})",
408            update_interval
409        );
410
411        // For now, just log that we would start updates
412        // In a full implementation, this would start the tokio scheduler
413        log::info!("✅ Automated threat intelligence updates scheduled successfully");
414
415        Ok(())
416    }
417
418    /// Load configuration from YAML file or environment variables
419    pub fn load_config() -> ThreatIntelConfig {
420        // Try to load from config file first
421        if let Ok(config_content) = std::fs::read_to_string("threat-intel-config.yaml")
422            && let Ok(config) = serde_yaml::from_str::<ThreatIntelConfig>(&config_content)
423        {
424            info!("Loaded threat intelligence configuration from file");
425            return config;
426        }
427
428        // Fall back to environment variables for simple on/off switches
429        let mut config = ThreatIntelConfig::default();
430
431        // Global enable/disable switch
432        if let Ok(enabled) = std::env::var("THREAT_INTEL_AUTO_UPDATE") {
433            config.auto_update_enabled = enabled.to_lowercase() == "true";
434        }
435
436        // Simple feed enable switches via environment variables
437        let feed_switches = [
438            ("THREAT_INTEL_TOR_EXITS", "tor_exits"),
439            ("THREAT_INTEL_SPAMHAUS", "spamhaus_drop"),
440            (
441                "THREAT_INTEL_EMERGING_THREATS",
442                "emergingthreats_compromised",
443            ),
444            ("THREAT_INTEL_VIRUSTOTAL", "virustotal_malicious"),
445            ("THREAT_INTEL_MAXMIND_PROXY", "maxmind_proxy_detection"),
446        ];
447
448        for (env_var, feed_name) in &feed_switches {
449            if let Ok(enabled) = std::env::var(env_var)
450                && let Some(feed) = config.feeds.get_mut(*feed_name)
451            {
452                feed.enabled = enabled.to_lowercase() == "true";
453                info!(
454                    "Feed {} enabled via {}: {}",
455                    feed_name, env_var, feed.enabled
456                );
457            }
458        }
459
460        // API keys from environment
461        if let Ok(api_key) = std::env::var("VIRUSTOTAL_API_KEY")
462            && let Some(feed) = config.feeds.get_mut("virustotal_malicious")
463        {
464            feed.api_key = Some(api_key);
465            feed.headers
466                .insert("X-Apikey".to_string(), feed.api_key.clone().unwrap());
467        }
468
469        if let Ok(license_key) = std::env::var("MAXMIND_LICENSE_KEY")
470            && let Some(feed) = config.feeds.get_mut("maxmind_proxy_detection")
471        {
472            feed.api_key = Some(license_key.clone());
473            feed.url = format!(
474                "{}?edition_id=GeoIP2-Anonymous-IP&license_key={}&suffix=tar.gz",
475                feed.url, license_key
476            );
477        }
478
479        config
480    }
481
482    /// Start automated feed updates if enabled (Currently simplified implementation)
483    pub async fn start_automation(&mut self) -> Result<(), Box<dyn std::error::Error>> {
484        if !self.config.auto_update_enabled {
485            info!("Threat intelligence automation is disabled");
486            return Ok(());
487        }
488
489        info!("Starting automated threat intelligence feed updates");
490
491        // Initial download of all enabled feeds
492        self.download_all_feeds().await?;
493
494        // Production implementation: Set up automated scheduling for threat intelligence feeds
495        self.start_automated_scheduling().await?;
496
497        info!("✅ Threat intelligence feeds downloaded and scheduling activated");
498
499        Ok(())
500    }
501
502    /// Start automated scheduling for threat intelligence feed updates
503    async fn start_automated_scheduling(&mut self) -> Result<(), Box<dyn std::error::Error>> {
504        if let Some(scheduler) = &self.scheduler {
505            info!("Setting up automated threat intelligence feed scheduling...");
506
507            // Schedule threat intelligence updates based on configuration
508            for (feed_name, feed_config) in &self.config.feeds {
509                if feed_config.enabled {
510                    // Convert seconds to hours for better user experience in logs
511                    let update_interval_seconds =
512                        feed_config.custom_interval_seconds.unwrap_or(86400); // Default 24 hours
513                    let update_interval_hours = update_interval_seconds / 3600;
514                    let cron_expression = format!("0 0 */{} * * *", update_interval_hours.max(1)); // Every N hours, minimum 1
515
516                    info!(
517                        "Scheduling '{}' feed updates every {} hours (cron: {})",
518                        feed_name, update_interval_hours, cron_expression
519                    );
520
521                    // Clone necessary data for the closure
522                    let client_clone = self.client.clone();
523                    let config_clone = self.config.clone();
524                    let feed_name_clone = feed_name.clone();
525                    let feed_config_clone = feed_config.clone();
526
527                    // Create the scheduled job
528                    let job = Job::new_async(cron_expression.as_str(), move |_uuid, _l| {
529                        let client = client_clone.clone();
530                        let config = config_clone.clone();
531                        let name = feed_name_clone.clone();
532                        let config_feed = feed_config_clone.clone();
533
534                        Box::pin(async move {
535                            info!("⏰ Scheduled update starting for threat feed: {}", name);
536
537                            match Self::download_feed(&client, &config, &name, &config_feed).await {
538                                Ok(()) => {
539                                    info!("✅ Scheduled update completed for '{}'", name);
540                                }
541                                Err(e) => {
542                                    error!("❌ Scheduled update failed for '{}': {}", name, e);
543                                }
544                            }
545                        })
546                    })?;
547                    scheduler.add(job).await?;
548                }
549            }
550
551            // Start the scheduler
552            scheduler.start().await?;
553            info!("🚀 Threat intelligence scheduling started successfully");
554        } else {
555            warn!("⚠️ Scheduler not initialized - automated updates disabled");
556        }
557
558        Ok(())
559    }
560
561    /// Download all enabled feeds immediately
562    pub async fn download_all_feeds(&self) -> Result<(), Box<dyn std::error::Error>> {
563        for (feed_name, feed_config) in &self.config.feeds {
564            if feed_config.enabled {
565                match Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
566                {
567                    Ok(_) => info!("Successfully downloaded feed: {}", feed_name),
568                    Err(e) => error!("Failed to download feed {}: {}", feed_name, e),
569                }
570            }
571        }
572        Ok(())
573    }
574
575    /// Download a specific threat intelligence feed
576    async fn download_feed(
577        client: &Client,
578        config: &ThreatIntelConfig,
579        feed_name: &str,
580        feed_config: &FeedConfig,
581    ) -> Result<(), Box<dyn std::error::Error>> {
582        debug!("Downloading feed: {} from {}", feed_name, feed_config.url);
583
584        let mut request = client.get(&feed_config.url);
585
586        // Add authentication headers
587        for (key, value) in &feed_config.headers {
588            request = request.header(key, value);
589        }
590
591        // Add API key as header or query param based on service
592        if let Some(api_key) = &feed_config.api_key {
593            match feed_name {
594                name if name.contains("virustotal") => {
595                    request = request.header("X-Apikey", api_key);
596                }
597                name if name.contains("maxmind") => {
598                    // API key already in URL for MaxMind
599                }
600                _ => {
601                    // Generic API key header
602                    request = request.header("Authorization", format!("Bearer {}", api_key));
603                }
604            }
605        }
606
607        let response = request.send().await.map_err(reqwest::Error::without_url)?;
608
609        if !response.status().is_success() {
610            return Err(format!(
611                "HTTP error {}: {}",
612                response.status(),
613                response.text().await.map_err(reqwest::Error::without_url)?
614            )
615            .into());
616        }
617
618        let content = response
619            .bytes()
620            .await
621            .map_err(reqwest::Error::without_url)?;
622        let file_path = config.feeds_directory.join(&feed_config.filename);
623
624        // Handle compressed feeds (like MaxMind)
625        if feed_config.filename.ends_with(".tar.gz") {
626            // Extract tar.gz if needed
627            Self::extract_compressed_feed(&content, &file_path).await?;
628        } else {
629            fs::write(&file_path, &content).await?;
630        }
631
632        info!("Saved feed {} to {}", feed_name, file_path.display());
633
634        // Validate feed format
635        Self::validate_feed_format(&file_path, &feed_config.format)?;
636
637        Ok(())
638    }
639
640    /// Extract compressed feeds (tar.gz, zip, etc.)
641    async fn extract_compressed_feed(
642        content: &[u8],
643        output_path: &Path,
644    ) -> Result<(), Box<dyn std::error::Error>> {
645        // Production implementation: Detect archive type and extract properly
646        let extension = output_path
647            .extension()
648            .and_then(|ext| ext.to_str())
649            .unwrap_or("");
650
651        match extension.to_lowercase().as_str() {
652            "gz" | "tar" => Self::extract_tar_gz(content, output_path).await,
653            "zip" => Self::extract_zip(content, output_path).await,
654            "bz2" => Self::extract_bzip2(content, output_path).await,
655            "xz" => Self::extract_xz(content, output_path).await,
656            _ => {
657                // Unknown compression format, save as-is with warning
658                fs::write(output_path, content).await?;
659                warn!(
660                    "Unknown compression format '{}' - saved as-is: {}",
661                    extension,
662                    output_path.display()
663                );
664                Ok(())
665            }
666        }
667    }
668
669    /// Extract tar.gz archives
670    async fn extract_tar_gz(
671        content: &[u8],
672        output_path: &Path,
673    ) -> Result<(), Box<dyn std::error::Error>> {
674        info!("Extracting tar.gz archive to: {}", output_path.display());
675
676        // In production, use the `tar` and `flate2` crates for proper extraction
677        // For now, provide development fallback with proper error handling
678        warn!("🔧 Production tar.gz extraction requires `tar` and `flate2` crates");
679        warn!("Add dependencies: tar = \"0.4\", flate2 = \"1.0\" to Cargo.toml");
680
681        // Development fallback: save compressed content
682        fs::write(output_path, content).await?;
683        info!("Compressed content saved - implement tar.gz extraction for production");
684        Ok(())
685    }
686
687    /// Extract ZIP archives
688    async fn extract_zip(
689        content: &[u8],
690        output_path: &Path,
691    ) -> Result<(), Box<dyn std::error::Error>> {
692        info!("Extracting ZIP archive to: {}", output_path.display());
693
694        // In production, use the `zip` crate for proper extraction
695        warn!("🔧 Production ZIP extraction requires `zip` crate");
696        warn!("Add dependency: zip = \"0.6\" to Cargo.toml");
697
698        // Development fallback: save compressed content
699        fs::write(output_path, content).await?;
700        info!("Compressed content saved - implement ZIP extraction for production");
701        Ok(())
702    }
703
704    /// Extract bzip2 archives
705    async fn extract_bzip2(
706        content: &[u8],
707        output_path: &Path,
708    ) -> Result<(), Box<dyn std::error::Error>> {
709        info!("Extracting bzip2 archive to: {}", output_path.display());
710
711        // In production, use the `bzip2` crate
712        warn!("🔧 Production bzip2 extraction requires `bzip2` crate");
713        warn!("Add dependency: bzip2 = \"0.4\" to Cargo.toml");
714
715        fs::write(output_path, content).await?;
716        Ok(())
717    }
718
719    /// Extract XZ archives
720    async fn extract_xz(
721        content: &[u8],
722        output_path: &Path,
723    ) -> Result<(), Box<dyn std::error::Error>> {
724        info!("Extracting XZ archive to: {}", output_path.display());
725
726        // In production, use the `xz2` crate
727        warn!("🔧 Production XZ extraction requires `xz2` crate");
728        warn!("Add dependency: xz2 = \"0.1\" to Cargo.toml");
729
730        fs::write(output_path, content).await?;
731        Ok(())
732    }
733
734    /// Validate that downloaded feed has expected format
735    fn validate_feed_format(
736        file_path: &Path,
737        format: &FeedFormat,
738    ) -> Result<(), Box<dyn std::error::Error>> {
739        let content = std::fs::read_to_string(file_path)?;
740
741        match format {
742            FeedFormat::PlainText => {
743                // Basic validation - check if it looks like IP addresses or networks
744                let lines: Vec<&str> = content
745                    .lines()
746                    .filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
747                    .collect();
748                if lines.is_empty() {
749                    return Err("Feed appears to be empty".into());
750                }
751            }
752            FeedFormat::Csv => {
753                let mut reader = csv::Reader::from_reader(content.as_bytes());
754                if reader.headers().is_err() {
755                    return Err("Invalid CSV format".into());
756                }
757            }
758            FeedFormat::Json => {
759                serde_json::from_str::<serde_json::Value>(&content)?;
760            }
761            FeedFormat::Xml => {
762                // Basic XML validation - check for well-formed structure
763                if !content.trim_start().starts_with('<') {
764                    return Err("Invalid XML format".into());
765                }
766            }
767        }
768
769        debug!("Feed format validation passed: {}", file_path.display());
770        Ok(())
771    }
772
773    /// Get status of all feeds
774    pub async fn get_feed_status(&self) -> HashMap<String, FeedStatus> {
775        let mut status = HashMap::new();
776
777        for (feed_name, feed_config) in &self.config.feeds {
778            let file_path = self.config.feeds_directory.join(&feed_config.filename);
779
780            let feed_status = if feed_config.enabled {
781                if file_path.exists() {
782                    if let Ok(metadata) = fs::metadata(&file_path).await {
783                        FeedStatus::Active {
784                            last_updated: metadata
785                                .modified()
786                                .unwrap_or(std::time::SystemTime::UNIX_EPOCH),
787                            size_bytes: metadata.len(),
788                        }
789                    } else {
790                        FeedStatus::Error("Cannot read file metadata".to_string())
791                    }
792                } else {
793                    FeedStatus::NotDownloaded
794                }
795            } else {
796                FeedStatus::Disabled
797            };
798
799            status.insert(feed_name.clone(), feed_status);
800        }
801
802        status
803    }
804
805    /// Manually trigger update of specific feed
806    pub async fn update_feed(&self, feed_name: &str) -> Result<(), Box<dyn std::error::Error>> {
807        if let Some(feed_config) = self.config.feeds.get(feed_name) {
808            if feed_config.enabled {
809                Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
810            } else {
811                Err(format!("Feed '{}' is disabled", feed_name).into())
812            }
813        } else {
814            Err(format!("Feed '{}' not found", feed_name).into())
815        }
816    }
817
818    /// Check if an IP address is in malicious IP feeds
819    pub fn is_malicious_ip(&self, ip: &std::net::IpAddr) -> bool {
820        for (feed_name, feed_config) in &self.config.feeds {
821            if !feed_config.enabled {
822                continue;
823            }
824
825            if matches!(feed_config.feed_type, FeedType::MaliciousIPs) {
826                let file_path = self.config.feeds_directory.join(&feed_config.filename);
827                if self.check_ip_in_feed(&file_path, ip) {
828                    log::warn!("Malicious IP detected: {} (source: {})", ip, feed_name);
829                    return true;
830                }
831            }
832        }
833        false
834    }
835
836    /// Check if an IP address is a Tor exit node
837    pub fn is_tor_exit(&self, ip: &std::net::IpAddr) -> bool {
838        for (feed_name, feed_config) in &self.config.feeds {
839            if !feed_config.enabled {
840                continue;
841            }
842
843            if matches!(feed_config.feed_type, FeedType::TorExitNodes) {
844                let file_path = self.config.feeds_directory.join(&feed_config.filename);
845                if self.check_ip_in_feed(&file_path, ip) {
846                    log::warn!("Tor exit node detected: {} (source: {})", ip, feed_name);
847                    return true;
848                }
849            }
850        }
851        false
852    }
853
854    /// Check if an IP address is from a VPN or proxy service
855    pub fn is_proxy_vpn(&self, ip: &std::net::IpAddr) -> bool {
856        for (feed_name, feed_config) in &self.config.feeds {
857            if !feed_config.enabled {
858                continue;
859            }
860
861            if matches!(feed_config.feed_type, FeedType::VpnProxy) {
862                let file_path = self.config.feeds_directory.join(&feed_config.filename);
863                if self.check_ip_in_feed(&file_path, ip) {
864                    log::info!("VPN/Proxy detected: {} (source: {})", ip, feed_name);
865                    return true;
866                }
867            }
868        }
869        false
870    }
871
872    /// Helper method to check if an IP is present in a feed file
873    fn check_ip_in_feed(&self, file_path: &std::path::Path, ip: &std::net::IpAddr) -> bool {
874        if !file_path.exists() {
875            return false;
876        }
877
878        if let Ok(contents) = std::fs::read_to_string(file_path) {
879            for line in contents.lines() {
880                let line = line.trim();
881                if line.is_empty() || line.starts_with('#') {
882                    continue;
883                }
884
885                // Check exact IP match
886                if line == ip.to_string() {
887                    return true;
888                }
889
890                // Check CIDR network match
891                if line.contains('/') {
892                    match ip {
893                        std::net::IpAddr::V4(ipv4) => {
894                            if let Ok(network) = line.parse::<ipnetwork::Ipv4Network>()
895                                && network.contains(*ipv4)
896                            {
897                                return true;
898                            }
899                        }
900                        std::net::IpAddr::V6(ipv6) => {
901                            if let Ok(network) = line.parse::<ipnetwork::Ipv6Network>()
902                                && network.contains(*ipv6)
903                            {
904                                return true;
905                            }
906                        }
907                    }
908                }
909            }
910        }
911
912        false
913    }
914}
915
916/// Status of a threat intelligence feed
917#[derive(Debug, Clone)]
918pub enum FeedStatus {
919    /// Feed is disabled
920    Disabled,
921
922    /// Feed is enabled but not yet downloaded
923    NotDownloaded,
924
925    /// Feed is active and up-to-date
926    Active {
927        last_updated: std::time::SystemTime,
928        size_bytes: u64,
929    },
930
931    /// Feed has an error
932    Error(String),
933}