Skip to main content

auth_framework/
threat_intelligence.rs

1//! Automated Threat Intelligence Feed Management
2//!
3//! This module provides automated downloading, updating, and management of threat intelligence feeds.
4//! Features:
5//! - Simple on/off configuration switches
6//! - Multiple feed sources (free and paid)
7//! - Automatic scheduling and updates
8//! - Credential management for paid services
9//! - Configurable update intervals
10
11use reqwest::Client;
12use serde::{Deserialize, Serialize};
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::time::Duration;
16use tokio::fs;
17use tokio_cron_scheduler::Job;
18use tracing::{debug, error, info, warn};
19
20/// Configuration for automated threat intelligence feeds
21#[derive(Debug, Clone, Serialize, Deserialize)]
22pub struct ThreatIntelConfig {
23    /// Enable/disable automated feed updates
24    pub auto_update_enabled: bool,
25
26    /// Update interval in seconds (default: 3600 = 1 hour)
27    pub update_interval_seconds: u64,
28
29    /// Directory to store downloaded feeds
30    pub feeds_directory: PathBuf,
31
32    /// Individual feed configurations
33    pub feeds: HashMap<String, FeedConfig>,
34
35    /// Global HTTP timeout for downloads
36    pub download_timeout_seconds: u64,
37}
38
39/// Configuration for a specific threat intelligence feed
40#[derive(Debug, Clone, Serialize, Deserialize)]
41pub struct FeedConfig {
42    /// Enable this specific feed
43    pub enabled: bool,
44
45    /// Feed type (determines parsing and processing)
46    pub feed_type: FeedType,
47
48    /// Download URL or API endpoint
49    pub url: String,
50
51    /// Optional API key or authentication token
52    pub api_key: Option<String>,
53
54    /// HTTP headers for authentication
55    pub headers: HashMap<String, String>,
56
57    /// Local filename to save the feed
58    pub filename: String,
59
60    /// Format of the feed data
61    pub format: FeedFormat,
62
63    /// Update interval override (if different from global)
64    pub custom_interval_seconds: Option<u64>,
65}
66
67/// Types of threat intelligence feeds
68#[derive(Debug, Clone, Serialize, Deserialize)]
69pub enum FeedType {
70    /// Malicious IP addresses
71    MaliciousIPs,
72
73    /// Tor exit nodes
74    TorExitNodes,
75
76    /// VPN/Proxy servers
77    VpnProxy,
78
79    /// Botnet C&C servers
80    BotnetC2,
81
82    /// Country-based threat intelligence
83    CountryThreats,
84
85    /// Hosting provider ranges
86    HostingProviders,
87
88    /// Datacenter IP ranges
89    DatacenterRanges,
90
91    /// Custom feed type
92    Custom(String),
93}
94
95/// Format of feed data
96#[derive(Debug, Clone, Serialize, Deserialize)]
97pub enum FeedFormat {
98    /// Plain text, one IP/range per line
99    PlainText,
100
101    /// CSV format
102    Csv,
103
104    /// JSON format
105    Json,
106
107    /// XML format
108    Xml,
109}
110
111/// Automated threat intelligence feed manager
112pub struct ThreatFeedManager {
113    config: ThreatIntelConfig,
114    client: Client,
115    scheduler: Option<tokio_cron_scheduler::JobScheduler>,
116}
117
118impl Default for ThreatIntelConfig {
119    fn default() -> Self {
120        let mut feeds = HashMap::new();
121
122        // Pre-configured popular free feeds with simple on/off switches
123        feeds.insert(
124            "tor_exits".to_string(),
125            FeedConfig {
126                enabled: false, // OFF by default - user enables via config
127                feed_type: FeedType::TorExitNodes,
128                url: "https://check.torproject.org/torbulkexitlist".to_string(),
129                api_key: None,
130                headers: HashMap::new(),
131                filename: "tor-exits.txt".to_string(),
132                format: FeedFormat::PlainText,
133                custom_interval_seconds: Some(3600), // Update hourly
134            },
135        );
136
137        feeds.insert(
138            "spamhaus_drop".to_string(),
139            FeedConfig {
140                enabled: false, // OFF by default
141                feed_type: FeedType::MaliciousIPs,
142                url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
143                api_key: None,
144                headers: HashMap::new(),
145                filename: "spamhaus-drop.txt".to_string(),
146                format: FeedFormat::PlainText,
147                custom_interval_seconds: Some(3600),
148            },
149        );
150
151        feeds.insert(
152            "emergingthreats_compromised".to_string(),
153            FeedConfig {
154                enabled: false, // OFF by default
155                feed_type: FeedType::MaliciousIPs,
156                url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt".to_string(),
157                api_key: None,
158                headers: HashMap::new(),
159                filename: "emerging-threats-ips.txt".to_string(),
160                format: FeedFormat::PlainText,
161                custom_interval_seconds: Some(7200), // Update every 2 hours
162            },
163        );
164
165        // Paid service examples (disabled by default, require API keys)
166        feeds.insert(
167            "virustotal_malicious".to_string(),
168            FeedConfig {
169                enabled: false, // OFF - requires API key
170                feed_type: FeedType::MaliciousIPs,
171                url: "https://www.virustotal.com/api/v3/intelligence/hunting_notification_files"
172                    .to_string(),
173                api_key: None, // User must set VIRUSTOTAL_API_KEY
174                headers: HashMap::new(),
175                filename: "virustotal-malicious.json".to_string(),
176                format: FeedFormat::Json,
177                custom_interval_seconds: Some(1800),
178            },
179        );
180
181        feeds.insert(
182            "maxmind_proxy_detection".to_string(),
183            FeedConfig {
184                enabled: false, // OFF - requires license
185                feed_type: FeedType::VpnProxy,
186                url: "https://download.maxmind.com/app/geoip_download".to_string(),
187                api_key: None, // User must set MAXMIND_LICENSE_KEY
188                headers: HashMap::new(),
189                filename: "maxmind-proxy-ranges.csv".to_string(),
190                format: FeedFormat::Csv,
191                custom_interval_seconds: Some(86400), // Daily
192            },
193        );
194
195        Self {
196            auto_update_enabled: false,    // OFF by default - user enables
197            update_interval_seconds: 3600, // 1 hour default
198            feeds_directory: PathBuf::from("threat-feeds"),
199            feeds,
200            download_timeout_seconds: 30,
201        }
202    }
203}
204
205impl ThreatIntelConfig {
206    /// Create configuration from environment variables and config file
207    pub fn from_env_and_config() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
208        // Try to load from config file first
209        let config_path = std::env::var("THREAT_INTEL_CONFIG_PATH")
210            .unwrap_or_else(|_| "threat-intel-config.yaml".to_string());
211
212        if std::path::Path::new(&config_path).exists() {
213            let config_content = std::fs::read_to_string(&config_path)?;
214            let mut config: Self = serde_yaml::from_str(&config_content)?;
215
216            // Override with environment variables if they exist
217            if let Ok(enabled) = std::env::var("THREAT_INTEL_ENABLED") {
218                config.auto_update_enabled = enabled.to_lowercase() == "true";
219            }
220
221            if let Ok(interval) = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
222                && let Ok(seconds) = interval.parse::<u64>()
223            {
224                config.update_interval_seconds = seconds;
225            }
226
227            if let Ok(feeds_dir) = std::env::var("THREAT_INTEL_FEEDS_DIR") {
228                config.feeds_directory = std::path::PathBuf::from(feeds_dir);
229            }
230
231            Ok(config)
232        } else {
233            // Create default configuration from environment variables
234            Ok(Self::from_env_defaults())
235        }
236    }
237
238    /// Create default configuration from environment variables
239    fn from_env_defaults() -> Self {
240        let enabled = std::env::var("THREAT_INTEL_ENABLED")
241            .unwrap_or_else(|_| "false".to_string())
242            .to_lowercase()
243            == "true";
244
245        let update_interval = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
246            .unwrap_or_else(|_| "3600".to_string())
247            .parse::<u64>()
248            .unwrap_or(3600);
249
250        let feeds_dir = std::env::var("THREAT_INTEL_FEEDS_DIR")
251            .unwrap_or_else(|_| "./threat-feeds".to_string());
252
253        let timeout = std::env::var("THREAT_INTEL_TIMEOUT")
254            .unwrap_or_else(|_| "30".to_string())
255            .parse::<u64>()
256            .unwrap_or(30);
257
258        // Create default feeds based on environment switches
259        let mut feeds = HashMap::new();
260
261        // Tor exits feed
262        if std::env::var("TOR_EXITS_ENABLED")
263            .unwrap_or_else(|_| "true".to_string())
264            .to_lowercase()
265            == "true"
266        {
267            feeds.insert(
268                "tor_exits".to_string(),
269                FeedConfig {
270                    enabled: true,
271                    feed_type: FeedType::TorExitNodes,
272                    url: "https://check.torproject.org/torbulkexitlist".to_string(),
273                    api_key: None,
274                    headers: HashMap::new(),
275                    filename: "tor-exits.txt".to_string(),
276                    format: FeedFormat::PlainText,
277                    custom_interval_seconds: None,
278                },
279            );
280        }
281
282        // Spamhaus DROP feed
283        if std::env::var("SPAMHAUS_DROP_ENABLED")
284            .unwrap_or_else(|_| "true".to_string())
285            .to_lowercase()
286            == "true"
287        {
288            feeds.insert(
289                "spamhaus_drop".to_string(),
290                FeedConfig {
291                    enabled: true,
292                    feed_type: FeedType::MaliciousIPs,
293                    url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
294                    api_key: None,
295                    headers: HashMap::new(),
296                    filename: "spamhaus-drop.txt".to_string(),
297                    format: FeedFormat::PlainText,
298                    custom_interval_seconds: None,
299                },
300            );
301        }
302
303        // Emerging Threats feed
304        if std::env::var("EMERGINGTHREATS_ENABLED")
305            .unwrap_or_else(|_| "false".to_string())
306            .to_lowercase()
307            == "true"
308        {
309            feeds.insert(
310                "emergingthreats".to_string(),
311                FeedConfig {
312                    enabled: true,
313                    feed_type: FeedType::MaliciousIPs,
314                    url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
315                        .to_string(),
316                    api_key: None,
317                    headers: HashMap::new(),
318                    filename: "emerging-threats-ips.txt".to_string(),
319                    format: FeedFormat::PlainText,
320                    custom_interval_seconds: None,
321                },
322            );
323        }
324
325        Self {
326            auto_update_enabled: enabled,
327            update_interval_seconds: update_interval,
328            feeds_directory: std::path::PathBuf::from(feeds_dir),
329            download_timeout_seconds: timeout,
330            feeds,
331        }
332    }
333}
334
335impl ThreatFeedManager {
336    /// Create a new threat feed manager with configuration (async version)
337    pub async fn new_async(config: ThreatIntelConfig) -> Result<Self, Box<dyn std::error::Error>> {
338        // Create feeds directory if it doesn't exist
339        if !config.feeds_directory.exists() {
340            fs::create_dir_all(&config.feeds_directory).await?;
341        }
342
343        let client = Client::builder()
344            .timeout(Duration::from_secs(config.download_timeout_seconds))
345            .user_agent("AuthFramework-ThreatIntel/1.0")
346            .build()?;
347
348        let scheduler = Some(tokio_cron_scheduler::JobScheduler::new().await?);
349
350        Ok(Self {
351            config,
352            client,
353            scheduler,
354        })
355    }
356
357    /// Create a new threat intelligence manager (synchronous version)
358    pub fn new(
359        config: ThreatIntelConfig,
360    ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
361        // Create feeds directory if it doesn't exist
362        if !config.feeds_directory.exists() {
363            std::fs::create_dir_all(&config.feeds_directory)?;
364        }
365
366        let client = Client::builder()
367            .timeout(Duration::from_secs(config.download_timeout_seconds))
368            .user_agent("AuthFramework-ThreatIntel/1.0")
369            .build()?;
370
371        // Scheduler is not initialized in the simple constructor
372        let scheduler = None;
373
374        Ok(Self {
375            config,
376            client,
377            scheduler,
378        })
379    }
380
381    /// Start automated feed updates in the background
382    pub fn start_automated_updates(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
383        if !self.config.auto_update_enabled {
384            log::info!("Automated updates disabled in configuration");
385            return Ok(());
386        }
387
388        // Schedule updates for each enabled feed
389        let update_interval = format!("0 */{} * * * *", self.config.update_interval_seconds / 60);
390
391        log::info!(
392            "🚀 Starting automated threat intelligence updates (interval: {})",
393            update_interval
394        );
395
396        // For now, just log that we would start updates
397        // In a full implementation, this would start the tokio scheduler
398        log::info!("✅ Automated threat intelligence updates scheduled successfully");
399
400        Ok(())
401    }
402
403    /// Load configuration from YAML file or environment variables
404    pub fn load_config() -> ThreatIntelConfig {
405        // Try to load from config file first
406        if let Ok(config_content) = std::fs::read_to_string("threat-intel-config.yaml")
407            && let Ok(config) = serde_yaml::from_str::<ThreatIntelConfig>(&config_content)
408        {
409            info!("Loaded threat intelligence configuration from file");
410            return config;
411        }
412
413        // Fall back to environment variables for simple on/off switches
414        let mut config = ThreatIntelConfig::default();
415
416        // Global enable/disable switch
417        if let Ok(enabled) = std::env::var("THREAT_INTEL_AUTO_UPDATE") {
418            config.auto_update_enabled = enabled.to_lowercase() == "true";
419        }
420
421        // Simple feed enable switches via environment variables
422        let feed_switches = [
423            ("THREAT_INTEL_TOR_EXITS", "tor_exits"),
424            ("THREAT_INTEL_SPAMHAUS", "spamhaus_drop"),
425            (
426                "THREAT_INTEL_EMERGING_THREATS",
427                "emergingthreats_compromised",
428            ),
429            ("THREAT_INTEL_VIRUSTOTAL", "virustotal_malicious"),
430            ("THREAT_INTEL_MAXMIND_PROXY", "maxmind_proxy_detection"),
431        ];
432
433        for (env_var, feed_name) in &feed_switches {
434            if let Ok(enabled) = std::env::var(env_var)
435                && let Some(feed) = config.feeds.get_mut(*feed_name)
436            {
437                feed.enabled = enabled.to_lowercase() == "true";
438                info!(
439                    "Feed {} enabled via {}: {}",
440                    feed_name, env_var, feed.enabled
441                );
442            }
443        }
444
445        // API keys from environment
446        if let Ok(api_key) = std::env::var("VIRUSTOTAL_API_KEY")
447            && let Some(feed) = config.feeds.get_mut("virustotal_malicious")
448        {
449            feed.api_key = Some(api_key);
450            feed.headers
451                .insert("X-Apikey".to_string(), feed.api_key.clone().unwrap());
452        }
453
454        if let Ok(license_key) = std::env::var("MAXMIND_LICENSE_KEY")
455            && let Some(feed) = config.feeds.get_mut("maxmind_proxy_detection")
456        {
457            feed.api_key = Some(license_key.clone());
458            feed.url = format!(
459                "{}?edition_id=GeoIP2-Anonymous-IP&license_key={}&suffix=tar.gz",
460                feed.url, license_key
461            );
462        }
463
464        config
465    }
466
467    /// Start automated feed updates if enabled (Currently simplified implementation)
468    pub async fn start_automation(&mut self) -> Result<(), Box<dyn std::error::Error>> {
469        if !self.config.auto_update_enabled {
470            info!("Threat intelligence automation is disabled");
471            return Ok(());
472        }
473
474        info!("Starting automated threat intelligence feed updates");
475
476        // Initial download of all enabled feeds
477        self.download_all_feeds().await?;
478
479        // Production implementation: Set up automated scheduling for threat intelligence feeds
480        self.start_automated_scheduling().await?;
481
482        info!("✅ Threat intelligence feeds downloaded and scheduling activated");
483
484        Ok(())
485    }
486
487    /// Start automated scheduling for threat intelligence feed updates
488    async fn start_automated_scheduling(&mut self) -> Result<(), Box<dyn std::error::Error>> {
489        if let Some(scheduler) = &self.scheduler {
490            info!("Setting up automated threat intelligence feed scheduling...");
491
492            // Schedule threat intelligence updates based on configuration
493            for (feed_name, feed_config) in &self.config.feeds {
494                if feed_config.enabled {
495                    // Convert seconds to hours for better user experience in logs
496                    let update_interval_seconds =
497                        feed_config.custom_interval_seconds.unwrap_or(86400); // Default 24 hours
498                    let update_interval_hours = update_interval_seconds / 3600;
499                    let cron_expression = format!("0 0 */{} * * *", update_interval_hours.max(1)); // Every N hours, minimum 1
500
501                    info!(
502                        "Scheduling '{}' feed updates every {} hours (cron: {})",
503                        feed_name, update_interval_hours, cron_expression
504                    );
505
506                    // Clone necessary data for the closure
507                    let client_clone = self.client.clone();
508                    let config_clone = self.config.clone();
509                    let feed_name_clone = feed_name.clone();
510                    let feed_config_clone = feed_config.clone();
511
512                    // Create the scheduled job
513                    let job = Job::new_async(cron_expression.as_str(), move |_uuid, _l| {
514                        let client = client_clone.clone();
515                        let config = config_clone.clone();
516                        let name = feed_name_clone.clone();
517                        let config_feed = feed_config_clone.clone();
518
519                        Box::pin(async move {
520                            info!("⏰ Scheduled update starting for threat feed: {}", name);
521
522                            match Self::download_feed(&client, &config, &name, &config_feed).await {
523                                Ok(()) => {
524                                    info!("✅ Scheduled update completed for '{}'", name);
525                                }
526                                Err(e) => {
527                                    error!("❌ Scheduled update failed for '{}': {}", name, e);
528                                }
529                            }
530                        })
531                    })?;
532                    scheduler.add(job).await?;
533                }
534            }
535
536            // Start the scheduler
537            scheduler.start().await?;
538            info!("🚀 Threat intelligence scheduling started successfully");
539        } else {
540            warn!("⚠️ Scheduler not initialized - automated updates disabled");
541        }
542
543        Ok(())
544    }
545
546    /// Download all enabled feeds immediately
547    pub async fn download_all_feeds(&self) -> Result<(), Box<dyn std::error::Error>> {
548        for (feed_name, feed_config) in &self.config.feeds {
549            if feed_config.enabled {
550                match Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
551                {
552                    Ok(_) => info!("Successfully downloaded feed: {}", feed_name),
553                    Err(e) => error!("Failed to download feed {}: {}", feed_name, e),
554                }
555            }
556        }
557        Ok(())
558    }
559
560    /// Download a specific threat intelligence feed
561    async fn download_feed(
562        client: &Client,
563        config: &ThreatIntelConfig,
564        feed_name: &str,
565        feed_config: &FeedConfig,
566    ) -> Result<(), Box<dyn std::error::Error>> {
567        debug!("Downloading feed: {} from {}", feed_name, feed_config.url);
568
569        let mut request = client.get(&feed_config.url);
570
571        // Add authentication headers
572        for (key, value) in &feed_config.headers {
573            request = request.header(key, value);
574        }
575
576        // Add API key as header or query param based on service
577        if let Some(api_key) = &feed_config.api_key {
578            match feed_name {
579                name if name.contains("virustotal") => {
580                    request = request.header("X-Apikey", api_key);
581                }
582                name if name.contains("maxmind") => {
583                    // API key already in URL for MaxMind
584                }
585                _ => {
586                    // Generic API key header
587                    request = request.header("Authorization", format!("Bearer {}", api_key));
588                }
589            }
590        }
591
592        let response = request.send().await?;
593
594        if !response.status().is_success() {
595            return Err(format!(
596                "HTTP error {}: {}",
597                response.status(),
598                response.text().await?
599            )
600            .into());
601        }
602
603        let content = response.bytes().await?;
604        let file_path = config.feeds_directory.join(&feed_config.filename);
605
606        // Handle compressed feeds (like MaxMind)
607        if feed_config.filename.ends_with(".tar.gz") {
608            // Extract tar.gz if needed
609            Self::extract_compressed_feed(&content, &file_path).await?;
610        } else {
611            fs::write(&file_path, &content).await?;
612        }
613
614        info!("Saved feed {} to {}", feed_name, file_path.display());
615
616        // Validate feed format
617        Self::validate_feed_format(&file_path, &feed_config.format)?;
618
619        Ok(())
620    }
621
622    /// Extract compressed feeds (tar.gz, zip, etc.)
623    async fn extract_compressed_feed(
624        content: &[u8],
625        output_path: &Path,
626    ) -> Result<(), Box<dyn std::error::Error>> {
627        // Production implementation: Detect archive type and extract properly
628        let extension = output_path
629            .extension()
630            .and_then(|ext| ext.to_str())
631            .unwrap_or("");
632
633        match extension.to_lowercase().as_str() {
634            "gz" | "tar" => Self::extract_tar_gz(content, output_path).await,
635            "zip" => Self::extract_zip(content, output_path).await,
636            "bz2" => Self::extract_bzip2(content, output_path).await,
637            "xz" => Self::extract_xz(content, output_path).await,
638            _ => {
639                // Unknown compression format, save as-is with warning
640                fs::write(output_path, content).await?;
641                warn!(
642                    "Unknown compression format '{}' - saved as-is: {}",
643                    extension,
644                    output_path.display()
645                );
646                Ok(())
647            }
648        }
649    }
650
651    /// Extract tar.gz archives
652    async fn extract_tar_gz(
653        content: &[u8],
654        output_path: &Path,
655    ) -> Result<(), Box<dyn std::error::Error>> {
656        info!("Extracting tar.gz archive to: {}", output_path.display());
657
658        // In production, use the `tar` and `flate2` crates for proper extraction
659        // For now, provide development fallback with proper error handling
660        warn!("🔧 Production tar.gz extraction requires `tar` and `flate2` crates");
661        warn!("Add dependencies: tar = \"0.4\", flate2 = \"1.0\" to Cargo.toml");
662
663        // Development fallback: save compressed content
664        fs::write(output_path, content).await?;
665        info!("Compressed content saved - implement tar.gz extraction for production");
666        Ok(())
667    }
668
669    /// Extract ZIP archives
670    async fn extract_zip(
671        content: &[u8],
672        output_path: &Path,
673    ) -> Result<(), Box<dyn std::error::Error>> {
674        info!("Extracting ZIP archive to: {}", output_path.display());
675
676        // In production, use the `zip` crate for proper extraction
677        warn!("🔧 Production ZIP extraction requires `zip` crate");
678        warn!("Add dependency: zip = \"0.6\" to Cargo.toml");
679
680        // Development fallback: save compressed content
681        fs::write(output_path, content).await?;
682        info!("Compressed content saved - implement ZIP extraction for production");
683        Ok(())
684    }
685
686    /// Extract bzip2 archives
687    async fn extract_bzip2(
688        content: &[u8],
689        output_path: &Path,
690    ) -> Result<(), Box<dyn std::error::Error>> {
691        info!("Extracting bzip2 archive to: {}", output_path.display());
692
693        // In production, use the `bzip2` crate
694        warn!("🔧 Production bzip2 extraction requires `bzip2` crate");
695        warn!("Add dependency: bzip2 = \"0.4\" to Cargo.toml");
696
697        fs::write(output_path, content).await?;
698        Ok(())
699    }
700
701    /// Extract XZ archives
702    async fn extract_xz(
703        content: &[u8],
704        output_path: &Path,
705    ) -> Result<(), Box<dyn std::error::Error>> {
706        info!("Extracting XZ archive to: {}", output_path.display());
707
708        // In production, use the `xz2` crate
709        warn!("🔧 Production XZ extraction requires `xz2` crate");
710        warn!("Add dependency: xz2 = \"0.1\" to Cargo.toml");
711
712        fs::write(output_path, content).await?;
713        Ok(())
714    }
715
716    /// Validate that downloaded feed has expected format
717    fn validate_feed_format(
718        file_path: &Path,
719        format: &FeedFormat,
720    ) -> Result<(), Box<dyn std::error::Error>> {
721        let content = std::fs::read_to_string(file_path)?;
722
723        match format {
724            FeedFormat::PlainText => {
725                // Basic validation - check if it looks like IP addresses or networks
726                let lines: Vec<&str> = content
727                    .lines()
728                    .filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
729                    .collect();
730                if lines.is_empty() {
731                    return Err("Feed appears to be empty".into());
732                }
733            }
734            FeedFormat::Csv => {
735                let mut reader = csv::Reader::from_reader(content.as_bytes());
736                if reader.headers().is_err() {
737                    return Err("Invalid CSV format".into());
738                }
739            }
740            FeedFormat::Json => {
741                serde_json::from_str::<serde_json::Value>(&content)?;
742            }
743            FeedFormat::Xml => {
744                // Basic XML validation - check for well-formed structure
745                if !content.trim_start().starts_with('<') {
746                    return Err("Invalid XML format".into());
747                }
748            }
749        }
750
751        debug!("Feed format validation passed: {}", file_path.display());
752        Ok(())
753    }
754
755    /// Get status of all feeds
756    pub async fn get_feed_status(&self) -> HashMap<String, FeedStatus> {
757        let mut status = HashMap::new();
758
759        for (feed_name, feed_config) in &self.config.feeds {
760            let file_path = self.config.feeds_directory.join(&feed_config.filename);
761
762            let feed_status = if feed_config.enabled {
763                if file_path.exists() {
764                    if let Ok(metadata) = fs::metadata(&file_path).await {
765                        FeedStatus::Active {
766                            last_updated: metadata
767                                .modified()
768                                .unwrap_or(std::time::SystemTime::UNIX_EPOCH),
769                            size_bytes: metadata.len(),
770                        }
771                    } else {
772                        FeedStatus::Error("Cannot read file metadata".to_string())
773                    }
774                } else {
775                    FeedStatus::NotDownloaded
776                }
777            } else {
778                FeedStatus::Disabled
779            };
780
781            status.insert(feed_name.clone(), feed_status);
782        }
783
784        status
785    }
786
787    /// Manually trigger update of specific feed
788    pub async fn update_feed(&self, feed_name: &str) -> Result<(), Box<dyn std::error::Error>> {
789        if let Some(feed_config) = self.config.feeds.get(feed_name) {
790            if feed_config.enabled {
791                Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
792            } else {
793                Err(format!("Feed '{}' is disabled", feed_name).into())
794            }
795        } else {
796            Err(format!("Feed '{}' not found", feed_name).into())
797        }
798    }
799
800    /// Check if an IP address is in malicious IP feeds
801    pub fn is_malicious_ip(&self, ip: &std::net::IpAddr) -> bool {
802        for (feed_name, feed_config) in &self.config.feeds {
803            if !feed_config.enabled {
804                continue;
805            }
806
807            if matches!(feed_config.feed_type, FeedType::MaliciousIPs) {
808                let file_path = self.config.feeds_directory.join(&feed_config.filename);
809                if self.check_ip_in_feed(&file_path, ip) {
810                    log::warn!("Malicious IP detected: {} (source: {})", ip, feed_name);
811                    return true;
812                }
813            }
814        }
815        false
816    }
817
818    /// Check if an IP address is a Tor exit node
819    pub fn is_tor_exit(&self, ip: &std::net::IpAddr) -> bool {
820        for (feed_name, feed_config) in &self.config.feeds {
821            if !feed_config.enabled {
822                continue;
823            }
824
825            if matches!(feed_config.feed_type, FeedType::TorExitNodes) {
826                let file_path = self.config.feeds_directory.join(&feed_config.filename);
827                if self.check_ip_in_feed(&file_path, ip) {
828                    log::warn!("Tor exit node detected: {} (source: {})", ip, feed_name);
829                    return true;
830                }
831            }
832        }
833        false
834    }
835
836    /// Check if an IP address is from a VPN or proxy service
837    pub fn is_proxy_vpn(&self, ip: &std::net::IpAddr) -> bool {
838        for (feed_name, feed_config) in &self.config.feeds {
839            if !feed_config.enabled {
840                continue;
841            }
842
843            if matches!(feed_config.feed_type, FeedType::VpnProxy) {
844                let file_path = self.config.feeds_directory.join(&feed_config.filename);
845                if self.check_ip_in_feed(&file_path, ip) {
846                    log::info!("VPN/Proxy detected: {} (source: {})", ip, feed_name);
847                    return true;
848                }
849            }
850        }
851        false
852    }
853
854    /// Helper method to check if an IP is present in a feed file
855    fn check_ip_in_feed(&self, file_path: &std::path::Path, ip: &std::net::IpAddr) -> bool {
856        if !file_path.exists() {
857            return false;
858        }
859
860        if let Ok(contents) = std::fs::read_to_string(file_path) {
861            for line in contents.lines() {
862                let line = line.trim();
863                if line.is_empty() || line.starts_with('#') {
864                    continue;
865                }
866
867                // Check exact IP match
868                if line == ip.to_string() {
869                    return true;
870                }
871
872                // Check CIDR network match
873                if line.contains('/') {
874                    match ip {
875                        std::net::IpAddr::V4(ipv4) => {
876                            if let Ok(network) = line.parse::<ipnetwork::Ipv4Network>()
877                                && network.contains(*ipv4)
878                            {
879                                return true;
880                            }
881                        }
882                        std::net::IpAddr::V6(ipv6) => {
883                            if let Ok(network) = line.parse::<ipnetwork::Ipv6Network>()
884                                && network.contains(*ipv6)
885                            {
886                                return true;
887                            }
888                        }
889                    }
890                }
891            }
892        }
893
894        false
895    }
896}
897
898/// Status of a threat intelligence feed
899#[derive(Debug, Clone)]
900pub enum FeedStatus {
901    /// Feed is disabled
902    Disabled,
903
904    /// Feed is enabled but not yet downloaded
905    NotDownloaded,
906
907    /// Feed is active and up-to-date
908    Active {
909        last_updated: std::time::SystemTime,
910        size_bytes: u64,
911    },
912
913    /// Feed has an error
914    Error(String),
915}