1use reqwest::Client;
12use serde::{Deserialize, Serialize};
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::time::Duration;
16use tokio::fs;
17use tokio_cron_scheduler::Job;
18use tracing::{debug, error, info, warn};
19
20#[cfg(test)]
21mod extraction_cap_tests;
22#[cfg(test)]
23mod tar_extraction_tests;
24
25#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct ThreatIntelConfig {
28 pub auto_update_enabled: bool,
30
31 pub update_interval_seconds: u64,
33
34 pub feeds_directory: PathBuf,
36
37 pub feeds: HashMap<String, FeedConfig>,
39
40 pub download_timeout_seconds: u64,
42
43 #[serde(default)]
46 pub extraction_limits: ExtractionLimits,
47}
48
49#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
52pub struct ExtractionLimits {
53 pub max_entries: usize,
55
56 pub max_entry_bytes: u64,
58
59 pub max_total_bytes: u64,
61}
62
63impl Default for ExtractionLimits {
64 fn default() -> Self {
66 Self {
67 max_entries: 10_000,
68 max_entry_bytes: 512 << 20,
69 max_total_bytes: 1 << 30,
70 }
71 }
72}
73
74#[derive(Clone, Serialize, Deserialize)]
76pub struct FeedConfig {
77 pub enabled: bool,
79
80 pub feed_type: FeedType,
82
83 pub url: String,
85
86 pub api_key: Option<String>,
88
89 pub headers: HashMap<String, String>,
91
92 pub filename: String,
94
95 pub format: FeedFormat,
97
98 pub custom_interval_seconds: Option<u64>,
100}
101
102impl std::fmt::Debug for FeedConfig {
103 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104 let mut d = f.debug_struct("FeedConfig");
105 d.field("enabled", &self.enabled);
106 d.field("feed_type", &self.feed_type);
107 d.field("url", &self.url);
108 d.field("api_key", &crate::utils::redact::redacted(&self.api_key));
109 d.field("headers", &self.headers);
110 d.field("filename", &self.filename);
111 d.field("format", &self.format);
112 d.field("custom_interval_seconds", &self.custom_interval_seconds);
113 d.finish()
114 }
115}
116
117#[derive(Debug, Clone, Serialize, Deserialize)]
119pub enum FeedType {
120 MaliciousIPs,
122
123 TorExitNodes,
125
126 VpnProxy,
128
129 BotnetC2,
131
132 CountryThreats,
134
135 HostingProviders,
137
138 DatacenterRanges,
140
141 Custom(String),
143}
144
145#[derive(Debug, Clone, Serialize, Deserialize)]
147pub enum FeedFormat {
148 PlainText,
150
151 Csv,
153
154 Json,
156
157 Xml,
159}
160
161pub struct ThreatFeedManager {
163 config: ThreatIntelConfig,
164 client: Client,
165 scheduler: Option<tokio_cron_scheduler::JobScheduler>,
166}
167
168#[cfg(test)]
169mod feed_dispatch_tests;
170
171impl Default for ThreatIntelConfig {
172 fn default() -> Self {
173 let mut feeds = HashMap::new();
174
175 feeds.insert(
177 "tor_exits".to_string(),
178 FeedConfig {
179 enabled: false, feed_type: FeedType::TorExitNodes,
181 url: "https://check.torproject.org/torbulkexitlist".to_string(),
182 api_key: None,
183 headers: HashMap::new(),
184 filename: "tor-exits.txt".to_string(),
185 format: FeedFormat::PlainText,
186 custom_interval_seconds: Some(3600), },
188 );
189
190 feeds.insert(
191 "spamhaus_drop".to_string(),
192 FeedConfig {
193 enabled: false, feed_type: FeedType::MaliciousIPs,
195 url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
196 api_key: None,
197 headers: HashMap::new(),
198 filename: "spamhaus-drop.txt".to_string(),
199 format: FeedFormat::PlainText,
200 custom_interval_seconds: Some(3600),
201 },
202 );
203
204 feeds.insert(
205 "emergingthreats_compromised".to_string(),
206 FeedConfig {
207 enabled: false, feed_type: FeedType::MaliciousIPs,
209 url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt".to_string(),
210 api_key: None,
211 headers: HashMap::new(),
212 filename: "emerging-threats-ips.txt".to_string(),
213 format: FeedFormat::PlainText,
214 custom_interval_seconds: Some(7200), },
216 );
217
218 feeds.insert(
220 "virustotal_malicious".to_string(),
221 FeedConfig {
222 enabled: false, feed_type: FeedType::MaliciousIPs,
224 url: "https://www.virustotal.com/api/v3/intelligence/hunting_notification_files"
225 .to_string(),
226 api_key: None, headers: HashMap::new(),
228 filename: "virustotal-malicious.json".to_string(),
229 format: FeedFormat::Json,
230 custom_interval_seconds: Some(1800),
231 },
232 );
233
234 feeds.insert(
235 "maxmind_proxy_detection".to_string(),
236 FeedConfig {
237 enabled: false, feed_type: FeedType::VpnProxy,
239 url: "https://download.maxmind.com/app/geoip_download".to_string(),
240 api_key: None, headers: HashMap::new(),
242 filename: "maxmind-proxy-ranges.csv".to_string(),
243 format: FeedFormat::Csv,
244 custom_interval_seconds: Some(86400), },
246 );
247
248 Self {
249 auto_update_enabled: false, update_interval_seconds: 3600, feeds_directory: PathBuf::from("threat-feeds"),
252 feeds,
253 download_timeout_seconds: 30,
254 extraction_limits: ExtractionLimits::default(),
255 }
256 }
257}
258
259impl ThreatIntelConfig {
260 pub fn disabled() -> Self {
270 Self {
271 auto_update_enabled: false,
272 update_interval_seconds: 0,
273 feeds_directory: PathBuf::from("./threat-feeds"),
274 feeds: HashMap::new(),
275 download_timeout_seconds: 30,
276 extraction_limits: ExtractionLimits::default(),
277 }
278 }
279
280 pub fn aggressive() -> Self {
290 let mut base = Self {
291 auto_update_enabled: true,
292 update_interval_seconds: 300, download_timeout_seconds: 10,
294 extraction_limits: ExtractionLimits::default(),
295 ..Default::default()
296 };
297 for feed in base.feeds.values_mut() {
299 feed.enabled = true;
300 }
301 base
302 }
303
304 pub fn from_env_and_config() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
306 let config_path = std::env::var("THREAT_INTEL_CONFIG_PATH")
308 .unwrap_or_else(|_| "threat-intel-config.yaml".to_string());
309
310 if std::path::Path::new(&config_path).exists() {
311 let config_content = std::fs::read_to_string(&config_path)?;
312 let mut config: Self = serde_saphyr::from_str(&config_content)?;
313
314 if let Ok(enabled) = std::env::var("THREAT_INTEL_ENABLED") {
316 config.auto_update_enabled = enabled.to_lowercase() == "true";
317 }
318
319 if let Ok(interval) = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
320 && let Ok(seconds) = interval.parse::<u64>()
321 {
322 config.update_interval_seconds = seconds;
323 }
324
325 if let Ok(feeds_dir) = std::env::var("THREAT_INTEL_FEEDS_DIR") {
326 config.feeds_directory = std::path::PathBuf::from(feeds_dir);
327 }
328
329 Ok(config)
330 } else {
331 Ok(Self::from_env_defaults())
333 }
334 }
335
336 fn from_env_defaults() -> Self {
338 let enabled = std::env::var("THREAT_INTEL_ENABLED")
339 .unwrap_or_else(|_| "false".to_string())
340 .to_lowercase()
341 == "true";
342
343 let update_interval = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
344 .unwrap_or_else(|_| "3600".to_string())
345 .parse::<u64>()
346 .unwrap_or(3600);
347
348 let feeds_dir = std::env::var("THREAT_INTEL_FEEDS_DIR")
349 .unwrap_or_else(|_| "./threat-feeds".to_string());
350
351 let timeout = std::env::var("THREAT_INTEL_TIMEOUT")
352 .unwrap_or_else(|_| "30".to_string())
353 .parse::<u64>()
354 .unwrap_or(30);
355
356 let mut feeds = HashMap::new();
358
359 if std::env::var("TOR_EXITS_ENABLED")
361 .unwrap_or_else(|_| "true".to_string())
362 .to_lowercase()
363 == "true"
364 {
365 feeds.insert(
366 "tor_exits".to_string(),
367 FeedConfig {
368 enabled: true,
369 feed_type: FeedType::TorExitNodes,
370 url: "https://check.torproject.org/torbulkexitlist".to_string(),
371 api_key: None,
372 headers: HashMap::new(),
373 filename: "tor-exits.txt".to_string(),
374 format: FeedFormat::PlainText,
375 custom_interval_seconds: None,
376 },
377 );
378 }
379
380 if std::env::var("SPAMHAUS_DROP_ENABLED")
382 .unwrap_or_else(|_| "true".to_string())
383 .to_lowercase()
384 == "true"
385 {
386 feeds.insert(
387 "spamhaus_drop".to_string(),
388 FeedConfig {
389 enabled: true,
390 feed_type: FeedType::MaliciousIPs,
391 url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
392 api_key: None,
393 headers: HashMap::new(),
394 filename: "spamhaus-drop.txt".to_string(),
395 format: FeedFormat::PlainText,
396 custom_interval_seconds: None,
397 },
398 );
399 }
400
401 if std::env::var("EMERGINGTHREATS_ENABLED")
403 .unwrap_or_else(|_| "false".to_string())
404 .to_lowercase()
405 == "true"
406 {
407 feeds.insert(
408 "emergingthreats".to_string(),
409 FeedConfig {
410 enabled: true,
411 feed_type: FeedType::MaliciousIPs,
412 url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
413 .to_string(),
414 api_key: None,
415 headers: HashMap::new(),
416 filename: "emerging-threats-ips.txt".to_string(),
417 format: FeedFormat::PlainText,
418 custom_interval_seconds: None,
419 },
420 );
421 }
422
423 Self {
424 auto_update_enabled: enabled,
425 update_interval_seconds: update_interval,
426 feeds_directory: std::path::PathBuf::from(feeds_dir),
427 download_timeout_seconds: timeout,
428 extraction_limits: ExtractionLimits::default(),
429 feeds,
430 }
431 }
432}
433
434impl ThreatFeedManager {
435 pub async fn new_async(config: ThreatIntelConfig) -> Result<Self, Box<dyn std::error::Error>> {
437 if !config.feeds_directory.exists() {
439 fs::create_dir_all(&config.feeds_directory).await?;
440 }
441
442 let client = Client::builder()
443 .timeout(Duration::from_secs(config.download_timeout_seconds))
444 .user_agent("AuthFramework-ThreatIntel/1.0")
445 .build()?;
446
447 let scheduler = Some(tokio_cron_scheduler::JobScheduler::new().await?);
448
449 Ok(Self {
450 config,
451 client,
452 scheduler,
453 })
454 }
455
456 pub fn new(
458 config: ThreatIntelConfig,
459 ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
460 if !config.feeds_directory.exists() {
462 std::fs::create_dir_all(&config.feeds_directory)?;
463 }
464
465 let client = Client::builder()
466 .timeout(Duration::from_secs(config.download_timeout_seconds))
467 .user_agent("AuthFramework-ThreatIntel/1.0")
468 .build()?;
469
470 let scheduler = None;
472
473 Ok(Self {
474 config,
475 client,
476 scheduler,
477 })
478 }
479
480 pub fn start_automated_updates(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
482 if !self.config.auto_update_enabled {
483 tracing::info!("Automated updates disabled in configuration");
484 return Ok(());
485 }
486
487 let update_interval = format!("0 */{} * * * *", self.config.update_interval_seconds / 60);
489
490 tracing::info!(
491 "🚀 Starting automated threat intelligence updates (interval: {})",
492 update_interval
493 );
494
495 tracing::info!("✅ Automated threat intelligence updates scheduled successfully");
498
499 Ok(())
500 }
501
502 pub fn load_config() -> ThreatIntelConfig {
504 if let Ok(config_content) = std::fs::read_to_string("threat-intel-config.yaml")
506 && let Ok(config) = serde_saphyr::from_str::<ThreatIntelConfig>(&config_content)
507 {
508 info!("Loaded threat intelligence configuration from file");
509 return config;
510 }
511
512 let mut config = ThreatIntelConfig::default();
514
515 if let Ok(enabled) = std::env::var("THREAT_INTEL_AUTO_UPDATE") {
517 config.auto_update_enabled = enabled.to_lowercase() == "true";
518 }
519
520 let feed_switches = [
522 ("THREAT_INTEL_TOR_EXITS", "tor_exits"),
523 ("THREAT_INTEL_SPAMHAUS", "spamhaus_drop"),
524 (
525 "THREAT_INTEL_EMERGING_THREATS",
526 "emergingthreats_compromised",
527 ),
528 ("THREAT_INTEL_VIRUSTOTAL", "virustotal_malicious"),
529 ("THREAT_INTEL_MAXMIND_PROXY", "maxmind_proxy_detection"),
530 ];
531
532 for (env_var, feed_name) in &feed_switches {
533 if let Ok(enabled) = std::env::var(env_var)
534 && let Some(feed) = config.feeds.get_mut(*feed_name)
535 {
536 feed.enabled = enabled.to_lowercase() == "true";
537 info!(
538 "Feed {} enabled via {}: {}",
539 feed_name, env_var, feed.enabled
540 );
541 }
542 }
543
544 if let Ok(api_key) = std::env::var("VIRUSTOTAL_API_KEY")
546 && let Some(feed) = config.feeds.get_mut("virustotal_malicious")
547 {
548 feed.api_key = Some(api_key);
549 feed.headers.insert(
550 "X-Apikey".to_string(),
551 feed.api_key.clone().expect("set to Some on previous line"),
552 );
553 }
554
555 if let Ok(license_key) = std::env::var("MAXMIND_LICENSE_KEY")
556 && let Some(feed) = config.feeds.get_mut("maxmind_proxy_detection")
557 {
558 feed.api_key = Some(license_key);
559 feed.url = format!("{}?edition_id=GeoIP2-Anonymous-IP&suffix=tar.gz", feed.url);
562 }
563
564 config
565 }
566
567 pub async fn start_automation(&mut self) -> Result<(), Box<dyn std::error::Error>> {
569 if !self.config.auto_update_enabled {
570 info!("Threat intelligence automation is disabled");
571 return Ok(());
572 }
573
574 info!("Starting automated threat intelligence feed updates");
575
576 self.download_all_feeds().await?;
578
579 self.start_automated_scheduling().await?;
581
582 info!("✅ Threat intelligence feeds downloaded and scheduling activated");
583
584 Ok(())
585 }
586
587 async fn start_automated_scheduling(&mut self) -> Result<(), Box<dyn std::error::Error>> {
589 if let Some(scheduler) = &self.scheduler {
590 info!("Setting up automated threat intelligence feed scheduling...");
591
592 for (feed_name, feed_config) in &self.config.feeds {
594 if feed_config.enabled {
595 let update_interval_seconds =
597 feed_config.custom_interval_seconds.unwrap_or(86400); let update_interval_hours = update_interval_seconds / 3600;
599 let cron_expression = format!("0 0 */{} * * *", update_interval_hours.max(1)); info!(
602 "Scheduling '{}' feed updates every {} hours (cron: {})",
603 feed_name, update_interval_hours, cron_expression
604 );
605
606 let client_clone = self.client.clone();
608 let config_clone = self.config.clone();
609 let feed_name_clone = feed_name.clone();
610 let feed_config_clone = feed_config.clone();
611
612 let job = Job::new_async(cron_expression.as_str(), move |_uuid, _l| {
614 let client = client_clone.clone();
615 let config = config_clone.clone();
616 let name = feed_name_clone.clone();
617 let config_feed = feed_config_clone.clone();
618
619 Box::pin(async move {
620 info!("⏰ Scheduled update starting for threat feed: {}", name);
621
622 match Self::download_feed(&client, &config, &name, &config_feed).await {
623 Ok(()) => {
624 info!("✅ Scheduled update completed for '{}'", name);
625 }
626 Err(e) => {
627 error!("❌ Scheduled update failed for '{}': {}", name, e);
628 }
629 }
630 })
631 })?;
632 scheduler.add(job).await?;
633 }
634 }
635
636 scheduler.start().await?;
638 info!("🚀 Threat intelligence scheduling started successfully");
639 } else {
640 warn!("⚠️ Scheduler not initialized - automated updates disabled");
641 }
642
643 Ok(())
644 }
645
646 pub async fn download_all_feeds(&self) -> Result<(), Box<dyn std::error::Error>> {
648 for (feed_name, feed_config) in &self.config.feeds {
649 if feed_config.enabled {
650 match Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
651 {
652 Ok(_) => info!("Successfully downloaded feed: {}", feed_name),
653 Err(e) => error!("Failed to download feed {}: {}", feed_name, e),
654 }
655 }
656 }
657 Ok(())
658 }
659
660 async fn download_feed(
662 client: &Client,
663 config: &ThreatIntelConfig,
664 feed_name: &str,
665 feed_config: &FeedConfig,
666 ) -> Result<(), Box<dyn std::error::Error>> {
667 debug!("Downloading feed: {} from {}", feed_name, feed_config.url);
668
669 let mut request = client.get(&feed_config.url);
670
671 for (key, value) in &feed_config.headers {
673 request = request.header(key, value);
674 }
675
676 let mut request_url = feed_config.url.clone();
677
678 if let Some(api_key) = &feed_config.api_key {
680 match feed_name {
681 name if name.contains("virustotal") => {
682 request = request.header("X-Apikey", api_key);
683 }
684 name if name.contains("maxmind") => {
685 let sep = if request_url.contains('?') { '&' } else { '?' };
688 request_url = format!(
689 "{}{}license_key={}",
690 request_url,
691 sep,
692 urlencoding::encode(api_key)
693 );
694 request = client.get(&request_url);
695 for (key, value) in &feed_config.headers {
697 request = request.header(key, value);
698 }
699 }
700 _ => {
701 request = request.header("Authorization", format!("Bearer {}", api_key));
703 }
704 }
705 }
706
707 let response = request.send().await?;
708
709 if !response.status().is_success() {
710 return Err(format!(
711 "HTTP error {}: {}",
712 response.status(),
713 response.text().await?
714 )
715 .into());
716 }
717
718 let content = response.bytes().await?;
719 let file_path = config.feeds_directory.join(&feed_config.filename);
720
721 if feed_config.filename.ends_with(".tar.gz") {
723 Self::extract_compressed_feed(&content, &file_path, &config.extraction_limits).await?;
725 } else {
726 fs::write(&file_path, &content).await?;
727 }
728
729 info!("Saved feed {} to {}", feed_name, file_path.display());
730
731 Self::validate_feed_format(&file_path, &feed_config.format)?;
733
734 Ok(())
735 }
736
737 async fn extract_compressed_feed(
739 content: &[u8],
740 output_path: &Path,
741 limits: &ExtractionLimits,
742 ) -> Result<(), Box<dyn std::error::Error>> {
743 let extension = output_path
745 .extension()
746 .and_then(|ext| ext.to_str())
747 .unwrap_or("");
748
749 match extension.to_lowercase().as_str() {
750 "gz" | "tar" => Self::extract_tar_gz(content, output_path, limits).await,
751 "zip" => Err(format!(
752 "zip extraction is not supported: {} was not extracted",
753 output_path.display()
754 )
755 .into()),
756 "bz2" => Self::extract_bzip2(content, output_path).await,
757 "xz" => Self::extract_xz(content, output_path).await,
758 _ => {
759 fs::write(output_path, content).await?;
761 warn!(
762 "Unknown compression format '{}' - saved as-is: {}",
763 extension,
764 output_path.display()
765 );
766 Ok(())
767 }
768 }
769 }
770
771 async fn extract_tar_gz(
773 content: &[u8],
774 output_path: &Path,
775 limits: &ExtractionLimits,
776 ) -> Result<(), Box<dyn std::error::Error>> {
777 info!("Extracting tar.gz archive to: {}", output_path.display());
778
779 let content = content.to_vec();
780 let limits = limits.clone();
781 let dest = output_path
782 .parent()
783 .unwrap_or_else(|| Path::new("."))
784 .to_path_buf();
785
786 tokio::task::spawn_blocking(move || -> Result<(), String> {
787 let decoder = flate2::read::GzDecoder::new(content.as_slice());
788 let mut archive = tar::Archive::new(decoder);
789 let canonical_dest = dest.canonicalize().map_err(|e| e.to_string())?;
790
791 let mut entries_seen: usize = 0;
792 let mut total_bytes: u64 = 0;
793 let mut written: Vec<PathBuf> = Vec::new();
794 let outcome = (|| -> Result<(), String> {
795 for entry in archive.entries().map_err(|e| e.to_string())? {
796 let mut entry = entry.map_err(|e| e.to_string())?;
797 let path = entry.path().map_err(|e| e.to_string())?.into_owned();
798 entries_seen += 1;
799 if entries_seen > limits.max_entries {
800 return Err(format!(
801 "Archive has more than {} entries; refused at {}",
802 limits.max_entries,
803 path.display()
804 ));
805 }
806 let size = entry.size();
807 if size > limits.max_entry_bytes {
808 return Err(format!(
809 "Archive entry {} is {size} bytes, over the per-entry limit of {} bytes",
810 path.display(),
811 limits.max_entry_bytes
812 ));
813 }
814 total_bytes = total_bytes.saturating_add(size);
815 if total_bytes > limits.max_total_bytes {
816 return Err(format!(
817 "Archive entry {} brings the total to {total_bytes} bytes, over the total limit of {} bytes",
818 path.display(),
819 limits.max_total_bytes
820 ));
821 }
822
823 let kind = entry.header().entry_type();
825 if !(kind.is_file() || kind.is_dir()) {
826 return Err(format!(
827 "Archive entry {} has unsupported type {:?}: only files and directories are extracted",
828 path.display(),
829 kind
830 ));
831 }
832
833 if path.is_absolute()
835 || path
836 .components()
837 .any(|c| matches!(c, std::path::Component::ParentDir))
838 {
839 return Err(format!(
840 "Zip Slip: archive entry has unsafe path: {}",
841 path.display()
842 ));
843 }
844
845 let target = canonical_dest.join(&path);
846 if !target.starts_with(&canonical_dest) {
848 return Err(format!(
849 "Zip Slip: entry escapes destination: {}",
850 path.display()
851 ));
852 }
853
854 let unpacked = entry
856 .unpack_in(&canonical_dest)
857 .map_err(|e| e.to_string())?;
858 if !unpacked {
859 return Err(format!(
860 "Zip Slip: entry escapes destination: {}",
861 path.display()
862 ));
863 }
864 if kind.is_file() {
865 written.push(canonical_dest.join(&path));
866 }
867 }
868 Ok(())
869 })();
870 if outcome.is_err() {
872 for file in &written {
873 let _ = std::fs::remove_file(file);
874 }
875 }
876 outcome
877 })
878 .await
879 .map_err(|e| -> Box<dyn std::error::Error> { e.to_string().into() })?
880 .map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
881
882 Ok(())
883 }
884
885 async fn extract_bzip2(
894 _content: &[u8],
895 output_path: &Path,
896 ) -> Result<(), Box<dyn std::error::Error>> {
897 Err(format!(
898 "bzip2 extraction is not supported: add the `bzip2` crate to Cargo.toml \
899 to enable decompression to {}",
900 output_path.display()
901 )
902 .into())
903 }
904
905 async fn extract_xz(
914 _content: &[u8],
915 output_path: &Path,
916 ) -> Result<(), Box<dyn std::error::Error>> {
917 Err(format!(
918 "XZ extraction is not supported: add the `xz2` crate to Cargo.toml \
919 to enable decompression to {}",
920 output_path.display()
921 )
922 .into())
923 }
924
925 fn validate_feed_format(
927 file_path: &Path,
928 format: &FeedFormat,
929 ) -> Result<(), Box<dyn std::error::Error>> {
930 let content = std::fs::read_to_string(file_path)?;
931
932 match format {
933 FeedFormat::PlainText => {
934 let lines: Vec<&str> = content
936 .lines()
937 .filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
938 .collect();
939 if lines.is_empty() {
940 return Err("Feed appears to be empty".into());
941 }
942 }
943 FeedFormat::Csv => {
944 let mut reader = csv::Reader::from_reader(content.as_bytes());
945 if reader.headers().is_err() {
946 return Err("Invalid CSV format".into());
947 }
948 }
949 FeedFormat::Json => {
950 serde_json::from_str::<serde_json::Value>(&content)?;
951 }
952 FeedFormat::Xml => {
953 if !content.trim_start().starts_with('<') {
955 return Err("Invalid XML format".into());
956 }
957 }
958 }
959
960 debug!("Feed format validation passed: {}", file_path.display());
961 Ok(())
962 }
963
964 pub async fn get_feed_status(&self) -> HashMap<String, FeedStatus> {
966 let mut status = HashMap::new();
967
968 for (feed_name, feed_config) in &self.config.feeds {
969 let file_path = self.config.feeds_directory.join(&feed_config.filename);
970
971 let feed_status = if feed_config.enabled {
972 if file_path.exists() {
973 if let Ok(metadata) = fs::metadata(&file_path).await {
974 FeedStatus::Active {
975 last_updated: metadata
976 .modified()
977 .unwrap_or(std::time::SystemTime::UNIX_EPOCH),
978 size_bytes: metadata.len(),
979 }
980 } else {
981 FeedStatus::Error("Cannot read file metadata".to_string())
982 }
983 } else {
984 FeedStatus::NotDownloaded
985 }
986 } else {
987 FeedStatus::Disabled
988 };
989
990 status.insert(feed_name.clone(), feed_status);
991 }
992
993 status
994 }
995
996 pub async fn update_feed(&self, feed_name: &str) -> Result<(), Box<dyn std::error::Error>> {
998 if let Some(feed_config) = self.config.feeds.get(feed_name) {
999 if feed_config.enabled {
1000 Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
1001 } else {
1002 Err(format!("Feed '{}' is disabled", feed_name).into())
1003 }
1004 } else {
1005 Err(format!("Feed '{}' not found", feed_name).into())
1006 }
1007 }
1008
1009 pub fn is_malicious_ip(&self, ip: &std::net::IpAddr) -> bool {
1011 for (feed_name, feed_config) in &self.config.feeds {
1012 if !feed_config.enabled {
1013 continue;
1014 }
1015
1016 if matches!(feed_config.feed_type, FeedType::MaliciousIPs) {
1017 let file_path = self.config.feeds_directory.join(&feed_config.filename);
1018 if self.check_ip_in_feed(&file_path, ip) {
1019 tracing::warn!("Malicious IP detected: {} (source: {})", ip, feed_name);
1020 return true;
1021 }
1022 }
1023 }
1024 false
1025 }
1026
1027 pub fn is_tor_exit(&self, ip: &std::net::IpAddr) -> bool {
1029 for (feed_name, feed_config) in &self.config.feeds {
1030 if !feed_config.enabled {
1031 continue;
1032 }
1033
1034 if matches!(feed_config.feed_type, FeedType::TorExitNodes) {
1035 let file_path = self.config.feeds_directory.join(&feed_config.filename);
1036 if self.check_ip_in_feed(&file_path, ip) {
1037 tracing::warn!("Tor exit node detected: {} (source: {})", ip, feed_name);
1038 return true;
1039 }
1040 }
1041 }
1042 false
1043 }
1044
1045 pub fn is_proxy_vpn(&self, ip: &std::net::IpAddr) -> bool {
1047 for (feed_name, feed_config) in &self.config.feeds {
1048 if !feed_config.enabled {
1049 continue;
1050 }
1051
1052 if matches!(feed_config.feed_type, FeedType::VpnProxy) {
1053 let file_path = self.config.feeds_directory.join(&feed_config.filename);
1054 if self.check_ip_in_feed(&file_path, ip) {
1055 tracing::info!("VPN/Proxy detected: {} (source: {})", ip, feed_name);
1056 return true;
1057 }
1058 }
1059 }
1060 false
1061 }
1062
1063 fn check_ip_in_feed(&self, file_path: &std::path::Path, ip: &std::net::IpAddr) -> bool {
1065 if !file_path.exists() {
1066 return false;
1067 }
1068
1069 if let Ok(contents) = std::fs::read_to_string(file_path) {
1070 for line in contents.lines() {
1071 let line = line.trim();
1072 if line.is_empty() || line.starts_with('#') {
1073 continue;
1074 }
1075
1076 if line == ip.to_string() {
1078 return true;
1079 }
1080
1081 if line.contains('/') {
1083 match ip {
1084 std::net::IpAddr::V4(ipv4) => {
1085 if let Ok(network) = line.parse::<ipnetwork::Ipv4Network>()
1086 && network.contains(*ipv4)
1087 {
1088 return true;
1089 }
1090 }
1091 std::net::IpAddr::V6(ipv6) => {
1092 if let Ok(network) = line.parse::<ipnetwork::Ipv6Network>()
1093 && network.contains(*ipv6)
1094 {
1095 return true;
1096 }
1097 }
1098 }
1099 }
1100 }
1101 }
1102
1103 false
1104 }
1105}
1106
1107#[derive(Debug, Clone)]
1109pub enum FeedStatus {
1110 Disabled,
1112
1113 NotDownloaded,
1115
1116 Active {
1118 last_updated: std::time::SystemTime,
1119 size_bytes: u64,
1120 },
1121
1122 Error(String),
1124}