1use reqwest::Client;
12use serde::{Deserialize, Serialize};
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::time::Duration;
16use tokio::fs;
17use tokio_cron_scheduler::Job;
18use tracing::{debug, error, info, warn};
19
20#[cfg(test)]
21mod extraction_cap_tests;
22#[cfg(test)]
23mod tar_extraction_tests;
24
25#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct ThreatIntelConfig {
28 pub auto_update_enabled: bool,
30
31 pub update_interval_seconds: u64,
33
34 pub feeds_directory: PathBuf,
36
37 pub feeds: HashMap<String, FeedConfig>,
39
40 pub download_timeout_seconds: u64,
42
43 #[serde(default)]
46 pub extraction_limits: ExtractionLimits,
47}
48
49#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
52pub struct ExtractionLimits {
53 pub max_entries: usize,
55
56 pub max_entry_bytes: u64,
58
59 pub max_total_bytes: u64,
61}
62
63impl Default for ExtractionLimits {
64 fn default() -> Self {
66 Self {
67 max_entries: 10_000,
68 max_entry_bytes: 512 << 20,
69 max_total_bytes: 1 << 30,
70 }
71 }
72}
73
74#[derive(Clone, Serialize, Deserialize)]
76pub struct FeedConfig {
77 pub enabled: bool,
79
80 pub feed_type: FeedType,
82
83 pub url: String,
85
86 pub api_key: Option<String>,
88
89 pub headers: HashMap<String, String>,
91
92 pub filename: String,
94
95 pub format: FeedFormat,
97
98 pub custom_interval_seconds: Option<u64>,
100}
101
102impl std::fmt::Debug for FeedConfig {
103 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104 let mut d = f.debug_struct("FeedConfig");
105 d.field("enabled", &self.enabled);
106 d.field("feed_type", &self.feed_type);
107 d.field("url", &crate::utils::redact::redacted(&self.url));
108 d.field("api_key", &crate::utils::redact::redacted(&self.api_key));
109 d.field("headers", &crate::utils::redact::redacted(&self.headers));
110 d.field("filename", &self.filename);
111 d.field("format", &self.format);
112 d.field("custom_interval_seconds", &self.custom_interval_seconds);
113 d.finish()
114 }
115}
116
117#[derive(Debug, Clone, Serialize, Deserialize)]
119pub enum FeedType {
120 MaliciousIPs,
122
123 TorExitNodes,
125
126 VpnProxy,
128
129 BotnetC2,
131
132 CountryThreats,
134
135 HostingProviders,
137
138 DatacenterRanges,
140
141 Custom(String),
143}
144
145#[derive(Debug, Clone, Serialize, Deserialize)]
147pub enum FeedFormat {
148 PlainText,
150
151 Csv,
153
154 Json,
156
157 Xml,
159}
160
161pub struct ThreatFeedManager {
163 config: ThreatIntelConfig,
164 client: Client,
165 scheduler: Option<tokio_cron_scheduler::JobScheduler>,
166}
167
168#[cfg(test)]
169mod feed_dispatch_tests;
170
171impl Default for ThreatIntelConfig {
172 fn default() -> Self {
173 let mut feeds = HashMap::new();
174
175 feeds.insert(
177 "tor_exits".to_string(),
178 FeedConfig {
179 enabled: false, feed_type: FeedType::TorExitNodes,
181 url: "https://check.torproject.org/torbulkexitlist".to_string(),
182 api_key: None,
183 headers: HashMap::new(),
184 filename: "tor-exits.txt".to_string(),
185 format: FeedFormat::PlainText,
186 custom_interval_seconds: Some(3600), },
188 );
189
190 feeds.insert(
191 "spamhaus_drop".to_string(),
192 FeedConfig {
193 enabled: false, feed_type: FeedType::MaliciousIPs,
195 url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
196 api_key: None,
197 headers: HashMap::new(),
198 filename: "spamhaus-drop.txt".to_string(),
199 format: FeedFormat::PlainText,
200 custom_interval_seconds: Some(3600),
201 },
202 );
203
204 feeds.insert(
205 "emergingthreats_compromised".to_string(),
206 FeedConfig {
207 enabled: false, feed_type: FeedType::MaliciousIPs,
209 url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt".to_string(),
210 api_key: None,
211 headers: HashMap::new(),
212 filename: "emerging-threats-ips.txt".to_string(),
213 format: FeedFormat::PlainText,
214 custom_interval_seconds: Some(7200), },
216 );
217
218 feeds.insert(
220 "virustotal_malicious".to_string(),
221 FeedConfig {
222 enabled: false, feed_type: FeedType::MaliciousIPs,
224 url: "https://www.virustotal.com/api/v3/intelligence/hunting_notification_files"
225 .to_string(),
226 api_key: None, headers: HashMap::new(),
228 filename: "virustotal-malicious.json".to_string(),
229 format: FeedFormat::Json,
230 custom_interval_seconds: Some(1800),
231 },
232 );
233
234 feeds.insert(
235 "maxmind_proxy_detection".to_string(),
236 FeedConfig {
237 enabled: false, feed_type: FeedType::VpnProxy,
239 url: "https://download.maxmind.com/app/geoip_download".to_string(),
240 api_key: None, headers: HashMap::new(),
242 filename: "maxmind-proxy-ranges.csv".to_string(),
243 format: FeedFormat::Csv,
244 custom_interval_seconds: Some(86400), },
246 );
247
248 Self {
249 auto_update_enabled: false, update_interval_seconds: 3600, feeds_directory: PathBuf::from("threat-feeds"),
252 feeds,
253 download_timeout_seconds: 30,
254 extraction_limits: ExtractionLimits::default(),
255 }
256 }
257}
258
259impl ThreatIntelConfig {
260 pub fn disabled() -> Self {
270 Self {
271 auto_update_enabled: false,
272 update_interval_seconds: 0,
273 feeds_directory: PathBuf::from("./threat-feeds"),
274 feeds: HashMap::new(),
275 download_timeout_seconds: 30,
276 extraction_limits: ExtractionLimits::default(),
277 }
278 }
279
280 pub fn aggressive() -> Self {
290 let mut base = Self {
291 auto_update_enabled: true,
292 update_interval_seconds: 300, download_timeout_seconds: 10,
294 extraction_limits: ExtractionLimits::default(),
295 ..Default::default()
296 };
297 for feed in base.feeds.values_mut() {
299 feed.enabled = true;
300 }
301 base
302 }
303
304 pub fn from_env_and_config() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
306 let config_path = std::env::var("THREAT_INTEL_CONFIG_PATH")
308 .unwrap_or_else(|_| "threat-intel-config.yaml".to_string());
309
310 if std::path::Path::new(&config_path).exists() {
311 let config_content = std::fs::read_to_string(&config_path)?;
312 let mut config: Self = serde_saphyr::from_str(&config_content)?;
313
314 if let Ok(enabled) = std::env::var("THREAT_INTEL_ENABLED") {
316 config.auto_update_enabled = enabled.to_lowercase() == "true";
317 }
318
319 if let Ok(interval) = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
320 && let Ok(seconds) = interval.parse::<u64>()
321 {
322 config.update_interval_seconds = seconds;
323 }
324
325 if let Ok(feeds_dir) = std::env::var("THREAT_INTEL_FEEDS_DIR") {
326 config.feeds_directory = std::path::PathBuf::from(feeds_dir);
327 }
328
329 Ok(config)
330 } else {
331 Ok(Self::from_env_defaults())
333 }
334 }
335
336 fn from_env_defaults() -> Self {
338 let enabled = std::env::var("THREAT_INTEL_ENABLED")
339 .unwrap_or_else(|_| "false".to_string())
340 .to_lowercase()
341 == "true";
342
343 let update_interval = std::env::var("THREAT_INTEL_UPDATE_INTERVAL")
344 .unwrap_or_else(|_| "3600".to_string())
345 .parse::<u64>()
346 .unwrap_or(3600);
347
348 let feeds_dir = std::env::var("THREAT_INTEL_FEEDS_DIR")
349 .unwrap_or_else(|_| "./threat-feeds".to_string());
350
351 let timeout = std::env::var("THREAT_INTEL_TIMEOUT")
352 .unwrap_or_else(|_| "30".to_string())
353 .parse::<u64>()
354 .unwrap_or(30);
355
356 let mut feeds = HashMap::new();
358
359 if std::env::var("TOR_EXITS_ENABLED")
361 .unwrap_or_else(|_| "true".to_string())
362 .to_lowercase()
363 == "true"
364 {
365 feeds.insert(
366 "tor_exits".to_string(),
367 FeedConfig {
368 enabled: true,
369 feed_type: FeedType::TorExitNodes,
370 url: "https://check.torproject.org/torbulkexitlist".to_string(),
371 api_key: None,
372 headers: HashMap::new(),
373 filename: "tor-exits.txt".to_string(),
374 format: FeedFormat::PlainText,
375 custom_interval_seconds: None,
376 },
377 );
378 }
379
380 if std::env::var("SPAMHAUS_DROP_ENABLED")
382 .unwrap_or_else(|_| "true".to_string())
383 .to_lowercase()
384 == "true"
385 {
386 feeds.insert(
387 "spamhaus_drop".to_string(),
388 FeedConfig {
389 enabled: true,
390 feed_type: FeedType::MaliciousIPs,
391 url: "https://www.spamhaus.org/drop/drop.txt".to_string(),
392 api_key: None,
393 headers: HashMap::new(),
394 filename: "spamhaus-drop.txt".to_string(),
395 format: FeedFormat::PlainText,
396 custom_interval_seconds: None,
397 },
398 );
399 }
400
401 if std::env::var("EMERGINGTHREATS_ENABLED")
403 .unwrap_or_else(|_| "false".to_string())
404 .to_lowercase()
405 == "true"
406 {
407 feeds.insert(
408 "emergingthreats".to_string(),
409 FeedConfig {
410 enabled: true,
411 feed_type: FeedType::MaliciousIPs,
412 url: "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
413 .to_string(),
414 api_key: None,
415 headers: HashMap::new(),
416 filename: "emerging-threats-ips.txt".to_string(),
417 format: FeedFormat::PlainText,
418 custom_interval_seconds: None,
419 },
420 );
421 }
422
423 Self {
424 auto_update_enabled: enabled,
425 update_interval_seconds: update_interval,
426 feeds_directory: std::path::PathBuf::from(feeds_dir),
427 download_timeout_seconds: timeout,
428 extraction_limits: ExtractionLimits::default(),
429 feeds,
430 }
431 }
432}
433
434impl ThreatFeedManager {
435 pub async fn new_async(config: ThreatIntelConfig) -> Result<Self, Box<dyn std::error::Error>> {
437 if !config.feeds_directory.exists() {
439 fs::create_dir_all(&config.feeds_directory).await?;
440 }
441
442 let client = Client::builder()
443 .timeout(Duration::from_secs(config.download_timeout_seconds))
444 .user_agent("AuthFramework-ThreatIntel/1.0")
445 .build()?;
446
447 let scheduler = Some(tokio_cron_scheduler::JobScheduler::new().await?);
448
449 Ok(Self {
450 config,
451 client,
452 scheduler,
453 })
454 }
455
456 pub fn new(
458 config: ThreatIntelConfig,
459 ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
460 if !config.feeds_directory.exists() {
462 std::fs::create_dir_all(&config.feeds_directory)?;
463 }
464
465 let client = Client::builder()
466 .timeout(Duration::from_secs(config.download_timeout_seconds))
467 .user_agent("AuthFramework-ThreatIntel/1.0")
468 .build()?;
469
470 let scheduler = None;
472
473 Ok(Self {
474 config,
475 client,
476 scheduler,
477 })
478 }
479
480 pub fn start_automated_updates(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
482 if !self.config.auto_update_enabled {
483 tracing::info!("Automated updates disabled in configuration");
484 return Ok(());
485 }
486
487 let update_interval = format!("0 */{} * * * *", self.config.update_interval_seconds / 60);
489
490 tracing::info!(
491 "🚀 Starting automated threat intelligence updates (interval: {})",
492 update_interval
493 );
494
495 tracing::info!("✅ Automated threat intelligence updates scheduled successfully");
498
499 Ok(())
500 }
501
502 pub fn load_config() -> ThreatIntelConfig {
504 if let Ok(config_content) = std::fs::read_to_string("threat-intel-config.yaml")
506 && let Ok(config) = serde_saphyr::from_str::<ThreatIntelConfig>(&config_content)
507 {
508 info!("Loaded threat intelligence configuration from file");
509 return config;
510 }
511
512 let mut config = ThreatIntelConfig::default();
514
515 if let Ok(enabled) = std::env::var("THREAT_INTEL_AUTO_UPDATE") {
517 config.auto_update_enabled = enabled.to_lowercase() == "true";
518 }
519
520 let feed_switches = [
522 ("THREAT_INTEL_TOR_EXITS", "tor_exits"),
523 ("THREAT_INTEL_SPAMHAUS", "spamhaus_drop"),
524 (
525 "THREAT_INTEL_EMERGING_THREATS",
526 "emergingthreats_compromised",
527 ),
528 ("THREAT_INTEL_VIRUSTOTAL", "virustotal_malicious"),
529 ("THREAT_INTEL_MAXMIND_PROXY", "maxmind_proxy_detection"),
530 ];
531
532 for (env_var, feed_name) in &feed_switches {
533 if let Ok(enabled) = std::env::var(env_var)
534 && let Some(feed) = config.feeds.get_mut(*feed_name)
535 {
536 feed.enabled = enabled.to_lowercase() == "true";
537 info!(
538 "Feed {} enabled via {}: {}",
539 feed_name, env_var, feed.enabled
540 );
541 }
542 }
543
544 if let Ok(api_key) = std::env::var("VIRUSTOTAL_API_KEY")
546 && let Some(feed) = config.feeds.get_mut("virustotal_malicious")
547 {
548 feed.api_key = Some(api_key);
549 feed.headers.insert(
550 "X-Apikey".to_string(),
551 feed.api_key.clone().expect("set to Some on previous line"),
552 );
553 }
554
555 if let Ok(license_key) = std::env::var("MAXMIND_LICENSE_KEY")
556 && let Some(feed) = config.feeds.get_mut("maxmind_proxy_detection")
557 {
558 feed.api_key = Some(license_key);
559 feed.url = format!("{}?edition_id=GeoIP2-Anonymous-IP&suffix=tar.gz", feed.url);
562 }
563
564 config
565 }
566
567 pub async fn start_automation(&mut self) -> Result<(), Box<dyn std::error::Error>> {
569 if !self.config.auto_update_enabled {
570 info!("Threat intelligence automation is disabled");
571 return Ok(());
572 }
573
574 info!("Starting automated threat intelligence feed updates");
575
576 self.download_all_feeds().await?;
578
579 self.start_automated_scheduling().await?;
581
582 info!("✅ Threat intelligence feeds downloaded and scheduling activated");
583
584 Ok(())
585 }
586
587 async fn start_automated_scheduling(&mut self) -> Result<(), Box<dyn std::error::Error>> {
589 if let Some(scheduler) = &self.scheduler {
590 info!("Setting up automated threat intelligence feed scheduling...");
591
592 for (feed_name, feed_config) in &self.config.feeds {
594 if feed_config.enabled {
595 let update_interval_seconds =
597 feed_config.custom_interval_seconds.unwrap_or(86400); let update_interval_hours = update_interval_seconds / 3600;
599 let cron_expression = format!("0 0 */{} * * *", update_interval_hours.max(1)); info!(
602 "Scheduling '{}' feed updates every {} hours (cron: {})",
603 feed_name, update_interval_hours, cron_expression
604 );
605
606 let client_clone = self.client.clone();
608 let config_clone = self.config.clone();
609 let feed_name_clone = feed_name.clone();
610 let feed_config_clone = feed_config.clone();
611
612 let job = Job::new_async(cron_expression.as_str(), move |_uuid, _l| {
614 let client = client_clone.clone();
615 let config = config_clone.clone();
616 let name = feed_name_clone.clone();
617 let config_feed = feed_config_clone.clone();
618
619 Box::pin(async move {
620 info!("⏰ Scheduled update starting for threat feed: {}", name);
621
622 match Self::download_feed(&client, &config, &name, &config_feed).await {
623 Ok(()) => {
624 info!("✅ Scheduled update completed for '{}'", name);
625 }
626 Err(e) => {
627 error!("❌ Scheduled update failed for '{}': {}", name, e);
628 }
629 }
630 })
631 })?;
632 scheduler.add(job).await?;
633 }
634 }
635
636 scheduler.start().await?;
638 info!("🚀 Threat intelligence scheduling started successfully");
639 } else {
640 warn!("⚠️ Scheduler not initialized - automated updates disabled");
641 }
642
643 Ok(())
644 }
645
646 pub async fn download_all_feeds(&self) -> Result<(), Box<dyn std::error::Error>> {
648 for (feed_name, feed_config) in &self.config.feeds {
649 if feed_config.enabled {
650 match Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
651 {
652 Ok(_) => info!("Successfully downloaded feed: {}", feed_name),
653 Err(e) => error!("Failed to download feed {}: {}", feed_name, e),
654 }
655 }
656 }
657 Ok(())
658 }
659
660 async fn download_feed(
662 client: &Client,
663 config: &ThreatIntelConfig,
664 feed_name: &str,
665 feed_config: &FeedConfig,
666 ) -> Result<(), Box<dyn std::error::Error>> {
667 debug!("Downloading feed: {} from {}", feed_name, feed_config.url);
668
669 let mut request = client.get(&feed_config.url);
670
671 for (key, value) in &feed_config.headers {
673 request = request.header(key, value);
674 }
675
676 let mut request_url = feed_config.url.clone();
677
678 if let Some(api_key) = &feed_config.api_key {
680 match feed_name {
681 name if name.contains("virustotal") => {
682 request = request.header("X-Apikey", api_key);
683 }
684 name if name.contains("maxmind") => {
685 let sep = if request_url.contains('?') { '&' } else { '?' };
688 request_url = format!(
689 "{}{}license_key={}",
690 request_url,
691 sep,
692 urlencoding::encode(api_key)
693 );
694 request = client.get(&request_url);
695 for (key, value) in &feed_config.headers {
697 request = request.header(key, value);
698 }
699 }
700 _ => {
701 request = request.header("Authorization", format!("Bearer {}", api_key));
703 }
704 }
705 }
706
707 let response = request.send().await.map_err(reqwest::Error::without_url)?;
708
709 if !response.status().is_success() {
710 return Err(format!(
711 "HTTP error {}: {}",
712 response.status(),
713 response.text().await.map_err(reqwest::Error::without_url)?
714 )
715 .into());
716 }
717
718 let content = response
719 .bytes()
720 .await
721 .map_err(reqwest::Error::without_url)?;
722 let file_path = config.feeds_directory.join(&feed_config.filename);
723
724 if feed_config.filename.ends_with(".tar.gz") {
726 Self::extract_compressed_feed(&content, &file_path, &config.extraction_limits).await?;
728 } else {
729 fs::write(&file_path, &content).await?;
730 }
731
732 info!("Saved feed {} to {}", feed_name, file_path.display());
733
734 Self::validate_feed_format(&file_path, &feed_config.format)?;
736
737 Ok(())
738 }
739
740 async fn extract_compressed_feed(
742 content: &[u8],
743 output_path: &Path,
744 limits: &ExtractionLimits,
745 ) -> Result<(), Box<dyn std::error::Error>> {
746 let extension = output_path
748 .extension()
749 .and_then(|ext| ext.to_str())
750 .unwrap_or("");
751
752 match extension.to_lowercase().as_str() {
753 "gz" | "tar" => Self::extract_tar_gz(content, output_path, limits).await,
754 "zip" => Err(format!(
755 "zip extraction is not supported: {} was not extracted",
756 output_path.display()
757 )
758 .into()),
759 "bz2" => Self::extract_bzip2(content, output_path).await,
760 "xz" => Self::extract_xz(content, output_path).await,
761 _ => {
762 fs::write(output_path, content).await?;
764 warn!(
765 "Unknown compression format '{}' - saved as-is: {}",
766 extension,
767 output_path.display()
768 );
769 Ok(())
770 }
771 }
772 }
773
774 async fn extract_tar_gz(
776 content: &[u8],
777 output_path: &Path,
778 limits: &ExtractionLimits,
779 ) -> Result<(), Box<dyn std::error::Error>> {
780 info!("Extracting tar.gz archive to: {}", output_path.display());
781
782 let content = content.to_vec();
783 let limits = limits.clone();
784 let dest = output_path
785 .parent()
786 .unwrap_or_else(|| Path::new("."))
787 .to_path_buf();
788
789 tokio::task::spawn_blocking(move || -> Result<(), String> {
790 let decoder = flate2::read::GzDecoder::new(content.as_slice());
791 let mut archive = tar::Archive::new(decoder);
792 let canonical_dest = dest.canonicalize().map_err(|e| e.to_string())?;
793
794 let mut entries_seen: usize = 0;
795 let mut total_bytes: u64 = 0;
796 let mut written: Vec<PathBuf> = Vec::new();
797 let outcome = (|| -> Result<(), String> {
798 for entry in archive.entries().map_err(|e| e.to_string())? {
799 let mut entry = entry.map_err(|e| e.to_string())?;
800 let path = entry.path().map_err(|e| e.to_string())?.into_owned();
801 entries_seen += 1;
802 if entries_seen > limits.max_entries {
803 return Err(format!(
804 "Archive has more than {} entries; refused at {}",
805 limits.max_entries,
806 path.display()
807 ));
808 }
809 let size = entry.size();
810 if size > limits.max_entry_bytes {
811 return Err(format!(
812 "Archive entry {} is {size} bytes, over the per-entry limit of {} bytes",
813 path.display(),
814 limits.max_entry_bytes
815 ));
816 }
817 total_bytes = total_bytes.saturating_add(size);
818 if total_bytes > limits.max_total_bytes {
819 return Err(format!(
820 "Archive entry {} brings the total to {total_bytes} bytes, over the total limit of {} bytes",
821 path.display(),
822 limits.max_total_bytes
823 ));
824 }
825
826 let kind = entry.header().entry_type();
828 if !(kind.is_file() || kind.is_dir()) {
829 return Err(format!(
830 "Archive entry {} has unsupported type {:?}: only files and directories are extracted",
831 path.display(),
832 kind
833 ));
834 }
835
836 if path.is_absolute()
838 || path
839 .components()
840 .any(|c| matches!(c, std::path::Component::ParentDir))
841 {
842 return Err(format!(
843 "Zip Slip: archive entry has unsafe path: {}",
844 path.display()
845 ));
846 }
847
848 let target = canonical_dest.join(&path);
849 if !target.starts_with(&canonical_dest) {
851 return Err(format!(
852 "Zip Slip: entry escapes destination: {}",
853 path.display()
854 ));
855 }
856
857 let unpacked = entry
859 .unpack_in(&canonical_dest)
860 .map_err(|e| e.to_string())?;
861 if !unpacked {
862 return Err(format!(
863 "Zip Slip: entry escapes destination: {}",
864 path.display()
865 ));
866 }
867 if kind.is_file() {
868 written.push(canonical_dest.join(&path));
869 }
870 }
871 Ok(())
872 })();
873 if outcome.is_err() {
875 for file in &written {
876 let _ = std::fs::remove_file(file);
877 }
878 }
879 outcome
880 })
881 .await
882 .map_err(|e| -> Box<dyn std::error::Error> { e.to_string().into() })?
883 .map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
884
885 Ok(())
886 }
887
888 async fn extract_bzip2(
897 _content: &[u8],
898 output_path: &Path,
899 ) -> Result<(), Box<dyn std::error::Error>> {
900 Err(format!(
901 "bzip2 extraction is not supported: add the `bzip2` crate to Cargo.toml \
902 to enable decompression to {}",
903 output_path.display()
904 )
905 .into())
906 }
907
908 async fn extract_xz(
917 _content: &[u8],
918 output_path: &Path,
919 ) -> Result<(), Box<dyn std::error::Error>> {
920 Err(format!(
921 "XZ extraction is not supported: add the `xz2` crate to Cargo.toml \
922 to enable decompression to {}",
923 output_path.display()
924 )
925 .into())
926 }
927
928 fn validate_feed_format(
930 file_path: &Path,
931 format: &FeedFormat,
932 ) -> Result<(), Box<dyn std::error::Error>> {
933 let content = std::fs::read_to_string(file_path)?;
934
935 match format {
936 FeedFormat::PlainText => {
937 let lines: Vec<&str> = content
939 .lines()
940 .filter(|l| !l.trim().is_empty() && !l.starts_with('#'))
941 .collect();
942 if lines.is_empty() {
943 return Err("Feed appears to be empty".into());
944 }
945 }
946 FeedFormat::Csv => {
947 let mut reader = csv::Reader::from_reader(content.as_bytes());
948 if reader.headers().is_err() {
949 return Err("Invalid CSV format".into());
950 }
951 }
952 FeedFormat::Json => {
953 serde_json::from_str::<serde_json::Value>(&content)?;
954 }
955 FeedFormat::Xml => {
956 if !content.trim_start().starts_with('<') {
958 return Err("Invalid XML format".into());
959 }
960 }
961 }
962
963 debug!("Feed format validation passed: {}", file_path.display());
964 Ok(())
965 }
966
967 pub async fn get_feed_status(&self) -> HashMap<String, FeedStatus> {
969 let mut status = HashMap::new();
970
971 for (feed_name, feed_config) in &self.config.feeds {
972 let file_path = self.config.feeds_directory.join(&feed_config.filename);
973
974 let feed_status = if feed_config.enabled {
975 if file_path.exists() {
976 if let Ok(metadata) = fs::metadata(&file_path).await {
977 FeedStatus::Active {
978 last_updated: metadata
979 .modified()
980 .unwrap_or(std::time::SystemTime::UNIX_EPOCH),
981 size_bytes: metadata.len(),
982 }
983 } else {
984 FeedStatus::Error("Cannot read file metadata".to_string())
985 }
986 } else {
987 FeedStatus::NotDownloaded
988 }
989 } else {
990 FeedStatus::Disabled
991 };
992
993 status.insert(feed_name.clone(), feed_status);
994 }
995
996 status
997 }
998
999 pub async fn update_feed(&self, feed_name: &str) -> Result<(), Box<dyn std::error::Error>> {
1001 if let Some(feed_config) = self.config.feeds.get(feed_name) {
1002 if feed_config.enabled {
1003 Self::download_feed(&self.client, &self.config, feed_name, feed_config).await
1004 } else {
1005 Err(format!("Feed '{}' is disabled", feed_name).into())
1006 }
1007 } else {
1008 Err(format!("Feed '{}' not found", feed_name).into())
1009 }
1010 }
1011
1012 pub fn is_malicious_ip(&self, ip: &std::net::IpAddr) -> bool {
1014 for (feed_name, feed_config) in &self.config.feeds {
1015 if !feed_config.enabled {
1016 continue;
1017 }
1018
1019 if matches!(feed_config.feed_type, FeedType::MaliciousIPs) {
1020 let file_path = self.config.feeds_directory.join(&feed_config.filename);
1021 if self.check_ip_in_feed(&file_path, ip) {
1022 tracing::warn!("Malicious IP detected: {} (source: {})", ip, feed_name);
1023 return true;
1024 }
1025 }
1026 }
1027 false
1028 }
1029
1030 pub fn is_tor_exit(&self, ip: &std::net::IpAddr) -> bool {
1032 for (feed_name, feed_config) in &self.config.feeds {
1033 if !feed_config.enabled {
1034 continue;
1035 }
1036
1037 if matches!(feed_config.feed_type, FeedType::TorExitNodes) {
1038 let file_path = self.config.feeds_directory.join(&feed_config.filename);
1039 if self.check_ip_in_feed(&file_path, ip) {
1040 tracing::warn!("Tor exit node detected: {} (source: {})", ip, feed_name);
1041 return true;
1042 }
1043 }
1044 }
1045 false
1046 }
1047
1048 pub fn is_proxy_vpn(&self, ip: &std::net::IpAddr) -> bool {
1050 for (feed_name, feed_config) in &self.config.feeds {
1051 if !feed_config.enabled {
1052 continue;
1053 }
1054
1055 if matches!(feed_config.feed_type, FeedType::VpnProxy) {
1056 let file_path = self.config.feeds_directory.join(&feed_config.filename);
1057 if self.check_ip_in_feed(&file_path, ip) {
1058 tracing::info!("VPN/Proxy detected: {} (source: {})", ip, feed_name);
1059 return true;
1060 }
1061 }
1062 }
1063 false
1064 }
1065
1066 fn check_ip_in_feed(&self, file_path: &std::path::Path, ip: &std::net::IpAddr) -> bool {
1068 if !file_path.exists() {
1069 return false;
1070 }
1071
1072 if let Ok(contents) = std::fs::read_to_string(file_path) {
1073 for line in contents.lines() {
1074 let line = line.trim();
1075 if line.is_empty() || line.starts_with('#') {
1076 continue;
1077 }
1078
1079 if line == ip.to_string() {
1081 return true;
1082 }
1083
1084 if line.contains('/') {
1086 match ip {
1087 std::net::IpAddr::V4(ipv4) => {
1088 if let Ok(network) = line.parse::<ipnetwork::Ipv4Network>()
1089 && network.contains(*ipv4)
1090 {
1091 return true;
1092 }
1093 }
1094 std::net::IpAddr::V6(ipv6) => {
1095 if let Ok(network) = line.parse::<ipnetwork::Ipv6Network>()
1096 && network.contains(*ipv6)
1097 {
1098 return true;
1099 }
1100 }
1101 }
1102 }
1103 }
1104 }
1105
1106 false
1107 }
1108}
1109
1110#[derive(Debug, Clone)]
1112pub enum FeedStatus {
1113 Disabled,
1115
1116 NotDownloaded,
1118
1119 Active {
1121 last_updated: std::time::SystemTime,
1122 size_bytes: u64,
1123 },
1124
1125 Error(String),
1127}