ploit_malper/engine/
dedup.rs

1use std::collections::HashSet;
2
3use regex::Regex;
4use sha2::{Digest, Sha256};
5
6use crate::models::{DedupResult, ScanRecord};
7
8fn normalize_title(title: &str) -> String {
9    let cleaned = title
10        .trim()
11        .to_lowercase()
12        .split_whitespace()
13        .collect::<Vec<&str>>()
14        .join(" ");
15
16    let re = Regex::new(r"\s*[/\\]\s*$").expect("valid regex");
17    re.replace_all(&cleaned, "").to_string()
18}
19
20fn composite_key(record: &ScanRecord) -> String {
21    let normalized = normalize_title(&record.title);
22    let raw = format!("{}|{}|{}", record.target, record.tool, normalized);
23    let mut hasher = Sha256::new();
24    hasher.update(raw.as_bytes());
25    format!("{:x}", hasher.finalize())
26}
27
28pub fn deduplicate_records(records: Vec<ScanRecord>) -> DedupResult {
29    let total_count = records.len();
30    let mut seen: HashSet<String> = HashSet::new();
31    let mut unique_records = Vec::new();
32
33    for record in records {
34        let key = composite_key(&record);
35        if seen.insert(key) {
36            unique_records.push(record);
37        }
38    }
39
40    let unique_count = unique_records.len();
41    let removed_count = total_count.saturating_sub(unique_count);
42
43    DedupResult {
44        unique_count,
45        removed_count,
46        records: unique_records,
47    }
48}
49
50pub fn normalize_title_text(title: &str) -> String {
51    normalize_title(title)
52}
53
54#[cfg(test)]
55mod tests {
56    use super::*;
57    use crate::models::ScanRecord;
58    use serde_json::json;
59
60    #[test]
61    fn deduplicates_on_normalized_title() {
62        let records = vec![
63            ScanRecord {
64                target: "10.0.0.1".into(),
65                tool: "nmap".into(),
66                title: "Apache Version /".into(),
67                severity: "high".into(),
68                port: Some(80),
69                cve: None,
70                service: Some("apache".into()),
71                raw: json!({}),
72                ..Default::default()
73            },
74            ScanRecord {
75                target: "10.0.0.1".into(),
76                tool: "nmap".into(),
77                title: "apache version".into(),
78                severity: "high".into(),
79                port: Some(80),
80                cve: None,
81                service: Some("apache".into()),
82                raw: json!({}),
83                ..Default::default()
84            },
85        ];
86
87        let result = deduplicate_records(records);
88        assert_eq!(result.unique_count, 1);
89        assert_eq!(result.removed_count, 1);
90    }
91}
ploit_malper/engine/dedup.rs

ploit_malper/engine/
dedup.rs
