ploit_malper/engine/
dedup.rs

1use std::collections::HashMap;
2
3use regex::Regex;
4use sha2::{Digest, Sha256};
5
6use crate::analyze::extract_endpoint;
7use crate::models::{DedupResult, ScanRecord};
8
9/// Normalize a finding title for deduplication:
10///
11/// * strip leading HTTP method prefixes (`GET`, `POST`, ...)
12/// * normalize endpoint prefixes so `"/:"`, `"GET /:"` and `"/"` collapse
13/// * remove trailing and duplicated punctuation (e.g. stray `:` / `.`)
14/// * strip trailing reference URLs (they are truncated at arbitrary lengths)
15/// * collapse whitespace
16/// * lowercase (case-insensitive comparison)
17///
18/// The raw title is preserved on the record (`raw_findings`) for reporting.
19pub fn normalize_title(title: &str) -> String {
20    let mut text = title.trim().to_lowercase();
21
22    // Strip leading HTTP verbs (GET/POST/HEAD/...).
23    text = Regex::new(r"^\s*(get|post|put|head|delete|patch|options|trace|connect)\s+")
24        .expect("valid method regex")
25        .replace(&text, "")
26        .to_string();
27
28    // Normalize leading endpoint markers ("/:", "/", "GET /:" → no prefix).
29    text = text.trim_start_matches('/').trim_start().to_string();
30    text = text.trim_start_matches(':').trim_start().to_string();
31
32    // Drop trailing reference material: "See: <url>" (which scanners truncate
33    // at arbitrary lengths) or "See: <CVE>" noise. A bare trailing CVE id is
34    // preserved because it is the finding itself.
35    text =
36        Regex::new(r"\s*(?:(?:see\s*:?\s*)?https?://\S+|\bsee\s*:?\s*cve-\d{4}-\d{3,7})\s*:?\s*$")
37            .expect("valid reference regex")
38            .replace(&text, "")
39            .to_string();
40
41    // Collapse whitespace.
42    text = text.split_whitespace().collect::<Vec<_>>().join(" ");
43
44    // Strip trailing punctuation and path separators, then re-trim.
45    text = text
46        .trim_end_matches(|c: char| ".,:;\\/".contains(c))
47        .trim()
48        .to_string();
49
50    text
51}
52
53fn composite_key(record: &ScanRecord) -> String {
54    let target = normalize_target(&record.target);
55    let normalized = normalize_title(&record.title);
56    let raw = format!("{}|{}", target, normalized);
57    let mut hasher = Sha256::new();
58    hasher.update(raw.as_bytes());
59    format!("{:x}", hasher.finalize())
60}
61
62/// Normalize the target for comparison so `http://h/` and `http://h` merge.
63fn normalize_target(target: &str) -> String {
64    target.trim().trim_end_matches('/').to_lowercase()
65}
66
67pub fn deduplicate_records(records: Vec<ScanRecord>) -> DedupResult {
68    let total_count = records.len();
69    let mut seen: HashMap<String, usize> = HashMap::new();
70    let mut unique_records = Vec::new();
71
72    for mut record in records {
73        let key = composite_key(&record);
74        record.normalized_title = normalize_title(&record.title);
75        record.merged_count = 1;
76        record.raw_findings = vec![record.title.clone()];
77
78        if let Some(&existing_idx) = seen.get(&key) {
79            merge_record(&mut unique_records[existing_idx], &record);
80        } else {
81            if record.affected_endpoints.is_empty() {
82                record.affected_endpoints = extract_endpoint(&record).into_iter().collect();
83            }
84            seen.insert(key, unique_records.len());
85            unique_records.push(record);
86        }
87    }
88
89    let unique_count = unique_records.len();
90    let removed_count = total_count.saturating_sub(unique_count);
91
92    DedupResult {
93        unique_count,
94        removed_count,
95        records: unique_records,
96    }
97}
98
99/// Merge a duplicate into the kept record, preserving every piece of
100/// scanner evidence and aggregating endpoints, CVEs, tools and count.
101fn merge_record(kept: &mut ScanRecord, duplicate: &ScanRecord) {
102    kept.merged_count += 1;
103
104    if let Some(endpoint) = extract_endpoint(duplicate) {
105        if !kept.affected_endpoints.contains(&endpoint) {
106            kept.affected_endpoints.push(endpoint);
107        }
108    }
109    for cve in &duplicate.cves {
110        if !kept.cves.contains(cve) {
111            kept.cves.push(cve.clone());
112        }
113    }
114    if kept.cve.is_none() {
115        kept.cve = duplicate.cve.clone();
116    }
117    if kept.service.is_none() {
118        kept.service = duplicate.service.clone();
119    }
120    if kept.port.is_none() {
121        kept.port = duplicate.port;
122    }
123    if kept.detail.is_empty() {
124        kept.detail = duplicate.detail.clone();
125    }
126    if kept.reference.is_empty() {
127        kept.reference = duplicate.reference.clone();
128    }
129    for raw in &duplicate.raw_findings {
130        if !kept.raw_findings.contains(raw) {
131            kept.raw_findings.push(raw.clone());
132        }
133    }
134
135    // Keep the most severe classification across merged records.
136    if crate::analyze::severity_rank(&duplicate.severity)
137        < crate::analyze::severity_rank(&kept.severity)
138    {
139        kept.severity = duplicate.severity.clone();
140    }
141}
142
143pub fn normalize_title_text(title: &str) -> String {
144    normalize_title(title)
145}
146
147#[cfg(test)]
148mod tests {
149    use super::*;
150    use crate::models::ScanRecord;
151    use serde_json::json;
152
153    fn record(title: &str, target: &str) -> ScanRecord {
154        ScanRecord {
155            target: target.into(),
156            tool: "nikto".into(),
157            title: title.into(),
158            severity: "low".into(),
159            raw: json!({}),
160            ..Default::default()
161        }
162    }
163
164    #[test]
165    fn strips_http_method_and_endpoint_prefix() {
166        assert_eq!(
167            normalize_title("/: Suggested security header missing: content-security-policy"),
168            normalize_title("GET /: Suggested security header missing: content-security-policy")
169        );
170    }
171
172    #[test]
173    fn removes_trailing_punctuation_and_truncated_urls() {
174        let with_url = normalize_title(
175            "GET /: Suggested security header missing: content-security-policy. See: https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP:",
176        );
177        let truncated_url = normalize_title(
178            "/: Suggested security header missing: content-security-policy. See: https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP",
179        );
180        assert_eq!(with_url, truncated_url);
181        assert!(!with_url.contains("See:"));
182        assert!(!with_url.ends_with(':'));
183    }
184
185    #[test]
186    fn merges_etag_findings_with_truncated_see_references() {
187        let with_url = normalize_title(
188            "/: Server may leak inodes via ETags, header found with file /, inode: 917, size: 524, mtime: Wed Nov 11 20:09:58 2020. See: http://cve.mitre",
189        );
190        let with_cve = normalize_title(
191            "GET /: Server may leak inodes via ETags, header found with file /, inode: 917, size: 524, mtime: Wed Nov 11 20:09:58 2020. See: CVE-2003-141",
192        );
193        assert_eq!(with_url, with_cve);
194        assert!(!with_url.contains("See:"));
195        assert!(!with_cve.contains("CVE-2003"));
196    }
197
198    #[test]
199    fn preserves_bare_trailing_cve_id() {
200        assert_eq!(
201            normalize_title("Exposed service CVE-2017-0144"),
202            "exposed service cve-2017-0144"
203        );
204    }
205
206    #[test]
207    fn case_insensitive_and_whitespace_collapsed() {
208        assert_eq!(
209            normalize_title("  APACHE   version  "),
210            normalize_title("apache version")
211        );
212    }
213
214    #[test]
215    fn deduplicates_duplicated_nikto_csp_findings() {
216        let records = vec![
217            record(
218                "/: Suggested security header missing: content-security-policy. See: https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP",
219                "http://192.168.1.2/",
220            ),
221            record(
222                "GET /: Suggested security header missing: content-security-policy. See: https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP:",
223                "http://192.168.1.2/",
224            ),
225        ];
226        let result = deduplicate_records(records);
227        assert_eq!(result.unique_count, 1);
228        assert_eq!(result.removed_count, 1);
229        assert_eq!(result.records[0].merged_count, 2);
230        assert_eq!(result.records[0].raw_findings.len(), 2);
231    }
232
233    #[test]
234    fn deduplicates_on_normalized_title() {
235        let records = vec![
236            ScanRecord {
237                target: "10.0.0.1".into(),
238                tool: "nmap".into(),
239                title: "Apache Version /".into(),
240                severity: "high".into(),
241                port: Some(80),
242                cve: None,
243                service: Some("apache".into()),
244                raw: json!({}),
245                ..Default::default()
246            },
247            ScanRecord {
248                target: "10.0.0.1".into(),
249                tool: "nmap".into(),
250                title: "apache version".into(),
251                severity: "high".into(),
252                port: Some(80),
253                cve: None,
254                service: Some("apache".into()),
255                raw: json!({}),
256                ..Default::default()
257            },
258        ];
259
260        let result = deduplicate_records(records);
261        assert_eq!(result.unique_count, 1);
262        assert_eq!(result.removed_count, 1);
263    }
264
265    #[test]
266    fn distinct_targets_stay_separate() {
267        let records = vec![
268            record("Alive (200)", "http://192.168.1.2/"),
269            record("Alive (200)", "http://192.168.1.2/index.html"),
270        ];
271        let result = deduplicate_records(records);
272        assert_eq!(result.unique_count, 2);
273    }
274
275    #[test]
276    fn keeps_raw_titles_for_traceability() {
277        let records = vec![
278            record("Apache version", "10.0.0.1"),
279            record("APACHE VERSION", "10.0.0.1"),
280        ];
281        let result = deduplicate_records(records);
282        assert_eq!(
283            result.records[0].raw_findings,
284            vec!["Apache version", "APACHE VERSION"]
285        );
286        assert_eq!(result.records[0].merged_count, 2);
287    }
288}
ploit_malper/engine/dedup.rs

ploit_malper/engine/
dedup.rs
