ploit_malper/engine/
dedup.rs

1use std::collections::HashMap;
2
3use regex::Regex;
4use sha2::{Digest, Sha256};
5
6use crate::analyze::extract_endpoint;
7use crate::models::{DedupResult, ScanRecord};
8
9/// Structured endpoint parts parsed from a raw URL/path so fingerprinting is
10/// immune to scheme, casing, and host-framing noise.
11#[derive(Debug, Clone, PartialEq, Eq, Default)]
12pub struct EndpointParts {
13    pub scheme: String,
14    pub host: String,
15    pub port: Option<u16>,
16    pub path: String,
17    pub query: String,
18}
19
20/// Normalize a target for comparison so `http://h/` and `http://h` merge,
21/// while a distinct path keeps records separated.
22pub fn normalize_target(target: &str) -> String {
23    let lower = target.trim().to_lowercase();
24    let stripped = lower
25        .strip_prefix("https://")
26        .or_else(|| lower.strip_prefix("http://"))
27        .unwrap_or(&lower);
28    let mut host = stripped.trim_end_matches('/').to_string();
29
30    // Keep the path portion so `/`, `/index.html` and `/admin` stay distinct.
31    let host_part;
32    if let Some(idx) = host.find('/') {
33        host_part = host[..idx].to_string();
34    } else {
35        host_part = host.clone();
36        host = host_part.clone();
37    }
38    if host_part.is_empty() {
39        return host.clone();
40    }
41
42    // Strip a `:port` suffix unless this is an IPv6 literal.
43    if host_part.parse::<std::net::Ipv6Addr>().is_err() {
44        if let Some(colon) = host_part.rfind(':') {
45            let after = &host_part[colon + 1..];
46            if !after.is_empty() && after.chars().all(|c| c.is_ascii_digit()) {
47                let rest = &host[host_part.len()..];
48                return format!("{}{}", &host_part[..colon], rest);
49            }
50        }
51    }
52    host
53}
54
55/// Parse an endpoint (URL or path) into structured components. Safe on
56/// arbitrary scanner output: any malformed input degrades to empty parts.
57pub fn parse_endpoint_parts(value: &str) -> EndpointParts {
58    let value = value.trim();
59    if value.is_empty() {
60        return EndpointParts::default();
61    }
62
63    let (scheme, rest) = match value.split_once("://") {
64        Some((scheme, rest)) => (scheme.to_lowercase(), rest),
65        None => (String::new(), value),
66    };
67
68    let (authority, path_and_query) = match rest.find('/') {
69        Some(idx) => (&rest[..idx], &rest[idx..]),
70        None => (rest, ""),
71    };
72
73    let (path, query) = match path_and_query.split_once('?') {
74        Some((p, q)) => (p.to_string(), q.to_string()),
75        None => (path_and_query.to_string(), String::new()),
76    };
77    let path = if path.is_empty() {
78        "/".to_string()
79    } else {
80        path
81    };
82
83    let (host, port) = parse_authority(authority);
84    EndpointParts {
85        scheme,
86        host,
87        port,
88        path,
89        query,
90    }
91}
92
93fn parse_authority(authority: &str) -> (String, Option<u16>) {
94    // IPv6 literal like `[::1]:8080`.
95    if let Some(rest) = authority.strip_prefix('[') {
96        if let Some(close) = rest.find(']') {
97            let host = rest[..close].to_string();
98            let after = &rest[close + 1..];
99            let port = after.strip_prefix(':').and_then(|p| p.parse::<u16>().ok());
100            return (host, port);
101        }
102    }
103    if let Some(colon) = authority.rfind(':') {
104        if let Ok(port) = authority[colon + 1..].parse::<u16>() {
105            return (authority[..colon].to_string(), Some(port));
106        }
107    }
108    (authority.to_string(), None)
109}
110
111/// First parameter name in a query string, used to fingerprint injectable
112/// endpoints neutrally (param name, not value).
113fn first_parameter(query: &str) -> Option<String> {
114    let first = query.split('&').next()?;
115    let key = first.split('=').next()?.trim();
116    if key.is_empty() {
117        None
118    } else {
119        Some(key.to_string())
120    }
121}
122
123fn normalize_key(value: &str) -> String {
124    value
125        .split_whitespace()
126        .collect::<Vec<_>>()
127        .join(" ")
128        .to_lowercase()
129}
130
131/// Normalize a finding title for deduplication:
132///
133/// * strip leading HTTP method prefixes (`GET`, `POST`, ...)
134/// * normalize endpoint prefixes so `"/:"`, `"GET /:"` and `"/"` collapse
135/// * remove trailing and duplicated punctuation (e.g. stray `:` / `.`)
136/// * strip trailing reference URLs (they are truncated at arbitrary lengths)
137/// * collapse whitespace
138/// * lowercase (case-insensitive comparison)
139///
140/// The raw title is preserved on the record (`raw_findings`) for reporting.
141pub fn normalize_title(title: &str) -> String {
142    let mut text = title.trim().to_lowercase();
143
144    // Strip leading HTTP verbs (GET/POST/HEAD/...).
145    text = Regex::new(r"^\s*(get|post|put|head|delete|patch|options|trace|connect)\s+")
146        .expect("valid method regex")
147        .replace(&text, "")
148        .to_string();
149
150    // Normalize leading endpoint markers ("/:", "/", "GET /:" → no prefix).
151    text = text.trim_start_matches('/').trim_start().to_string();
152    text = text.trim_start_matches(':').trim_start().to_string();
153
154    // Drop trailing reference material: "See: <url>" (which scanners truncate
155    // at arbitrary lengths) or "See: <CVE>" noise. A bare trailing CVE id is
156    // preserved because it is the finding itself.
157    text =
158        Regex::new(r"\s*(?:(?:see\s*:?\s*)?https?://\S+|\bsee\s*:?\s*cve-\d{4}-\d{3,7})\s*:?\s*$")
159            .expect("valid reference regex")
160            .replace(&text, "")
161            .to_string();
162
163    // Collapse whitespace.
164    text = text.split_whitespace().collect::<Vec<_>>().join(" ");
165
166    // Collapse `|`-framed scanner output: treat `|` as a pure segment
167    // separator and drop empty segments, so `"GET / |"` and `"GET / "`
168    // frame the same finding. Meaningful non-empty segments are preserved.
169    text = text
170        .split('|')
171        .map(str::trim)
172        .filter(|segment| !segment.is_empty())
173        .collect::<Vec<_>>()
174        .join("|");
175
176    // Strip trailing punctuation and path separators, then re-trim.
177    text = text
178        .trim_matches(|c: char| ".,:;\\/|".contains(c))
179        .trim()
180        .to_string();
181
182    text
183}
184
185/// Stable structured fingerprint for a record, built from normalized fields
186/// (asset, service/port, type, endpoint path and parameter) rather than from
187/// raw, framing-prone scanner strings. Empty optional fields are dropped so a
188/// record that simply lacks a port/service never accidentally forks identity.
189fn fingerprint_segments(record: &ScanRecord) -> Vec<String> {
190    let mut segments = vec![normalize_target(&record.target)];
191
192    if let Some(port) = record.port {
193        segments.push(format!("port:{}", port));
194    }
195    if let Some(service) = record.service.as_deref() {
196        let service = normalize_key(service);
197        if !service.is_empty() {
198            segments.push(format!("service:{}", service));
199        }
200    }
201
202    let normalized = normalize_title(&record.title);
203    if !normalized.is_empty() {
204        segments.push(format!("type:{}", normalized));
205    }
206
207    if let Some(endpoint) = extract_endpoint(record) {
208        let parts = parse_endpoint_parts(&endpoint);
209        // Trim framing delimiters off the path so `...CSP` and `...CSP:` and
210        // `/admin/` merge, while genuinely different paths stay distinct.
211        let path = normalize_path(&parts.path);
212        if !path.is_empty() {
213            segments.push(format!("path:{}", path));
214        }
215        if let Some(param) = first_parameter(&parts.query) {
216            segments.push(format!("param:{}", param));
217        }
218    }
219
220    segments
221}
222
223fn normalize_path(path: &str) -> String {
224    path.trim()
225        .trim_end_matches(|c: char| ".,:;\\/|".contains(c))
226        .trim()
227        .to_lowercase()
228}
229
230fn composite_key(record: &ScanRecord) -> String {
231    let raw = fingerprint_segments(record).join("|");
232    let mut hasher = Sha256::new();
233    hasher.update(raw.as_bytes());
234    format!("{:x}", hasher.finalize())
235}
236
237pub fn deduplicate_records(records: Vec<ScanRecord>) -> DedupResult {
238    let total_count = records.len();
239    let mut seen: HashMap<String, usize> = HashMap::new();
240    let mut unique_records = Vec::new();
241
242    for mut record in records {
243        let key = composite_key(&record);
244        record.normalized_title = normalize_title(&record.title);
245        record.merged_count = 1;
246        record.raw_findings = vec![record.title.clone()];
247
248        if let Some(&existing_idx) = seen.get(&key) {
249            merge_record(&mut unique_records[existing_idx], &record);
250        } else {
251            if record.affected_endpoints.is_empty() {
252                record.affected_endpoints = extract_endpoint(&record).into_iter().collect();
253            }
254            seen.insert(key, unique_records.len());
255            unique_records.push(record);
256        }
257    }
258
259    let unique_count = unique_records.len();
260    let removed_count = total_count.saturating_sub(unique_count);
261
262    DedupResult {
263        unique_count,
264        removed_count,
265        records: unique_records,
266    }
267}
268
269/// Merge a duplicate into the kept record, preserving every piece of
270/// scanner evidence and aggregating endpoints, CVEs, tools and count.
271fn merge_record(kept: &mut ScanRecord, duplicate: &ScanRecord) {
272    kept.merged_count += 1;
273
274    if let Some(endpoint) = extract_endpoint(duplicate) {
275        if !kept.affected_endpoints.contains(&endpoint) {
276            kept.affected_endpoints.push(endpoint);
277        }
278    }
279    for cve in &duplicate.cves {
280        if !kept.cves.contains(cve) {
281            kept.cves.push(cve.clone());
282        }
283    }
284    if kept.cve.is_none() {
285        kept.cve = duplicate.cve.clone();
286    }
287    if kept.service.is_none() {
288        kept.service = duplicate.service.clone();
289    }
290    if kept.port.is_none() {
291        kept.port = duplicate.port;
292    }
293    if kept.detail.is_empty() {
294        kept.detail = duplicate.detail.clone();
295    }
296    if kept.reference.is_empty() {
297        kept.reference = duplicate.reference.clone();
298    }
299    for raw in &duplicate.raw_findings {
300        if !kept.raw_findings.contains(raw) {
301            kept.raw_findings.push(raw.clone());
302        }
303    }
304
305    // Keep the most severe classification across merged records.
306    if crate::analyze::severity_rank(&duplicate.severity)
307        < crate::analyze::severity_rank(&kept.severity)
308    {
309        kept.severity = duplicate.severity.clone();
310    }
311}
312
313pub fn normalize_title_text(title: &str) -> String {
314    normalize_title(title)
315}
316
317#[cfg(test)]
318mod tests {
319    use super::*;
320    use crate::models::ScanRecord;
321    use serde_json::json;
322
323    fn record(title: &str, target: &str) -> ScanRecord {
324        ScanRecord {
325            target: target.into(),
326            tool: "nikto".into(),
327            title: title.into(),
328            severity: "low".into(),
329            raw: json!({}),
330            ..Default::default()
331        }
332    }
333
334    #[test]
335    fn strips_http_method_and_endpoint_prefix() {
336        assert_eq!(
337            normalize_title("/: Suggested security header missing: content-security-policy"),
338            normalize_title("GET /: Suggested security header missing: content-security-policy")
339        );
340    }
341
342    #[test]
343    fn removes_trailing_punctuation_and_truncated_urls() {
344        let with_url = normalize_title(
345            "GET /: Suggested security header missing: content-security-policy. See: https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP:",
346        );
347        let truncated_url = normalize_title(
348            "/: Suggested security header missing: content-security-policy. See: https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP",
349        );
350        assert_eq!(with_url, truncated_url);
351        assert!(!with_url.contains("See:"));
352        assert!(!with_url.ends_with(':'));
353    }
354
355    #[test]
356    fn merges_etag_findings_with_truncated_see_references() {
357        let with_url = normalize_title(
358            "/: Server may leak inodes via ETags, header found with file /, inode: 917, size: 524, mtime: Wed Nov 11 20:09:58 2020. See: http://cve.mitre",
359        );
360        let with_cve = normalize_title(
361            "GET /: Server may leak inodes via ETags, header found with file /, inode: 917, size: 524, mtime: Wed Nov 11 20:09:58 2020. See: CVE-2003-141",
362        );
363        assert_eq!(with_url, with_cve);
364        assert!(!with_url.contains("See:"));
365        assert!(!with_cve.contains("CVE-2003"));
366    }
367
368    #[test]
369    fn preserves_bare_trailing_cve_id() {
370        assert_eq!(
371            normalize_title("Exposed service CVE-2017-0144"),
372            "exposed service cve-2017-0144"
373        );
374    }
375
376    #[test]
377    fn case_insensitive_and_whitespace_collapsed() {
378        assert_eq!(
379            normalize_title("  APACHE   version  "),
380            normalize_title("apache version")
381        );
382    }
383
384    #[test]
385    fn deduplicates_duplicated_nikto_csp_findings() {
386        let records = vec![
387            record(
388                "/: Suggested security header missing: content-security-policy. See: https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP",
389                "http://192.168.1.2/",
390            ),
391            record(
392                "GET /: Suggested security header missing: content-security-policy. See: https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP:",
393                "http://192.168.1.2/",
394            ),
395        ];
396        let result = deduplicate_records(records);
397        assert_eq!(result.unique_count, 1);
398        assert_eq!(result.removed_count, 1);
399        assert_eq!(result.records[0].merged_count, 2);
400        assert_eq!(result.records[0].raw_findings.len(), 2);
401    }
402
403    #[test]
404    fn deduplicates_on_normalized_title() {
405        let records = vec![
406            ScanRecord {
407                target: "10.0.0.1".into(),
408                tool: "nmap".into(),
409                title: "Apache Version /".into(),
410                severity: "high".into(),
411                port: Some(80),
412                cve: None,
413                service: Some("apache".into()),
414                raw: json!({}),
415                ..Default::default()
416            },
417            ScanRecord {
418                target: "10.0.0.1".into(),
419                tool: "nmap".into(),
420                title: "apache version".into(),
421                severity: "high".into(),
422                port: Some(80),
423                cve: None,
424                service: Some("apache".into()),
425                raw: json!({}),
426                ..Default::default()
427            },
428        ];
429
430        let result = deduplicate_records(records);
431        assert_eq!(result.unique_count, 1);
432        assert_eq!(result.removed_count, 1);
433    }
434
435    #[test]
436    fn distinct_targets_stay_separate() {
437        let records = vec![
438            record("Alive (200)", "http://192.168.1.2/"),
439            record("Alive (200)", "http://192.168.1.2/index.html"),
440        ];
441        let result = deduplicate_records(records);
442        assert_eq!(result.unique_count, 2);
443    }
444
445    #[test]
446    fn keeps_raw_titles_for_traceability() {
447        let records = vec![
448            record("Apache version", "10.0.0.1"),
449            record("APACHE VERSION", "10.0.0.1"),
450        ];
451        let result = deduplicate_records(records);
452        assert_eq!(
453            result.records[0].raw_findings,
454            vec!["Apache version", "APACHE VERSION"]
455        );
456        assert_eq!(result.records[0].merged_count, 2);
457    }
458
459    #[test]
460    fn collapses_pipe_framed_titles_into_same_identity() {
461        let records = vec![record("GET / |", "10.0.0.1"), record("GET / ", "10.0.0.1")];
462        let result = deduplicate_records(records);
463        assert_eq!(result.unique_count, 1);
464        assert_eq!(result.records[0].merged_count, 2);
465    }
466
467    #[test]
468    fn structured_fingerprint_parts() {
469        let parts = parse_endpoint_parts("https://example.com:8443/admin?a=1&b=2");
470        assert_eq!(parts.scheme, "https");
471        assert_eq!(parts.host, "example.com");
472        assert_eq!(parts.port, Some(8443));
473        assert_eq!(parts.path, "/admin");
474        assert_eq!(parts.query, "a=1&b=2");
475
476        let ipv6 = parse_endpoint_parts("[::1]:8080/x");
477        assert_eq!(ipv6.host, "::1");
478        assert_eq!(ipv6.port, Some(8080));
479        assert_eq!(ipv6.path, "/x");
480    }
481
482    #[test]
483    fn same_title_different_ports_stay_separate() {
484        let mut a = record("/: Apache", "10.0.0.1");
485        a.port = Some(80);
486        a.service = Some("http".into());
487        let mut b = record("/: Apache", "10.0.0.1");
488        b.port = Some(8080);
489        b.service = Some("http".into());
490        let result = deduplicate_records(vec![a, b]);
491        assert_eq!(result.unique_count, 2);
492    }
493
494    #[test]
495    fn normalization_is_case_and_scheme_insensitive() {
496        let a = record("Apache version", "http://10.0.0.1/");
497        let b = record("APACHE VERSION", "10.0.0.1:80");
498        let result = deduplicate_records(vec![a, b]);
499        assert_eq!(result.unique_count, 1);
500        assert_eq!(result.removed_count, 1);
501    }
502}
ploit_malper/engine/dedup.rs

ploit_malper/engine/
dedup.rs
