Skip to main content

patronus_ark_core/
post_prediction.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2use std::net::{IpAddr, Ipv4Addr};
3use std::sync::{LazyLock, OnceLock};
4
5use aho_corasick::AhoCorasick;
6
7use crate::{EvidenceSpan, SecurityCategory};
8
9/// Extract DLP secret byte ranges, sorted by start/end, without cloning evidence.
10/// The host selects the DLP results belonging to the same source text.
11pub fn dlp_secret_ranges<'a>(
12    spans: impl IntoIterator<Item = &'a EvidenceSpan>,
13) -> Vec<(usize, usize)> {
14    let mut ranges = spans
15        .into_iter()
16        .filter(|span| crate::detectors::dlp::dlp::SECRET_GROUPS.contains(&span.label.as_str()))
17        .map(|span| (span.start_byte, span.end_byte))
18        .collect::<Vec<_>>();
19    ranges.sort_unstable();
20    ranges
21}
22
23/// Retain PII evidence outside secret ranges returned by [`dlp_secret_ranges`].
24/// Ranges must be sorted by start/end. Touching end-exclusive byte ranges do not
25/// overlap. This operation changes only evidence, never a classifier decision.
26pub fn retain_pii_outside_secret_ranges(spans: &mut Vec<EvidenceSpan>, secrets: &[(usize, usize)]) {
27    debug_assert!(secrets.is_sorted());
28    spans.retain(|span| {
29        let after = secrets.partition_point(|&(start, _)| start < span.end_byte);
30        !secrets[..after]
31            .iter()
32            .any(|&(_, end)| end > span.start_byte)
33    });
34}
35
36/// Post-prediction evidence filter. Predictions remain available for diagnostics;
37/// the hook only decides which spans are actionable evidence.
38pub trait PostPredictionHook: Send + Sync {
39    fn retain(&self, category: SecurityCategory, text: &str, span: &EvidenceSpan) -> bool;
40}
41
42/// Suppresses GLiNER/native \`person\` spans that are merely usernames in local
43/// POSIX/macOS paths. Other labels and person spans in prose are unchanged.
44#[derive(Debug, Default, Clone, Copy)]
45pub struct LocalPathPersonHook;
46
47#[derive(Debug, Default, Clone, Copy)]
48pub struct CredentialTemplateHook;
49
50/// Keeps private/local IP predictions for diagnostics while preventing their
51/// evidence spans from triggering redaction in network diagnostic output.
52#[derive(Debug, Default, Clone, Copy)]
53pub struct PrivateNetworkDiagnosticHook;
54
55impl PostPredictionHook for CredentialTemplateHook {
56    fn retain(&self, category: SecurityCategory, _text: &str, span: &EvidenceSpan) -> bool {
57        if category != SecurityCategory::Dlp
58            || !matches!(
59                span.label.to_ascii_uppercase().as_str(),
60                "CREDENTIAL" | "SECRET_TOKEN"
61            )
62        {
63            return true;
64        }
65        is_real_secret_assignment(&span.text)
66    }
67}
68
69impl PostPredictionHook for LocalPathPersonHook {
70    fn retain(&self, category: SecurityCategory, text: &str, span: &EvidenceSpan) -> bool {
71        if !matches!(
72            category,
73            SecurityCategory::Pii | SecurityCategory::DynamicPii
74        ) || !span.label.eq_ignore_ascii_case("person")
75        {
76            return true;
77        }
78        !is_local_path_span(text, span)
79    }
80}
81
82impl PostPredictionHook for PrivateNetworkDiagnosticHook {
83    fn retain(&self, category: SecurityCategory, text: &str, span: &EvidenceSpan) -> bool {
84        self.retain_with(category, span, || looks_like_network_diagnostic(text))
85    }
86}
87
88impl PrivateNetworkDiagnosticHook {
89    /// [`PostPredictionHook::retain`] with the text's diagnostic marker check
90    /// supplied by the caller, so that one text is searched once for all its
91    /// spans (H03).
92    fn retain_with(
93        &self,
94        category: SecurityCategory,
95        span: &EvidenceSpan,
96        network_diagnostic: impl FnOnce() -> bool,
97    ) -> bool {
98        if category != SecurityCategory::Pii || !span.label.eq_ignore_ascii_case("IP_ADDRESS") {
99            return true;
100        }
101        !(is_private_or_local_ip(&span.text) && network_diagnostic())
102    }
103}
104
105fn is_private_or_local_ip(value: &str) -> bool {
106    // An IP span may carry a zone ID or CIDR prefix length, and an IPv4-mapped
107    // address is one span (B-IP).
108    let v4 = |address: Ipv4Addr| {
109        address.is_private() || address.is_link_local() || address.is_loopback()
110    };
111    match crate::detectors::pii::validators::ip_address_part(value).parse::<IpAddr>() {
112        Ok(IpAddr::V4(address)) => v4(address),
113        Ok(IpAddr::V6(address)) => {
114            address.is_unique_local()
115                || address.is_unicast_link_local()
116                || address.is_loopback()
117                || address.to_ipv4_mapped().is_some_and(v4)
118        }
119        Err(_) => false,
120    }
121}
122
123fn looks_like_network_diagnostic(text: &str) -> bool {
124    static MARKERS: LazyLock<AhoCorasick> = LazyLock::new(|| {
125        AhoCorasick::builder()
126            .ascii_case_insensitive(true)
127            .build([
128                "networkstate",
129                "network state",
130                "network diagnostic",
131                "local endpoint",
132                "private endpoint",
133                "network interface",
134                "interface address",
135                "lan address",
136            ])
137            .expect("valid network diagnostic markers")
138    });
139    MARKERS.is_match(text)
140}
141
142fn is_local_path_span(text: &str, span: &EvidenceSpan) -> bool {
143    let before = &text[..span.start_byte.min(text.len())];
144    let after = &text[span.end_byte.min(text.len())..];
145    let path_start = before
146        .rfind(|ch: char| ch.is_whitespace() || matches!(ch, '"' | '\'' | '(' | '[' | '{'))
147        .map_or(0, |index| {
148            index + text[index..].chars().next().map_or(0, char::len_utf8)
149        });
150    let prefix = &text[path_start..span.start_byte.min(text.len())];
151    let suffix_end = after
152        .find(|ch: char| ch.is_whitespace() || matches!(ch, '"' | '\'' | ')' | ']' | '}'))
153        .unwrap_or(after.len());
154    let suffix = &after[..suffix_end];
155    let path = format!("{prefix}{}{suffix}", span.text);
156    (path.starts_with("/Users/")
157        || path.starts_with("/private/tmp/")
158        || path.starts_with("/tmp/")
159        || path.starts_with("~/"))
160        && path.contains('/')
161}
162
163pub fn filter_evidence(
164    category: SecurityCategory,
165    text: &str,
166    spans: Vec<EvidenceSpan>,
167) -> Vec<EvidenceSpan> {
168    let hook = LocalPathPersonHook;
169    let diagnostic = OnceLock::new();
170    let network_diagnostic = || *diagnostic.get_or_init(|| looks_like_network_diagnostic(text));
171    spans
172        .into_iter()
173        .filter(|span| hook.retain(category, text, span))
174        .filter(|span| CredentialTemplateHook.retain(category, text, span))
175        .filter(|span| PrivateNetworkDiagnosticHook.retain_with(category, span, network_diagnostic))
176        .collect()
177}
178
179pub fn is_real_secret_assignment(value: &str) -> bool {
180    // F-GEN2: base64 padding (`KEY: <base64>=` in a Kubernetes Secret) is no
181    // assignment.
182    let unpadded = value
183        .strip_suffix("==")
184        .or_else(|| value.strip_suffix('='))
185        .unwrap_or(value);
186    if unpadded.len() >= 16
187        && unpadded
188            .bytes()
189            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'-' | b'_'))
190    {
191        return true;
192    }
193    let Some((_, raw)) = value.split_once('=') else {
194        return true;
195    };
196    let value = raw
197        .trim()
198        .trim_matches('"')
199        .trim_matches('\'')
200        .to_ascii_lowercase();
201    if matches!(
202        value.as_str(),
203        "" | "changeme" | "your-token-here" | "replace-me" | "your_api_key_here"
204    ) {
205        return false;
206    }
207    value.len() >= 12 && shannon_entropy(value.as_bytes()) >= 2.5
208}
209
210fn shannon_entropy(value: &[u8]) -> f64 {
211    if value.is_empty() {
212        return 0.0;
213    }
214    let mut frequencies = [0usize; 256];
215    for byte in value {
216        frequencies[*byte as usize] += 1;
217    }
218    let length = value.len() as f64;
219    frequencies
220        .into_iter()
221        .filter(|count| *count > 0)
222        .map(|count| {
223            let probability = count as f64 / length;
224            -probability * probability.log2()
225        })
226        .sum()
227}
228
229#[cfg(test)]
230mod overlap_tests {
231    use super::*;
232
233    fn span(label: &str, start: usize, end: usize) -> EvidenceSpan {
234        EvidenceSpan {
235            label: label.into(),
236            text: String::new(),
237            score: 0.73,
238            start_byte: start,
239            end_byte: end,
240            start_char: 0,
241            end_char: 0,
242        }
243    }
244
245    #[test]
246    fn only_secret_labels_produce_sorted_ranges() {
247        let dlp = [
248            span("SECRET_TOKEN", 20, 30),
249            span("SENSITIVE_DOCUMENT", 0, 40),
250            span("CREDENTIAL", 4, 12),
251        ];
252        assert_eq!(dlp_secret_ranges(&dlp), [(4, 12), (20, 30)]);
253    }
254
255    #[test]
256    fn strict_byte_overlap_removes_nested_and_partial_spans_but_not_touching_spans() {
257        let secrets = dlp_secret_ranges(&[span("CREDENTIAL", 4, 12), span("API_KEY", 8, 16)]);
258        let mut pii = vec![
259            span("EMAIL", 0, 4),
260            span("EMAIL", 3, 5),
261            span("EMAIL", 5, 7),
262            span("EMAIL", 12, 17),
263            span("EMAIL", 16, 20),
264            span("EMAIL", 0, 20),
265        ];
266        retain_pii_outside_secret_ranges(&mut pii, &secrets);
267        assert_eq!(
268            pii.iter()
269                .map(|s| (s.start_byte, s.end_byte))
270                .collect::<Vec<_>>(),
271            [(0, 4), (16, 20)]
272        );
273        assert!(pii.iter().all(|s| s.score == 0.73));
274        retain_pii_outside_secret_ranges(&mut pii, &[]);
275        assert_eq!(pii.len(), 2);
276    }
277}