1use std::net::{IpAddr, Ipv4Addr};
3use std::sync::{LazyLock, OnceLock};
4
5use aho_corasick::AhoCorasick;
6
7use crate::{EvidenceSpan, SecurityCategory};
8
9pub fn dlp_secret_ranges<'a>(
12 spans: impl IntoIterator<Item = &'a EvidenceSpan>,
13) -> Vec<(usize, usize)> {
14 let mut ranges = spans
15 .into_iter()
16 .filter(|span| crate::detectors::dlp::dlp::SECRET_GROUPS.contains(&span.label.as_str()))
17 .map(|span| (span.start_byte, span.end_byte))
18 .collect::<Vec<_>>();
19 ranges.sort_unstable();
20 ranges
21}
22
23pub fn retain_pii_outside_secret_ranges(spans: &mut Vec<EvidenceSpan>, secrets: &[(usize, usize)]) {
27 debug_assert!(secrets.is_sorted());
28 spans.retain(|span| {
29 let after = secrets.partition_point(|&(start, _)| start < span.end_byte);
30 !secrets[..after]
31 .iter()
32 .any(|&(_, end)| end > span.start_byte)
33 });
34}
35
36pub trait PostPredictionHook: Send + Sync {
39 fn retain(&self, category: SecurityCategory, text: &str, span: &EvidenceSpan) -> bool;
40}
41
42#[derive(Debug, Default, Clone, Copy)]
45pub struct LocalPathPersonHook;
46
47#[derive(Debug, Default, Clone, Copy)]
48pub struct CredentialTemplateHook;
49
50#[derive(Debug, Default, Clone, Copy)]
53pub struct PrivateNetworkDiagnosticHook;
54
55impl PostPredictionHook for CredentialTemplateHook {
56 fn retain(&self, category: SecurityCategory, _text: &str, span: &EvidenceSpan) -> bool {
57 if category != SecurityCategory::Dlp
58 || !matches!(
59 span.label.to_ascii_uppercase().as_str(),
60 "CREDENTIAL" | "SECRET_TOKEN"
61 )
62 {
63 return true;
64 }
65 is_real_secret_assignment(&span.text)
66 }
67}
68
69impl PostPredictionHook for LocalPathPersonHook {
70 fn retain(&self, category: SecurityCategory, text: &str, span: &EvidenceSpan) -> bool {
71 if !matches!(
72 category,
73 SecurityCategory::Pii | SecurityCategory::DynamicPii
74 ) || !span.label.eq_ignore_ascii_case("person")
75 {
76 return true;
77 }
78 !is_local_path_span(text, span)
79 }
80}
81
82impl PostPredictionHook for PrivateNetworkDiagnosticHook {
83 fn retain(&self, category: SecurityCategory, text: &str, span: &EvidenceSpan) -> bool {
84 self.retain_with(category, span, || looks_like_network_diagnostic(text))
85 }
86}
87
88impl PrivateNetworkDiagnosticHook {
89 fn retain_with(
93 &self,
94 category: SecurityCategory,
95 span: &EvidenceSpan,
96 network_diagnostic: impl FnOnce() -> bool,
97 ) -> bool {
98 if category != SecurityCategory::Pii || !span.label.eq_ignore_ascii_case("IP_ADDRESS") {
99 return true;
100 }
101 !(is_private_or_local_ip(&span.text) && network_diagnostic())
102 }
103}
104
105fn is_private_or_local_ip(value: &str) -> bool {
106 let v4 = |address: Ipv4Addr| {
109 address.is_private() || address.is_link_local() || address.is_loopback()
110 };
111 match crate::detectors::pii::validators::ip_address_part(value).parse::<IpAddr>() {
112 Ok(IpAddr::V4(address)) => v4(address),
113 Ok(IpAddr::V6(address)) => {
114 address.is_unique_local()
115 || address.is_unicast_link_local()
116 || address.is_loopback()
117 || address.to_ipv4_mapped().is_some_and(v4)
118 }
119 Err(_) => false,
120 }
121}
122
123fn looks_like_network_diagnostic(text: &str) -> bool {
124 static MARKERS: LazyLock<AhoCorasick> = LazyLock::new(|| {
125 AhoCorasick::builder()
126 .ascii_case_insensitive(true)
127 .build([
128 "networkstate",
129 "network state",
130 "network diagnostic",
131 "local endpoint",
132 "private endpoint",
133 "network interface",
134 "interface address",
135 "lan address",
136 ])
137 .expect("valid network diagnostic markers")
138 });
139 MARKERS.is_match(text)
140}
141
142fn is_local_path_span(text: &str, span: &EvidenceSpan) -> bool {
143 let before = &text[..span.start_byte.min(text.len())];
144 let after = &text[span.end_byte.min(text.len())..];
145 let path_start = before
146 .rfind(|ch: char| ch.is_whitespace() || matches!(ch, '"' | '\'' | '(' | '[' | '{'))
147 .map_or(0, |index| {
148 index + text[index..].chars().next().map_or(0, char::len_utf8)
149 });
150 let prefix = &text[path_start..span.start_byte.min(text.len())];
151 let suffix_end = after
152 .find(|ch: char| ch.is_whitespace() || matches!(ch, '"' | '\'' | ')' | ']' | '}'))
153 .unwrap_or(after.len());
154 let suffix = &after[..suffix_end];
155 let path = format!("{prefix}{}{suffix}", span.text);
156 (path.starts_with("/Users/")
157 || path.starts_with("/private/tmp/")
158 || path.starts_with("/tmp/")
159 || path.starts_with("~/"))
160 && path.contains('/')
161}
162
163pub fn filter_evidence(
164 category: SecurityCategory,
165 text: &str,
166 spans: Vec<EvidenceSpan>,
167) -> Vec<EvidenceSpan> {
168 let hook = LocalPathPersonHook;
169 let diagnostic = OnceLock::new();
170 let network_diagnostic = || *diagnostic.get_or_init(|| looks_like_network_diagnostic(text));
171 spans
172 .into_iter()
173 .filter(|span| hook.retain(category, text, span))
174 .filter(|span| CredentialTemplateHook.retain(category, text, span))
175 .filter(|span| PrivateNetworkDiagnosticHook.retain_with(category, span, network_diagnostic))
176 .collect()
177}
178
179pub fn is_real_secret_assignment(value: &str) -> bool {
180 let unpadded = value
183 .strip_suffix("==")
184 .or_else(|| value.strip_suffix('='))
185 .unwrap_or(value);
186 if unpadded.len() >= 16
187 && unpadded
188 .bytes()
189 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'-' | b'_'))
190 {
191 return true;
192 }
193 let Some((_, raw)) = value.split_once('=') else {
194 return true;
195 };
196 let value = raw
197 .trim()
198 .trim_matches('"')
199 .trim_matches('\'')
200 .to_ascii_lowercase();
201 if matches!(
202 value.as_str(),
203 "" | "changeme" | "your-token-here" | "replace-me" | "your_api_key_here"
204 ) {
205 return false;
206 }
207 value.len() >= 12 && shannon_entropy(value.as_bytes()) >= 2.5
208}
209
210fn shannon_entropy(value: &[u8]) -> f64 {
211 if value.is_empty() {
212 return 0.0;
213 }
214 let mut frequencies = [0usize; 256];
215 for byte in value {
216 frequencies[*byte as usize] += 1;
217 }
218 let length = value.len() as f64;
219 frequencies
220 .into_iter()
221 .filter(|count| *count > 0)
222 .map(|count| {
223 let probability = count as f64 / length;
224 -probability * probability.log2()
225 })
226 .sum()
227}
228
229#[cfg(test)]
230mod overlap_tests {
231 use super::*;
232
233 fn span(label: &str, start: usize, end: usize) -> EvidenceSpan {
234 EvidenceSpan {
235 label: label.into(),
236 text: String::new(),
237 score: 0.73,
238 start_byte: start,
239 end_byte: end,
240 start_char: 0,
241 end_char: 0,
242 }
243 }
244
245 #[test]
246 fn only_secret_labels_produce_sorted_ranges() {
247 let dlp = [
248 span("SECRET_TOKEN", 20, 30),
249 span("SENSITIVE_DOCUMENT", 0, 40),
250 span("CREDENTIAL", 4, 12),
251 ];
252 assert_eq!(dlp_secret_ranges(&dlp), [(4, 12), (20, 30)]);
253 }
254
255 #[test]
256 fn strict_byte_overlap_removes_nested_and_partial_spans_but_not_touching_spans() {
257 let secrets = dlp_secret_ranges(&[span("CREDENTIAL", 4, 12), span("API_KEY", 8, 16)]);
258 let mut pii = vec![
259 span("EMAIL", 0, 4),
260 span("EMAIL", 3, 5),
261 span("EMAIL", 5, 7),
262 span("EMAIL", 12, 17),
263 span("EMAIL", 16, 20),
264 span("EMAIL", 0, 20),
265 ];
266 retain_pii_outside_secret_ranges(&mut pii, &secrets);
267 assert_eq!(
268 pii.iter()
269 .map(|s| (s.start_byte, s.end_byte))
270 .collect::<Vec<_>>(),
271 [(0, 4), (16, 20)]
272 );
273 assert!(pii.iter().all(|s| s.score == 0.73));
274 retain_pii_outside_secret_ranges(&mut pii, &[]);
275 assert_eq!(pii.len(), 2);
276 }
277}