provide_telemetry/
pii.rs

1// SPDX-FileCopyrightText: Copyright (C) 2026 provide.io llc
2// SPDX-License-Identifier: Apache-2.0
3// SPDX-Comment: Part of provide-telemetry.
4//
5
6use regex::Regex;
7use serde_json::{Map, Value};
8use sha2::{Digest, Sha256};
9use std::sync::{Mutex, OnceLock};
10
11use crate::classification::{classify_key, get_classification_policy};
12use crate::receipts::record_redaction;
13
14#[derive(Clone, Debug, PartialEq, Eq)]
15pub enum PIIMode {
16    Drop,
17    Redact,
18    Hash,
19    Truncate,
20}
21
22#[derive(Clone, Debug, PartialEq, Eq)]
23pub struct PIIRule {
24    pub path: Vec<String>,
25    pub mode: PIIMode,
26    pub truncate_to: usize,
27}
28
29impl PIIRule {
30    pub fn new(path: Vec<String>, mode: PIIMode, truncate_to: usize) -> Self {
31        Self {
32            path,
33            mode,
34            truncate_to,
35        }
36    }
37}
38
39const REDACTED: &str = "***";
40const TRUNC_SUFFIX: &str = "...";
41const DEFAULT_SENSITIVE: &[&str] = &[
42    "password",
43    "passwd",
44    "secret",
45    "token",
46    "api_key",
47    "apikey",
48    "auth",
49    "authorization",
50    "credential",
51    "private_key",
52    "ssn",
53    "credit_card",
54    "creditcard",
55    "cvv",
56    "pin",
57    "account_number",
58    "cookie",
59];
60
61/// A secret pattern with a diagnostic name and the compiled regex.
62#[derive(Clone, Debug)]
63pub struct SecretPattern {
64    pub name: String,
65    pub pattern: Regex,
66}
67
68static RULES: OnceLock<Mutex<Vec<PIIRule>>> = OnceLock::new();
69static CUSTOM_SECRET_PATTERNS: OnceLock<Mutex<Vec<(String, Regex)>>> = OnceLock::new();
70
71#[cfg_attr(test, mutants::skip)] // Equivalent mutants only rewrite Vec::new() syntax.
72fn empty_pii_rules_mutex() -> Mutex<Vec<PIIRule>> {
73    Mutex::new(Vec::new())
74}
75
76fn rules() -> &'static Mutex<Vec<PIIRule>> {
77    RULES.get_or_init(empty_pii_rules_mutex)
78}
79
80#[cfg_attr(test, mutants::skip)] // Equivalent mutants only rewrite Vec::new() syntax.
81fn empty_custom_patterns_mutex() -> Mutex<Vec<(String, Regex)>> {
82    Mutex::new(Vec::new())
83}
84
85fn custom_secret_patterns() -> &'static Mutex<Vec<(String, Regex)>> {
86    CUSTOM_SECRET_PATTERNS.get_or_init(empty_custom_patterns_mutex)
87}
88
89fn compiled_builtin_secret_patterns() -> Vec<Regex> {
90    crate::secret_patterns_generated::PATTERNS
91        .iter()
92        .map(|(_name, pattern)| Regex::new(pattern).expect("generated pattern must be valid"))
93        .collect()
94}
95
96fn builtin_secret_patterns() -> &'static [Regex] {
97    static COMPILED: OnceLock<Vec<Regex>> = OnceLock::new();
98    COMPILED
99        .get_or_init(compiled_builtin_secret_patterns)
100        .as_slice()
101}
102
103/// How many slash-separated parts a span needs before its shape reads as a path.
104const PATH_MIN_SEGMENTS: usize = 3;
105
106/// True when a matched span is a filesystem path rather than a secret.
107///
108/// The long_base64 pattern is `[A-Za-z0-9+/]{40,}` and `/` belongs to the
109/// base64 alphabet, so any deep path of unpunctuated segments matched it:
110/// `/home/deploy/apps/production/current/lib/service` is 48 characters of pure
111/// base64 alphabet containing no secret. Narrowing the charset is not the fix —
112/// dropping `/` costs 44% of detections on 32-byte secrets, because a 44-char
113/// base64 string holding one slash cannot be told from a path by charset alone.
114///
115/// Shape separates them: a path carries several short all-lowercase words
116/// (usr, local, lib), which random base64 effectively never yields — a
117/// 20-character all-lowercase run has probability (26/64)^20, about 1e-8.
118fn looks_like_path(span: &str) -> bool {
119    let segments: Vec<&str> = span.split('/').filter(|s| !s.is_empty()).collect();
120    if segments.len() < PATH_MIN_SEGMENTS {
121        return false;
122    }
123    let wordy = segments
124        .iter()
125        .filter(|s| !s.is_empty() && s.chars().all(|c| c.is_ascii_lowercase()))
126        .count();
127    wordy * 2 >= segments.len()
128}
129
130/// Every secret-looking byte span in *text*, widened to whole tokens, sorted
131/// and coalesced.
132///
133/// Every pattern is scanned across the WHOLE value, not stopped at its first
134/// match, and every pattern is tried even after one has hit. Skipping either
135/// leaks:
136///
137/// - Stopping a pattern at its first match let a path shadow a real secret.
138///   long_base64 matches a path first; suppressing that match as path-shaped
139///   moved the scan to the next pattern, and long_base64 is the last one, so
140///   the credential behind the path was never looked for at all.
141/// - Stopping at the first pattern to hit left a field's second and third
142///   secrets in the log, which whole-value blanking used to cover for free.
143///
144/// `find` runs first as a fast path: a clean value, which is nearly every log
145/// field, allocates nothing, because `find_iter` is only entered once a
146/// pattern is known to match.
147fn secret_spans(text: &str) -> Vec<(usize, usize)> {
148    if text.len() < crate::secret_patterns_generated::MIN_SECRET_LENGTH {
149        return Vec::new();
150    }
151    let mut spans: Vec<(usize, usize)> = Vec::new();
152    let mut collect = |pattern: &Regex| {
153        if pattern.find(text).is_none() {
154            return;
155        }
156        for m in pattern.find_iter(text) {
157            // A registered pattern that can match the empty string carries no
158            // secret; widening a zero-length match to its token would redact a
159            // word for nothing.
160            if m.start() == m.end() {
161                continue;
162            }
163            if !looks_like_path(m.as_str()) {
164                spans.push(expand_to_token(text, m.start(), m.end()));
165            }
166        }
167    };
168    for pattern in builtin_secret_patterns() {
169        collect(pattern);
170    }
171    {
172        let patterns = crate::_lock::lock(custom_secret_patterns());
173        for (_, pattern) in patterns.iter() {
174            collect(pattern);
175        }
176    }
177    merge_spans(spans)
178}
179
180/// Widen a match to its whitespace-delimited token.
181///
182/// Redacting the literal match alone can leave part of a credential behind:
183/// the jwt pattern matches header.payload, and a JWT has THREE dot-separated
184/// parts, so the signature would survive. Whitespace is the boundary a secret
185/// cannot cross without ceasing to be one token.
186///
187/// Found by search rather than by walking an index. A `while` loop stepping an
188/// index is one mutated assignment away from never terminating -- `start -= 1`
189/// becoming `start /= 1` hangs the process -- and a hung mutant is a gap in the
190/// suite that reads as a timeout rather than as a failure.
191fn expand_to_token(text: &str, start: usize, end: usize) -> (usize, usize) {
192    let left = text[..start]
193        .rfind(|c: char| c.is_ascii_whitespace())
194        .map_or(0, |index| index + 1);
195    let right = text[end..]
196        .find(|c: char| c.is_ascii_whitespace())
197        .map_or(text.len(), |index| end + index);
198    (left, right)
199}
200
201/// Sort and coalesce overlapping spans so each region is replaced once. Two
202/// patterns can match the same credential -- long_base64 and jwt both hit a
203/// JWT -- and after widening they overlap exactly, which would emit "******".
204///
205/// There is deliberately no `spans.len() < 2` shortcut. It saved nothing the
206/// loop below does not already handle for zero or one span, and every mutant
207/// of its comparison either changed behaviour in a way no test could see or
208/// skipped the merge for exactly two spans.
209fn merge_spans(mut spans: Vec<(usize, usize)>) -> Vec<(usize, usize)> {
210    spans.sort_unstable();
211    let mut merged: Vec<(usize, usize)> = Vec::with_capacity(spans.len());
212    for (start, end) in spans {
213        match merged.last_mut() {
214            Some(last) if start <= last.1 => last.1 = last.1.max(end),
215            _ => merged.push((start, end)),
216        }
217    }
218    merged
219}
220
221/// Redacted form of *text*, or None when it holds no secret.
222///
223/// Every secret-looking token is replaced and the rest stays readable. Every
224/// span goes, not just the first: whole-value blanking removed a field's
225/// second and third credentials for free, and scoping redaction to a token
226/// silently dropped that guarantee.
227///
228/// Each match is widened to its whitespace-delimited token first. Redacting
229/// the literal match alone can leave part of a credential behind: the jwt
230/// pattern matches header.payload, and a JWT has THREE dot-separated parts, so
231/// the signature would survive. Whitespace is the boundary a secret cannot
232/// cross without ceasing to be one token.
233///
234/// Returning an Option rather than the string keeps callers to one scan;
235/// asking whether a value held a secret and then redacting it ran the whole
236/// pattern sweep twice for every value carrying a credential.
237pub(crate) fn redact_if_secret(text: &str) -> Option<String> {
238    let spans = secret_spans(text);
239    if spans.is_empty() {
240        return None;
241    }
242    let mut out = String::with_capacity(text.len());
243    let mut previous_end = 0usize;
244    for (start, end) in spans {
245        out.push_str(&text[previous_end..start]);
246        out.push_str(REDACTED);
247        previous_end = end;
248    }
249    out.push_str(&text[previous_end..]);
250    Some(out)
251}
252
253/// The redaction sentinel emitted when a value or string matches a
254/// secret pattern (matches Python's `***` and Go's `piicore.Redacted`).
255pub(crate) const REDACTED_SENTINEL: &str = REDACTED;
256
257/// Register a custom secret detection pattern. If *name* already exists, the
258/// pattern is replaced.
259pub fn register_secret_pattern(name: &str, pattern: Regex) {
260    let mut patterns = crate::_lock::lock(custom_secret_patterns());
261    if let Some((_, existing)) = patterns
262        .iter_mut()
263        .find(|(existing_name, _)| existing_name == name)
264    {
265        *existing = pattern;
266    } else {
267        patterns.push((name.to_string(), pattern));
268    }
269}
270
271/// Return all secret patterns (built-in and custom).
272pub fn get_secret_patterns() -> Vec<SecretPattern> {
273    let mut out: Vec<SecretPattern> = crate::secret_patterns_generated::PATTERNS
274        .iter()
275        .zip(builtin_secret_patterns().iter())
276        .map(|((name, _), p)| SecretPattern {
277            name: name.to_string(),
278            pattern: p.clone(),
279        })
280        .collect();
281    let patterns = crate::_lock::lock(custom_secret_patterns());
282    for (name, pattern) in patterns.iter() {
283        out.push(SecretPattern {
284            name: name.clone(),
285            pattern: pattern.clone(),
286        });
287    }
288    out
289}
290
291/// Reset custom secret patterns — for test isolation only.
292pub fn reset_secret_patterns_for_tests() {
293    crate::_lock::lock(custom_secret_patterns()).clear();
294}
295
296pub fn register_pii_rule(rule: PIIRule) {
297    crate::_lock::lock(rules()).push(rule);
298}
299
300pub fn replace_pii_rules(next: Vec<PIIRule>) {
301    *crate::_lock::lock(rules()) = next;
302}
303
304pub fn get_pii_rules() -> Vec<PIIRule> {
305    crate::_lock::lock(rules()).clone()
306}
307
308fn hash_value(value: &Value) -> String {
309    let mut hasher = Sha256::new();
310    match value {
311        Value::String(text) => hasher.update(text.as_bytes()),
312        _ => hasher.update(value.to_string().as_bytes()),
313    }
314    let digest = hasher.finalize();
315    format!("{:x}", digest)[..12].to_string()
316}
317
318fn mask_value(value: &Value, mode: &PIIMode, truncate_to: usize) -> Option<Value> {
319    match mode {
320        PIIMode::Drop => None,
321        PIIMode::Redact => Some(Value::String(REDACTED.to_string())),
322        PIIMode::Hash => Some(Value::String(hash_value(value))),
323        PIIMode::Truncate => {
324            let text = match value {
325                Value::String(value) => value.clone(),
326                _ => value.to_string(),
327            };
328            let char_count = text.chars().count();
329            if char_count <= truncate_to {
330                Some(Value::String(text))
331            } else {
332                let head: String = text.chars().take(truncate_to).collect();
333                Some(Value::String(format!("{head}{TRUNC_SUFFIX}")))
334            }
335        }
336    }
337}
338
339/// Segment-wise path match: `"*"` in *rule_path* matches any single segment.
340fn match_rule_path(rule_path: &[String], child_path: &[String]) -> bool {
341    if rule_path.len() != child_path.len() {
342        return false;
343    }
344    rule_path
345        .iter()
346        .zip(child_path.iter())
347        .all(|(rp, cp)| rp == "*" || rp == cp)
348}
349
350fn apply_rules(node: &Value, path: &[String], rules: &[PIIRule], max_depth: usize) -> Value {
351    if max_depth == 0 {
352        return node.clone();
353    }
354
355    match node {
356        Value::Object(map) => {
357            let mut out = Map::new();
358            for (key, value) in map {
359                let mut child_path = path.to_vec();
360                child_path.push(key.clone());
361                if let Some(rule) = rules
362                    .iter()
363                    .find(|rule| match_rule_path(&rule.path, &child_path))
364                {
365                    if let Some(masked) = mask_value(value, &rule.mode, rule.truncate_to) {
366                        out.insert(key.clone(), masked);
367                    }
368                    record_redaction(
369                        &child_path.join("."),
370                        &format!("{:?}", rule.mode).to_ascii_lowercase(),
371                        value,
372                    );
373                    continue;
374                }
375
376                let lowered = key.to_ascii_lowercase();
377                let sensitive_key = DEFAULT_SENSITIVE
378                    .iter()
379                    .any(|candidate| candidate == &lowered);
380                // Span-scoped for a secret-bearing string: only the credential
381                // tokens go. A sensitive KEY still blanks wholesale, since
382                // there the whole value is the secret.
383                //
384                // One scan, not two: asking whether the value held a secret
385                // and then redacting it ran the whole pattern sweep twice for
386                // every value carrying a credential. Only a string can hold
387                // one, so no other variant is scanned.
388                let replacement = if sensitive_key {
389                    Some(REDACTED.to_string())
390                } else {
391                    match value {
392                        Value::String(text) => redact_if_secret(text),
393                        _ => None,
394                    }
395                };
396                if let Some(replacement) = replacement {
397                    out.insert(key.clone(), Value::String(replacement));
398                    record_redaction(&child_path.join("."), "redact", value);
399                    continue;
400                }
401
402                out.insert(
403                    key.clone(),
404                    apply_rules(value, &child_path, rules, max_depth - 1),
405                );
406            }
407            Value::Object(out)
408        }
409        // Fix 5: push "*" as path segment when recursing into array elements so
410        // rules like ["users", "*", "email"] can match each element's "email" key.
411        Value::Array(values) => {
412            let mut star_path = path.to_vec();
413            star_path.push("*".to_string());
414            Value::Array(
415                values
416                    .iter()
417                    .map(|value| apply_rules(value, &star_path, rules, max_depth - 1))
418                    .collect(),
419            )
420        }
421        _ => node.clone(),
422    }
423}
424
425fn annotate_governance_classes(cleaned: &mut Value) {
426    let Value::Object(map) = cleaned else {
427        return;
428    };
429    let policy = get_classification_policy();
430    let keys: Vec<String> = map.keys().cloned().collect();
431    for key in keys {
432        if let Some(label) = classify_key(&key) {
433            let action = policy.lookup_action(&label);
434            if action == "drop" {
435                map.remove(&key);
436                continue;
437            }
438            if matches!(action, "redact" | "hash" | "truncate") {
439                let current = map
440                    .get(&key)
441                    .cloned()
442                    .expect("classification key snapshot must still exist");
443                let already_redacted = current.as_str().map(|s| s == REDACTED).unwrap_or(false);
444                if !already_redacted {
445                    let mode = match action {
446                        "redact" => PIIMode::Redact,
447                        "hash" => PIIMode::Hash,
448                        _ => PIIMode::Truncate,
449                    };
450                    let masked = mask_value(&current, &mode, 8)
451                        .expect("classification governance modes never drop values");
452                    map.insert(key.clone(), masked);
453                }
454            }
455            map.insert(format!("__{key}__class"), Value::String(label));
456        }
457    }
458}
459
460pub fn sanitize_payload(payload: &Value, enabled: bool, max_depth: usize) -> Value {
461    if !enabled {
462        return payload.clone();
463    }
464    let rules = get_pii_rules();
465    let mut cleaned = apply_rules(payload, &[], &rules, max_depth.max(1));
466    annotate_governance_classes(&mut cleaned);
467    cleaned
468}
469
470#[cfg(test)]
471#[path = "pii_tests.rs"]
472mod tests;
provide_telemetry/pii.rs

provide_telemetry/
pii.rs
