forge_ops_tracker/
pii_scrubber.rs

1// Redacts likely-sensitive content out of a payload before it ever leaves this process: the
2// same patterns ForgeOps itself applies again on arrival (defense in depth: this layer keeps the
3// data off the wire and out of any request logging in between; the server-side layer is what
4// actually protects the database, and doesn't depend on every reporting app running an up-to-date
5// version of this client). Ported from
6// gems/forge_ops_tracker/lib/forge_ops_tracker/pii_scrubber.rb.
7//
8// Can be turned off via Configuration.scrub_pii = false for a host app that already scrubs its
9// own data before it ever reaches error context, or that has its own reasons to want the raw
10// payload. Off by default is not an option: the safe default has to be "on."
11
12use std::collections::HashMap;
13use std::sync::OnceLock;
14
15use regex::Regex;
16
17pub const REDACTED: &str = "[FILTERED]";
18
19/// A JSON-like value: what context/tags are built out of. Hand-rolled rather than depending on
20/// serde_json's Value: this crate already needs regex/backtrace/ureq for things Rust's standard
21/// library genuinely lacks (see README.md's "Dependencies" section), and a fourth dependency
22/// purely for a value type this small isn't worth it.
23#[derive(Clone, Debug, PartialEq)]
24pub enum Value {
25    Null,
26    Bool(bool),
27    Number(f64),
28    String(String),
29    Array(Vec<Value>),
30    Object(HashMap<String, Value>),
31}
32
33impl From<&str> for Value {
34    fn from(v: &str) -> Self {
35        Value::String(v.to_string())
36    }
37}
38impl From<String> for Value {
39    fn from(v: String) -> Self {
40        Value::String(v)
41    }
42}
43impl From<bool> for Value {
44    fn from(v: bool) -> Self {
45        Value::Bool(v)
46    }
47}
48impl From<i64> for Value {
49    fn from(v: i64) -> Self {
50        Value::Number(v as f64)
51    }
52}
53impl From<i32> for Value {
54    fn from(v: i32) -> Self {
55        Value::Number(v as f64)
56    }
57}
58impl From<u64> for Value {
59    fn from(v: u64) -> Self {
60        Value::Number(v as f64)
61    }
62}
63impl From<f64> for Value {
64    fn from(v: f64) -> Self {
65        Value::Number(v)
66    }
67}
68
69impl Value {
70    /// Serializes this value as JSON. Hand-rolled for the same reason Value itself is (see
71    /// above): object key order follows HashMap's own (unspecified, but stable within one
72    /// process) iteration order, which the ingestion API doesn't care about.
73    pub fn to_json(&self) -> String {
74        match self {
75            Value::Null => "null".to_string(),
76            Value::Bool(b) => b.to_string(),
77            Value::Number(n) => {
78                if n.fract() == 0.0 && n.abs() < 1e15 {
79                    format!("{}", *n as i64)
80                } else {
81                    n.to_string()
82                }
83            }
84            Value::String(s) => json_string(s),
85            Value::Array(items) => {
86                let parts: Vec<String> = items.iter().map(Value::to_json).collect();
87                format!("[{}]", parts.join(","))
88            }
89            Value::Object(map) => {
90                let parts: Vec<String> = map
91                    .iter()
92                    .map(|(k, v)| format!("{}:{}", json_string(k), v.to_json()))
93                    .collect();
94                format!("{{{}}}", parts.join(","))
95            }
96        }
97    }
98}
99
100pub fn json_string(s: &str) -> String {
101    let mut out = String::with_capacity(s.len() + 2);
102    out.push('"');
103    for c in s.chars() {
104        match c {
105            '"' => out.push_str("\\\""),
106            '\\' => out.push_str("\\\\"),
107            '\n' => out.push_str("\\n"),
108            '\r' => out.push_str("\\r"),
109            '\t' => out.push_str("\\t"),
110            c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
111            c => out.push(c),
112        }
113    }
114    out.push('"');
115    out
116}
117
118const SENSITIVE_KEYS: &[&str] = &[
119    "password",
120    "passwd",
121    "pwd",
122    "secret",
123    "apisecret",
124    "clientsecret",
125    "secretkey",
126    "token",
127    "accesstoken",
128    "refreshtoken",
129    "apikey",
130    "apitoken",
131    "authorization",
132    "authtoken",
133    "bearer",
134    "sessiontoken",
135    "csrftoken",
136    "creditcard",
137    "cardnumber",
138    "cardnum",
139    "cvv",
140    "cvv2",
141    "cvc",
142    "ssn",
143    "socialsecuritynumber",
144    "socialsecurity",
145    "privatekey",
146];
147
148fn patterns() -> &'static [(&'static str, Regex)] {
149    static PATTERNS: OnceLock<Vec<(&'static str, Regex)>> = OnceLock::new();
150    PATTERNS.get_or_init(|| {
151        vec![
152            (
153                "EMAIL",
154                Regex::new(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}").unwrap(),
155            ),
156            ("SSN", Regex::new(r"\b\d{3}-\d{2}-\d{4}\b").unwrap()),
157            (
158                "CREDIT CARD",
159                Regex::new(r"\b\d{4}[ -]\d{4}[ -]\d{4}[ -]\d{1,4}\b").unwrap(),
160            ),
161            (
162                "BEARER TOKEN",
163                Regex::new(r"(?i)\bBearer\s+[A-Za-z0-9\-._~+/]+=*").unwrap(),
164            ),
165            (
166                "JWT",
167                Regex::new(r"\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b")
168                    .unwrap(),
169            ),
170            ("AWS KEY", Regex::new(r"\bAKIA[0-9A-Z]{16}\b").unwrap()),
171            (
172                "STRIPE KEY",
173                Regex::new(r"\b[sr]k_(?:live|test)_[A-Za-z0-9]{10,}\b").unwrap(),
174            ),
175            (
176                "GITHUB TOKEN",
177                Regex::new(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b").unwrap(),
178            ),
179        ]
180    })
181}
182
183/// Runs every pattern above over a single string, independent of any key: used both directly (a
184/// message, a stack frame's file/method) and as the leaf case of scrub_value below.
185pub fn scrub_string(text: &str) -> String {
186    let mut result = text.to_string();
187    for (label, re) in patterns() {
188        result = re
189            .replace_all(&result, format!("[{label} FILTERED]").as_str())
190            .into_owned();
191    }
192    result
193}
194
195/// Redacts value based on key (an entire value redacted wholesale if key looks sensitive,
196/// regardless of type) and recurses into arrays/objects, matching every other client's behavior
197/// in this repo.
198pub fn scrub_value(value: &Value, key: &str) -> Value {
199    if !matches!(value, Value::Null) && is_sensitive_key(key) {
200        return Value::String(REDACTED.to_string());
201    }
202
203    match value {
204        Value::Object(map) => Value::Object(
205            map.iter()
206                .map(|(k, v)| (k.clone(), scrub_value(v, k)))
207                .collect(),
208        ),
209        Value::Array(items) => Value::Array(items.iter().map(|v| scrub_value(v, key)).collect()),
210        Value::String(s) => Value::String(scrub_string(s)),
211        other => other.clone(),
212    }
213}
214
215fn is_sensitive_key(key: &str) -> bool {
216    if key.is_empty() {
217        return false;
218    }
219    let normalized: String = key
220        .chars()
221        .filter(|c| c.is_ascii_alphanumeric())
222        .map(|c| c.to_ascii_lowercase())
223        .collect();
224    SENSITIVE_KEYS
225        .iter()
226        .any(|sensitive| normalized.contains(sensitive))
227}
228
229#[cfg(test)]
230mod tests {
231    use super::*;
232
233    #[test]
234    fn scrub_string_email() {
235        assert_eq!(
236            scrub_string("contact user@example.com for help"),
237            "contact [EMAIL FILTERED] for help"
238        );
239    }
240
241    #[test]
242    fn scrub_string_credit_card() {
243        assert_ne!(
244            scrub_string("charged card 4242-4242-4242-4242 successfully"),
245            "charged card 4242-4242-4242-4242 successfully"
246        );
247    }
248
249    #[test]
250    fn scrub_string_leaves_ordinary_numeric_id_alone() {
251        let text = "order id 1234567890123456";
252        assert_eq!(scrub_string(text), text);
253    }
254
255    #[test]
256    fn scrub_string_ssn() {
257        assert_eq!(
258            scrub_string("ssn on file: 123-45-6789"),
259            "ssn on file: [SSN FILTERED]"
260        );
261    }
262
263    #[test]
264    fn scrub_string_known_token_formats() {
265        for text in [
266            "Authorization: Bearer abc123DEF.456-xyz",
267            "aws key AKIAABCDEFGHIJKLMNOP in use",
268            "stripe key sk_live_abcdefghijklmnop",
269            "github token ghp_abcdefghijklmnopqrstuvwxyz0123456789",
270            "jwt eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dQw4w9WgXcQ",
271        ] {
272            assert_ne!(scrub_string(text), text, "expected {text:?} to be redacted");
273        }
274    }
275
276    #[test]
277    fn scrub_value_redacts_whole_value_under_sensitive_key() {
278        assert_eq!(
279            scrub_value(&Value::Number(12345.0), "apiKey"),
280            Value::String(REDACTED.to_string())
281        );
282    }
283
284    #[test]
285    fn scrub_value_recurses_into_objects_and_arrays() {
286        let mut input = HashMap::new();
287        input.insert("password".to_string(), Value::String("hunter2".to_string()));
288        input.insert(
289            "note".to_string(),
290            Value::String("email me at user@example.com".to_string()),
291        );
292
293        let mut nested = HashMap::new();
294        nested.insert("token".to_string(), Value::String("abc".to_string()));
295        input.insert(
296            "items".to_string(),
297            Value::Array(vec![
298                Value::Object(nested),
299                Value::String("visit user@example.com".to_string()),
300            ]),
301        );
302
303        let scrubbed = scrub_value(&Value::Object(input), "");
304        let Value::Object(map) = scrubbed else {
305            panic!("expected an object")
306        };
307
308        assert_eq!(map["password"], Value::String(REDACTED.to_string()));
309        assert_eq!(
310            map["note"],
311            Value::String("email me at [EMAIL FILTERED]".to_string())
312        );
313
314        let Value::Array(items) = &map["items"] else {
315            panic!("expected an array")
316        };
317        let Value::Object(first) = &items[0] else {
318            panic!("expected an object")
319        };
320        assert_eq!(first["token"], Value::String(REDACTED.to_string()));
321        assert_eq!(
322            items[1],
323            Value::String("visit [EMAIL FILTERED]".to_string())
324        );
325    }
326
327    #[test]
328    fn is_sensitive_key_ignores_case_and_punctuation() {
329        for key in ["API_KEY", "Api-Key", "apiKey", "X-Api-Key"] {
330            assert!(is_sensitive_key(key), "expected {key:?} to be sensitive");
331        }
332        assert!(!is_sensitive_key("username"));
333    }
334
335    #[test]
336    fn value_to_json() {
337        let mut map = HashMap::new();
338        map.insert("a".to_string(), Value::Number(1.0));
339        let value = Value::Object(map);
340        assert_eq!(value.to_json(), "{\"a\":1}");
341
342        assert_eq!(
343            Value::String("he said \"hi\"".to_string()).to_json(),
344            "\"he said \\\"hi\\\"\""
345        );
346        assert_eq!(
347            Value::Array(vec![Value::Bool(true), Value::Null]).to_json(),
348            "[true,null]"
349        );
350    }
351}
forge_ops_tracker/pii_scrubber.rs

forge_ops_tracker/
pii_scrubber.rs
