redact_core/recognizers/
generic.rs

1// Copyright 2026 Censgate LLC.
2// Licensed under the Apache License, Version 2.0. See the LICENSE file
3// in the project root for license information.
4
5//! Context-gated generic secret detection (`GENERIC_SECRET`).
6//!
7//! Assignment-like positions only. Value-only spans. Shared with pack loaders
8//! via [`evaluate_generic_candidate`].
9
10use lazy_static::lazy_static;
11use regex::Regex;
12
13use super::entropy::score_entropy;
14use super::Recognizer;
15use crate::types::{EntityType, RecognizerResult};
16use crate::utf8::{ceil_char_boundary, floor_char_boundary};
17use anyhow::Result;
18
19const MIN_SCORE: f32 = 0.5;
20const UUID_STRONG_CONFIDENCE: f32 = 0.70;
21
22const ALLOW_TOKENS: &[&str] = &[
23    "secret",
24    "token",
25    "password",
26    "passwd",
27    "pwd",
28    "apikey",
29    "credential",
30    "auth",
31    "bearer",
32];
33
34const ALLOW_PAIRS: &[(&str, &str)] = &[
35    ("api", "key"),
36    ("access", "key"),
37    ("access", "token"),
38    ("private", "key"),
39    ("client", "secret"),
40    ("x", "api"),
41];
42
43const DENY_TOKENS: &[&str] = &[
44    "checksum",
45    "integrity",
46    "digest",
47    "sha",
48    "sha1",
49    "sha256",
50    "md5",
51    "etag",
52    "revision",
53    "rev",
54    "commit",
55    "fingerprint",
56    "thumbprint",
57    "tokenizer",
58];
59
60const DENY_PAIRS: &[(&str, &str)] = &[
61    ("key", "id"),
62    ("public", "key"),
63    ("secret", "name"),
64    ("token", "type"),
65];
66
67const STRONG_KEYWORDS: &[&str] = &["api_key", "client_secret", "access_token", "secret"];
68
69const STOPWORDS: &[&str] = &[
70    "password",
71    "changeme",
72    "secret",
73    "xxx",
74    "xxxx",
75    "xxxxx",
76    "test",
77    "example",
78    "todo",
79    "placeholder",
80    "your_api_key_here",
81    "your-key-here",
82    "your_key_here",
83];
84
85lazy_static! {
86    static ref ASSIGNMENT: Regex = Regex::new(
87        r#"(?x)(?i)
88        (?:^|[\s;{,])
89        (?:export\s+|-\s*e\s+)?
90        ["']?
91        (?P<key>[A-Za-z_][A-Za-z0-9_.-]*)
92        ["']?
93        \s*[=:]\s*
94        ["']?
95        (?P<value>[^\s"'<>,;]+)
96        "#
97    )
98    .expect("assignment regex");
99    static ref BEARER: Regex =
100        Regex::new(r#"(?i)(?:^|[\s;])Authorization\s*:\s*Bearer\s+(?P<value>[^\s"'<>,;]+)"#)
101            .expect("bearer regex");
102    static ref UUID_RE: Regex =
103        Regex::new(r"(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
104            .expect("uuid regex");
105    static ref HEX_COLOR: Regex =
106        Regex::new(r"(?i)^#?(?:[0-9a-f]{3}|[0-9a-f]{6}|[0-9a-f]{8})$").expect("hex color regex");
107}
108
109/// How the left-hand side of an assignment should be treated.
110#[derive(Debug, Clone, Copy, PartialEq, Eq)]
111pub enum LhsClass {
112    Secret,
113    Digest,
114    Other,
115}
116
117/// Split a key on `_` / `-` / `.` and camelCase boundaries.
118pub fn segment_key(key: &str) -> Vec<String> {
119    let mut segments = Vec::new();
120    let mut current = String::new();
121    let chars: Vec<char> = key.chars().collect();
122    for (i, &c) in chars.iter().enumerate() {
123        if c == '_' || c == '-' || c == '.' {
124            if !current.is_empty() {
125                segments.push(std::mem::take(&mut current));
126            }
127            continue;
128        }
129        if c.is_ascii_uppercase()
130            && i > 0
131            && (chars[i - 1].is_ascii_lowercase() || chars[i - 1].is_ascii_digit())
132            && !current.is_empty()
133        {
134            segments.push(std::mem::take(&mut current));
135        }
136        current.push(c.to_ascii_lowercase());
137    }
138    if !current.is_empty() {
139        segments.push(current);
140    }
141    segments
142}
143
144fn has_pair(segments: &[String], left: &str, right: &str) -> bool {
145    segments.windows(2).any(|w| w[0] == left && w[1] == right)
146}
147
148fn has_token(segments: &[String], token: &str) -> bool {
149    if token.contains('_') || token.contains('-') {
150        let parts: Vec<&str> = token.split(['_', '-']).collect();
151        if parts.len() == 2 {
152            return has_pair(segments, parts[0], parts[1]);
153        }
154    }
155    segments.iter().any(|s| s == token)
156}
157
158fn env_suffix_is_secret(segments: &[String]) -> bool {
159    let n = segments.len();
160    if n == 0 {
161        return false;
162    }
163    matches!(
164        segments[n - 1].as_str(),
165        "secret" | "token" | "password" | "passwd"
166    ) || (n >= 2
167        && segments[n - 1] == "key"
168        && (segments[n - 2] == "api" || segments[n - 2] == "access"))
169}
170
171/// Classify an assignment key using the closed allow/deny lists.
172pub fn classify_lhs(key: &str) -> LhsClass {
173    let segments = segment_key(key);
174    let allow = ALLOW_TOKENS.iter().any(|t| has_token(&segments, t))
175        || ALLOW_PAIRS.iter().any(|(a, b)| has_pair(&segments, a, b))
176        || env_suffix_is_secret(&segments);
177    if allow {
178        return LhsClass::Secret;
179    }
180    let deny = DENY_TOKENS.iter().any(|t| has_token(&segments, t))
181        || DENY_PAIRS.iter().any(|(a, b)| has_pair(&segments, a, b))
182        || has_token(&segments, "hash");
183    if deny {
184        return LhsClass::Digest;
185    }
186    LhsClass::Other
187}
188
189fn is_strong_keyword(key: &str) -> bool {
190    let segments = segment_key(key);
191    STRONG_KEYWORDS.iter().any(|kw| has_token(&segments, kw))
192}
193
194fn is_stopword(value: &str) -> bool {
195    let lower = value.to_ascii_lowercase();
196    STOPWORDS.contains(&lower.as_str())
197        || lower.starts_with("your_")
198        || lower.starts_with("your-")
199        || lower.ends_with("_here")
200        || lower.ends_with("-here")
201}
202
203fn is_identifier_only(value: &str) -> bool {
204    !value.chars().any(|c| c.is_ascii_digit())
205        && value
206            .chars()
207            .all(|c| c.is_ascii_alphabetic() || matches!(c, '_' | '-' | '.'))
208}
209
210fn looks_like_jwt(value: &str) -> bool {
211    value.starts_with("eyJ") && value.matches('.').count() >= 2
212}
213
214fn looks_like_pem(value: &str) -> bool {
215    value.contains("BEGIN") && value.contains("PRIVATE KEY")
216}
217
218fn surrounding_is_data_uri(surrounding: &str) -> bool {
219    let lower = surrounding.to_ascii_lowercase();
220    lower.contains("data:image") || lower.contains("data:application") || lower.contains("data:")
221}
222
223/// Shared generic-candidate validator used by core and pack loaders.
224///
225/// Packs must pass the same LHS and surrounding context; they cannot score a
226/// bare value and skip the gate.
227pub fn evaluate_generic_candidate(value: &str, lhs: &str, surrounding: &str) -> Option<f32> {
228    let value = value.trim();
229    if value.is_empty() {
230        return None;
231    }
232    if is_stopword(value) {
233        return None;
234    }
235    if is_identifier_only(value) {
236        return None;
237    }
238    if HEX_COLOR.is_match(value) {
239        return None;
240    }
241    if looks_like_jwt(value) || looks_like_pem(value) {
242        return None;
243    }
244    if surrounding_is_data_uri(surrounding) {
245        return None;
246    }
247
248    match classify_lhs(lhs) {
249        LhsClass::Other => None,
250        LhsClass::Digest => None,
251        LhsClass::Secret => {
252            if UUID_RE.is_match(value) {
253                return if is_strong_keyword(lhs) {
254                    Some(UUID_STRONG_CONFIDENCE)
255                } else {
256                    None
257                };
258            }
259            score_entropy(value)
260        }
261    }
262}
263
264/// Context-gated generic secret recognizer.
265#[derive(Debug)]
266pub struct GenericSecretRecognizer {
267    name: String,
268}
269
270impl GenericSecretRecognizer {
271    pub fn new() -> Self {
272        Self {
273            name: "GenericSecretRecognizer".to_string(),
274        }
275    }
276}
277
278impl Default for GenericSecretRecognizer {
279    fn default() -> Self {
280        Self::new()
281    }
282}
283
284impl Recognizer for GenericSecretRecognizer {
285    fn name(&self) -> &str {
286        &self.name
287    }
288
289    fn supported_entities(&self) -> &[EntityType] {
290        &[EntityType::GenericSecret]
291    }
292
293    fn min_score(&self) -> f32 {
294        MIN_SCORE
295    }
296
297    fn analyze(&self, text: &str, _language: &str) -> Result<Vec<RecognizerResult>> {
298        let _span = crate::operations_enabled()
299            .then(|| tracing::info_span!("redact.gateway.detect.patterns").entered());
300        let mut results = Vec::new();
301        for caps in ASSIGNMENT.captures_iter(text) {
302            let Some(key) = caps.name("key") else {
303                continue;
304            };
305            let Some(value) = caps.name("value") else {
306                continue;
307            };
308            if key.as_str().eq_ignore_ascii_case("authorization")
309                && value.as_str().eq_ignore_ascii_case("basic")
310            {
311                continue;
312            }
313            let window_start = floor_char_boundary(text, key.start().saturating_sub(24));
314            let window_end = ceil_char_boundary(text, (value.end() + 24).min(text.len()));
315            let surrounding = &text[window_start..window_end];
316            if let Some(score) =
317                evaluate_generic_candidate(value.as_str(), key.as_str(), surrounding)
318            {
319                if score >= MIN_SCORE {
320                    results.push(
321                        RecognizerResult::new(
322                            EntityType::GenericSecret,
323                            value.start(),
324                            value.end(),
325                            score,
326                            self.name.clone(),
327                        )
328                        .with_text(text),
329                    );
330                }
331            }
332        }
333        for caps in BEARER.captures_iter(text) {
334            let Some(value) = caps.name("value") else {
335                continue;
336            };
337            let window_start = floor_char_boundary(text, value.start().saturating_sub(32));
338            let window_end = ceil_char_boundary(text, (value.end() + 8).min(text.len()));
339            let surrounding = &text[window_start..window_end];
340            if let Some(score) =
341                evaluate_generic_candidate(value.as_str(), "authorization_bearer", surrounding)
342            {
343                if score >= MIN_SCORE {
344                    results.push(
345                        RecognizerResult::new(
346                            EntityType::GenericSecret,
347                            value.start(),
348                            value.end(),
349                            score,
350                            self.name.clone(),
351                        )
352                        .with_text(text),
353                    );
354                }
355            }
356        }
357        Ok(results)
358    }
359}
360
361#[cfg(test)]
362mod tests {
363    use super::*;
364
365    #[test]
366    fn shared_secret_is_secret_lhs() {
367        assert_eq!(classify_lhs("SHARED_SECRET"), LhsClass::Secret);
368        assert_eq!(classify_lhs("shared_secret"), LhsClass::Secret);
369    }
370
371    #[test]
372    fn checksum_is_digest() {
373        assert_eq!(classify_lhs("checksum"), LhsClass::Digest);
374        assert_eq!(classify_lhs("content_sha256"), LhsClass::Digest);
375    }
376
377    #[test]
378    fn password_hash_is_secret() {
379        assert_eq!(classify_lhs("password_hash"), LhsClass::Secret);
380    }
381
382    #[test]
383    fn camel_case_public_key_is_digest() {
384        assert_eq!(classify_lhs("public_key"), LhsClass::Digest);
385        assert_eq!(classify_lhs("publicKey"), LhsClass::Digest);
386        // `token` is an allow token, so it wins over the public_key pair.
387        assert_eq!(classify_lhs("publicKeyToken"), LhsClass::Secret);
388    }
389
390    #[test]
391    fn stopword_rejected() {
392        assert!(evaluate_generic_candidate("password", "api_key", "").is_none());
393        assert!(evaluate_generic_candidate("your-key-here", "api_key", "").is_none());
394    }
395
396    #[test]
397    fn identifier_only_rejected() {
398        assert!(evaluate_generic_candidate("SessionInterface", "token", "").is_none());
399    }
400
401    #[test]
402    fn uuid_under_strong_keyword() {
403        let uuid = "550e8400-e29b-41d4-a716-446655440000";
404        assert!(evaluate_generic_candidate(uuid, "api_key", "").is_some());
405        assert!(evaluate_generic_candidate(uuid, "revision", "").is_none());
406        assert!(evaluate_generic_candidate(uuid, "name", "").is_none());
407    }
408
409    #[test]
410    fn value_only_span() {
411        let rec = GenericSecretRecognizer::new();
412        let secret = "a1b2c3d4e5f60718293a4b5c6d7e8f90";
413        let text = format!("api_key={secret}");
414        let hits = rec.analyze(&text, "en").unwrap();
415        assert_eq!(hits.len(), 1);
416        assert_eq!(hits[0].start, "api_key=".len());
417        assert_eq!(hits[0].end, text.len());
418        assert_eq!(hits[0].text.as_deref(), Some(secret));
419    }
420
421    #[test]
422    fn digest_assignment_emits_nothing() {
423        let rec = GenericSecretRecognizer::new();
424        let text = format!("integrity={}", "a".repeat(64));
425        let hits = rec.analyze(&text, "en").unwrap();
426        assert!(hits.is_empty());
427    }
428
429    #[test]
430    fn screaming_snake_assignment_is_not_a_generic_secret() {
431        let rec = GenericSecretRecognizer::new();
432        let text = "api_key=A1B2C3D4E5F6G7H8I9A_";
433        let hits = rec.analyze(text, "en").unwrap();
434        assert!(
435            hits.is_empty(),
436            "screaming-snake values must be rejected: {hits:?}"
437        );
438    }
439
440    #[test]
441    fn json_quoted_key_is_a_closed_form() {
442        let rec = GenericSecretRecognizer::new();
443        let secret = "a1b2c3d4e5f60718293a4b5c6d7e8f90";
444        let text = format!(r#"{{"client_secret": "{secret}"}}"#);
445        let hits = rec.analyze(&text, "en").unwrap();
446        assert_eq!(hits.len(), 1);
447        assert_eq!(&text[hits[0].start..hits[0].end], secret);
448    }
449}
redact_core/recognizers/generic.rs

redact_core/recognizers/
generic.rs
