redact_core/recognizers/
generic.rs

1// Copyright 2026 Censgate LLC.
2// Licensed under the Apache License, Version 2.0. See the LICENSE file
3// in the project root for license information.
4
5//! Context-gated generic secret detection (`GENERIC_SECRET`).
6//!
7//! Assignment-like positions only. Value-only spans. Shared with pack loaders
8//! via [`evaluate_generic_candidate`].
9
10use lazy_static::lazy_static;
11use regex::Regex;
12
13use super::entropy::score_entropy;
14use super::Recognizer;
15use crate::types::{EntityType, RecognizerResult};
16use anyhow::Result;
17
18const MIN_SCORE: f32 = 0.5;
19const UUID_STRONG_CONFIDENCE: f32 = 0.70;
20
21const ALLOW_TOKENS: &[&str] = &[
22    "secret",
23    "token",
24    "password",
25    "passwd",
26    "pwd",
27    "apikey",
28    "credential",
29    "auth",
30    "bearer",
31];
32
33const ALLOW_PAIRS: &[(&str, &str)] = &[
34    ("api", "key"),
35    ("access", "key"),
36    ("access", "token"),
37    ("private", "key"),
38    ("client", "secret"),
39    ("x", "api"),
40];
41
42const DENY_TOKENS: &[&str] = &[
43    "checksum",
44    "integrity",
45    "digest",
46    "sha",
47    "sha1",
48    "sha256",
49    "md5",
50    "etag",
51    "revision",
52    "rev",
53    "commit",
54    "fingerprint",
55    "thumbprint",
56    "tokenizer",
57];
58
59const DENY_PAIRS: &[(&str, &str)] = &[
60    ("key", "id"),
61    ("public", "key"),
62    ("secret", "name"),
63    ("token", "type"),
64];
65
66const STRONG_KEYWORDS: &[&str] = &["api_key", "client_secret", "access_token", "secret"];
67
68const STOPWORDS: &[&str] = &[
69    "password",
70    "changeme",
71    "secret",
72    "xxx",
73    "xxxx",
74    "xxxxx",
75    "test",
76    "example",
77    "todo",
78    "placeholder",
79    "your_api_key_here",
80    "your-key-here",
81    "your_key_here",
82];
83
84lazy_static! {
85    static ref ASSIGNMENT: Regex = Regex::new(
86        r#"(?x)(?i)
87        (?:^|[\s;{,])
88        (?:export\s+|-\s*e\s+)?
89        ["']?
90        (?P<key>[A-Za-z_][A-Za-z0-9_.-]*)
91        ["']?
92        \s*[=:]\s*
93        ["']?
94        (?P<value>[^\s"'<>,;]+)
95        "#
96    )
97    .expect("assignment regex");
98    static ref BEARER: Regex =
99        Regex::new(r#"(?i)(?:^|[\s;])Authorization\s*:\s*Bearer\s+(?P<value>[^\s"'<>,;]+)"#)
100            .expect("bearer regex");
101    static ref UUID_RE: Regex =
102        Regex::new(r"(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
103            .expect("uuid regex");
104    static ref HEX_COLOR: Regex =
105        Regex::new(r"(?i)^#?(?:[0-9a-f]{3}|[0-9a-f]{6}|[0-9a-f]{8})$").expect("hex color regex");
106}
107
108/// How the left-hand side of an assignment should be treated.
109#[derive(Debug, Clone, Copy, PartialEq, Eq)]
110pub enum LhsClass {
111    Secret,
112    Digest,
113    Other,
114}
115
116/// Split a key on `_` / `-` / `.` and camelCase boundaries.
117pub fn segment_key(key: &str) -> Vec<String> {
118    let mut segments = Vec::new();
119    let mut current = String::new();
120    let chars: Vec<char> = key.chars().collect();
121    for (i, &c) in chars.iter().enumerate() {
122        if c == '_' || c == '-' || c == '.' {
123            if !current.is_empty() {
124                segments.push(std::mem::take(&mut current));
125            }
126            continue;
127        }
128        if c.is_ascii_uppercase()
129            && i > 0
130            && (chars[i - 1].is_ascii_lowercase() || chars[i - 1].is_ascii_digit())
131            && !current.is_empty()
132        {
133            segments.push(std::mem::take(&mut current));
134        }
135        current.push(c.to_ascii_lowercase());
136    }
137    if !current.is_empty() {
138        segments.push(current);
139    }
140    segments
141}
142
143fn has_pair(segments: &[String], left: &str, right: &str) -> bool {
144    segments.windows(2).any(|w| w[0] == left && w[1] == right)
145}
146
147fn has_token(segments: &[String], token: &str) -> bool {
148    if token.contains('_') || token.contains('-') {
149        let parts: Vec<&str> = token.split(['_', '-']).collect();
150        if parts.len() == 2 {
151            return has_pair(segments, parts[0], parts[1]);
152        }
153    }
154    segments.iter().any(|s| s == token)
155}
156
157fn env_suffix_is_secret(segments: &[String]) -> bool {
158    let n = segments.len();
159    if n == 0 {
160        return false;
161    }
162    matches!(
163        segments[n - 1].as_str(),
164        "secret" | "token" | "password" | "passwd"
165    ) || (n >= 2
166        && segments[n - 1] == "key"
167        && (segments[n - 2] == "api" || segments[n - 2] == "access"))
168}
169
170/// Classify an assignment key using the closed allow/deny lists.
171pub fn classify_lhs(key: &str) -> LhsClass {
172    let segments = segment_key(key);
173    let allow = ALLOW_TOKENS.iter().any(|t| has_token(&segments, t))
174        || ALLOW_PAIRS.iter().any(|(a, b)| has_pair(&segments, a, b))
175        || env_suffix_is_secret(&segments);
176    if allow {
177        return LhsClass::Secret;
178    }
179    let deny = DENY_TOKENS.iter().any(|t| has_token(&segments, t))
180        || DENY_PAIRS.iter().any(|(a, b)| has_pair(&segments, a, b))
181        || has_token(&segments, "hash");
182    if deny {
183        return LhsClass::Digest;
184    }
185    LhsClass::Other
186}
187
188fn is_strong_keyword(key: &str) -> bool {
189    let segments = segment_key(key);
190    STRONG_KEYWORDS.iter().any(|kw| has_token(&segments, kw))
191}
192
193fn is_stopword(value: &str) -> bool {
194    let lower = value.to_ascii_lowercase();
195    STOPWORDS.contains(&lower.as_str())
196        || lower.starts_with("your_")
197        || lower.starts_with("your-")
198        || lower.ends_with("_here")
199        || lower.ends_with("-here")
200}
201
202fn is_identifier_only(value: &str) -> bool {
203    !value.chars().any(|c| c.is_ascii_digit())
204        && value
205            .chars()
206            .all(|c| c.is_ascii_alphabetic() || matches!(c, '_' | '-' | '.'))
207}
208
209fn looks_like_jwt(value: &str) -> bool {
210    value.starts_with("eyJ") && value.matches('.').count() >= 2
211}
212
213fn looks_like_pem(value: &str) -> bool {
214    value.contains("BEGIN") && value.contains("PRIVATE KEY")
215}
216
217fn surrounding_is_data_uri(surrounding: &str) -> bool {
218    let lower = surrounding.to_ascii_lowercase();
219    lower.contains("data:image") || lower.contains("data:application") || lower.contains("data:")
220}
221
222/// Shared generic-candidate validator used by core and pack loaders.
223///
224/// Packs must pass the same LHS and surrounding context; they cannot score a
225/// bare value and skip the gate.
226pub fn evaluate_generic_candidate(value: &str, lhs: &str, surrounding: &str) -> Option<f32> {
227    let value = value.trim();
228    if value.is_empty() {
229        return None;
230    }
231    if is_stopword(value) {
232        return None;
233    }
234    if is_identifier_only(value) {
235        return None;
236    }
237    if HEX_COLOR.is_match(value) {
238        return None;
239    }
240    if looks_like_jwt(value) || looks_like_pem(value) {
241        return None;
242    }
243    if surrounding_is_data_uri(surrounding) {
244        return None;
245    }
246
247    match classify_lhs(lhs) {
248        LhsClass::Other => None,
249        LhsClass::Digest => None,
250        LhsClass::Secret => {
251            if UUID_RE.is_match(value) {
252                return if is_strong_keyword(lhs) {
253                    Some(UUID_STRONG_CONFIDENCE)
254                } else {
255                    None
256                };
257            }
258            score_entropy(value)
259        }
260    }
261}
262
263fn floor_char_boundary(s: &str, mut i: usize) -> usize {
264    while i > 0 && !s.is_char_boundary(i) {
265        i -= 1;
266    }
267    i
268}
269
270fn ceil_char_boundary(s: &str, mut i: usize) -> usize {
271    while i < s.len() && !s.is_char_boundary(i) {
272        i += 1;
273    }
274    i
275}
276
277/// Context-gated generic secret recognizer.
278#[derive(Debug)]
279pub struct GenericSecretRecognizer {
280    name: String,
281}
282
283impl GenericSecretRecognizer {
284    pub fn new() -> Self {
285        Self {
286            name: "GenericSecretRecognizer".to_string(),
287        }
288    }
289}
290
291impl Default for GenericSecretRecognizer {
292    fn default() -> Self {
293        Self::new()
294    }
295}
296
297impl Recognizer for GenericSecretRecognizer {
298    fn name(&self) -> &str {
299        &self.name
300    }
301
302    fn supported_entities(&self) -> &[EntityType] {
303        &[EntityType::GenericSecret]
304    }
305
306    fn min_score(&self) -> f32 {
307        MIN_SCORE
308    }
309
310    fn analyze(&self, text: &str, _language: &str) -> Result<Vec<RecognizerResult>> {
311        let _span = crate::operations_enabled()
312            .then(|| tracing::info_span!("redact.gateway.detect.patterns").entered());
313        let mut results = Vec::new();
314        for caps in ASSIGNMENT.captures_iter(text) {
315            let Some(key) = caps.name("key") else {
316                continue;
317            };
318            let Some(value) = caps.name("value") else {
319                continue;
320            };
321            if key.as_str().eq_ignore_ascii_case("authorization")
322                && value.as_str().eq_ignore_ascii_case("basic")
323            {
324                continue;
325            }
326            let window_start = floor_char_boundary(text, key.start().saturating_sub(24));
327            let window_end = ceil_char_boundary(text, (value.end() + 24).min(text.len()));
328            let surrounding = &text[window_start..window_end];
329            if let Some(score) =
330                evaluate_generic_candidate(value.as_str(), key.as_str(), surrounding)
331            {
332                if score >= MIN_SCORE {
333                    results.push(
334                        RecognizerResult::new(
335                            EntityType::GenericSecret,
336                            value.start(),
337                            value.end(),
338                            score,
339                            self.name.clone(),
340                        )
341                        .with_text(text),
342                    );
343                }
344            }
345        }
346        for caps in BEARER.captures_iter(text) {
347            let Some(value) = caps.name("value") else {
348                continue;
349            };
350            let window_start = floor_char_boundary(text, value.start().saturating_sub(32));
351            let window_end = ceil_char_boundary(text, (value.end() + 8).min(text.len()));
352            let surrounding = &text[window_start..window_end];
353            if let Some(score) =
354                evaluate_generic_candidate(value.as_str(), "authorization_bearer", surrounding)
355            {
356                if score >= MIN_SCORE {
357                    results.push(
358                        RecognizerResult::new(
359                            EntityType::GenericSecret,
360                            value.start(),
361                            value.end(),
362                            score,
363                            self.name.clone(),
364                        )
365                        .with_text(text),
366                    );
367                }
368            }
369        }
370        Ok(results)
371    }
372}
373
374#[cfg(test)]
375mod tests {
376    use super::*;
377
378    #[test]
379    fn shared_secret_is_secret_lhs() {
380        assert_eq!(classify_lhs("SHARED_SECRET"), LhsClass::Secret);
381        assert_eq!(classify_lhs("shared_secret"), LhsClass::Secret);
382    }
383
384    #[test]
385    fn checksum_is_digest() {
386        assert_eq!(classify_lhs("checksum"), LhsClass::Digest);
387        assert_eq!(classify_lhs("content_sha256"), LhsClass::Digest);
388    }
389
390    #[test]
391    fn password_hash_is_secret() {
392        assert_eq!(classify_lhs("password_hash"), LhsClass::Secret);
393    }
394
395    #[test]
396    fn camel_case_public_key_is_digest() {
397        assert_eq!(classify_lhs("public_key"), LhsClass::Digest);
398        assert_eq!(classify_lhs("publicKey"), LhsClass::Digest);
399        // `token` is an allow token, so it wins over the public_key pair.
400        assert_eq!(classify_lhs("publicKeyToken"), LhsClass::Secret);
401    }
402
403    #[test]
404    fn stopword_rejected() {
405        assert!(evaluate_generic_candidate("password", "api_key", "").is_none());
406        assert!(evaluate_generic_candidate("your-key-here", "api_key", "").is_none());
407    }
408
409    #[test]
410    fn identifier_only_rejected() {
411        assert!(evaluate_generic_candidate("SessionInterface", "token", "").is_none());
412    }
413
414    #[test]
415    fn uuid_under_strong_keyword() {
416        let uuid = "550e8400-e29b-41d4-a716-446655440000";
417        assert!(evaluate_generic_candidate(uuid, "api_key", "").is_some());
418        assert!(evaluate_generic_candidate(uuid, "revision", "").is_none());
419        assert!(evaluate_generic_candidate(uuid, "name", "").is_none());
420    }
421
422    #[test]
423    fn value_only_span() {
424        let rec = GenericSecretRecognizer::new();
425        let secret = "a1b2c3d4e5f60718293a4b5c6d7e8f90";
426        let text = format!("api_key={secret}");
427        let hits = rec.analyze(&text, "en").unwrap();
428        assert_eq!(hits.len(), 1);
429        assert_eq!(hits[0].start, "api_key=".len());
430        assert_eq!(hits[0].end, text.len());
431        assert_eq!(hits[0].text.as_deref(), Some(secret));
432    }
433
434    #[test]
435    fn digest_assignment_emits_nothing() {
436        let rec = GenericSecretRecognizer::new();
437        let text = format!("integrity={}", "a".repeat(64));
438        let hits = rec.analyze(&text, "en").unwrap();
439        assert!(hits.is_empty());
440    }
441
442    #[test]
443    fn screaming_snake_assignment_is_not_a_generic_secret() {
444        let rec = GenericSecretRecognizer::new();
445        let text = "api_key=A1B2C3D4E5F6G7H8I9A_";
446        let hits = rec.analyze(text, "en").unwrap();
447        assert!(
448            hits.is_empty(),
449            "screaming-snake values must be rejected: {hits:?}"
450        );
451    }
452
453    #[test]
454    fn json_quoted_key_is_a_closed_form() {
455        let rec = GenericSecretRecognizer::new();
456        let secret = "a1b2c3d4e5f60718293a4b5c6d7e8f90";
457        let text = format!(r#"{{"client_secret": "{secret}"}}"#);
458        let hits = rec.analyze(&text, "en").unwrap();
459        assert_eq!(hits.len(), 1);
460        assert_eq!(&text[hits[0].start..hits[0].end], secret);
461    }
462}
redact_core/recognizers/generic.rs

redact_core/recognizers/
generic.rs
