redact_core/recognizers/
pattern.rs

1// Copyright 2026 Censgate LLC.
2// Licensed under the Apache License, Version 2.0. See the LICENSE file
3// in the project root for license information.
4
5use super::{validation::validate_entity, Recognizer, RecognizerResult};
6use crate::types::EntityType;
7use anyhow::Result;
8use lazy_static::lazy_static;
9use regex::Regex;
10use std::collections::HashMap;
11
12/// A built-in detection pattern expressed as data.
13///
14/// Secrets are kept in a flat table rather than as imperative registration
15/// calls so the full set can be reviewed at a glance and mirrored by an
16/// external pattern pack without touching detection logic.
17struct SecretPattern {
18    entity_type: EntityType,
19    regex: &'static str,
20    score: f32,
21}
22
23/// Built-in secret/credential detection patterns.
24///
25/// These are anchored/prefixed patterns only (e.g. `AKIA...`, `ghp_...`,
26/// `sk-ant-...`) chosen for high precision. Generic catch-alls like
27/// `api_key=...` or `password=...` are deliberately excluded here;
28/// those go through `GenericSecretRecognizer` (entropy-gated).
29const SECRET_PATTERNS: &[SecretPattern] = &[
30    SecretPattern {
31        entity_type: EntityType::PrivateKey,
32        regex: r"-----BEGIN (?:[A-Z]+ )*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?-----END (?:[A-Z]+ )*PRIVATE KEY(?: BLOCK)?-----",
33        score: 0.98,
34    },
35    SecretPattern {
36        entity_type: EntityType::JwtToken,
37        regex: r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]*",
38        score: 0.90,
39    },
40    SecretPattern {
41        entity_type: EntityType::AwsAccessKey,
42        regex: r"\b(?:AKIA|ASIA|ABIA|ACCA|A3T[A-Z0-9])[0-9A-Z]{16}\b",
43        score: 0.95,
44    },
45    SecretPattern {
46        entity_type: EntityType::AwsAccessKey,
47        // Padding `=` is not a word character, so a trailing `\b` drops the
48        // pad and can emit a truncated token. Capture the full key; the
49        // trailing delimiter is not part of the span.
50        regex: r"\b(ABSK[A-Za-z0-9+/]{109,269}={0,2})(?:\s|$|[^A-Za-z0-9+/=])",
51        score: 0.95,
52    },
53    SecretPattern {
54        entity_type: EntityType::AwsAccessKey,
55        regex: r"\bbedrock-api-key-YmVkcm9jay5hbWF6b25hd3MuY29t\b",
56        score: 0.95,
57    },
58    SecretPattern {
59        entity_type: EntityType::GithubToken,
60        regex: r"\b(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59})\b",
61        score: 0.95,
62    },
63    SecretPattern {
64        entity_type: EntityType::GitlabToken,
65        regex: r"\bglpat-[A-Za-z0-9_-]{20}\b",
66        score: 0.95,
67    },
68    SecretPattern {
69        entity_type: EntityType::GitlabToken,
70        regex: r"\bglpat-[0-9a-zA-Z_-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b",
71        score: 0.95,
72    },
73    SecretPattern {
74        entity_type: EntityType::GitlabToken,
75        regex: r"\bglcbt-[0-9a-zA-Z]{1,5}_[0-9a-zA-Z_-]{20}\b",
76        score: 0.95,
77    },
78    SecretPattern {
79        entity_type: EntityType::GitlabToken,
80        regex: r"\bglagent-[A-Za-z0-9_-]{50}\b",
81        score: 0.95,
82    },
83    SecretPattern {
84        entity_type: EntityType::GitlabToken,
85        regex: r"\bgloas-[A-Za-z0-9_-]{64}\b",
86        score: 0.95,
87    },
88    SecretPattern {
89        entity_type: EntityType::GitlabToken,
90        regex: r"\bgldt-[A-Za-z0-9_-]{20}\b",
91        score: 0.95,
92    },
93    SecretPattern {
94        entity_type: EntityType::GitlabToken,
95        regex: r"\bglft-[A-Za-z0-9_-]{20}\b",
96        score: 0.95,
97    },
98    SecretPattern {
99        entity_type: EntityType::GitlabToken,
100        regex: r"\bglptt-[0-9a-fA-F]{40}\b",
101        score: 0.95,
102    },
103    SecretPattern {
104        entity_type: EntityType::SlackToken,
105        regex: r"\bxox[baprs]-[A-Za-z0-9-]{10,72}\b",
106        score: 0.95,
107    },
108    SecretPattern {
109        entity_type: EntityType::SlackWebhook,
110        regex: r"https://hooks\.slack\.com/services/T[A-Za-z0-9_]+/B[A-Za-z0-9_]+/[A-Za-z0-9_]{20,}",
111        score: 0.95,
112    },
113    SecretPattern {
114        entity_type: EntityType::StripeApiKey,
115        // Secret (`sk_`) and restricted (`rk_`) keys only. Publishable `pk_`
116        // keys are designed to be embedded in client-side code and are not
117        // secret, so redacting them is noise rather than protection.
118        regex: r"\b(?:sk|rk)_(?:live|test)_[0-9a-zA-Z]{24,99}\b",
119        score: 0.95,
120    },
121    SecretPattern {
122        entity_type: EntityType::GoogleApiKey,
123        regex: r"\bAIza[0-9A-Za-z_-]{35}",
124        score: 0.95,
125    },
126    SecretPattern {
127        entity_type: EntityType::OpenAiApiKey,
128        // Two shapes, kept separate so neither has to be loose. Classic keys
129        // are pure alphanumeric, so requiring that rules out ordinary
130        // hyphenated identifiers (`sk-feature-branch-name`) that a combined
131        // `[A-Za-z0-9_-]{20,}` would otherwise match. Project keys do contain
132        // `-`/`_`, so they carry a longer minimum length instead.
133        regex: r"\bsk-(?:proj-[A-Za-z0-9_-]{40,}|[A-Za-z0-9]{32,})",
134        score: 0.90,
135    },
136    SecretPattern {
137        entity_type: EntityType::AnthropicApiKey,
138        regex: r"\bsk-ant-(?:api03-)?[A-Za-z0-9_-]{24,}",
139        score: 0.95,
140    },
141    SecretPattern {
142        entity_type: EntityType::NpmToken,
143        regex: r"\bnpm_[A-Za-z0-9]{36}\b",
144        score: 0.95,
145    },
146    SecretPattern {
147        entity_type: EntityType::PyPiToken,
148        regex: r"\bpypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{50,}",
149        score: 0.95,
150    },
151    SecretPattern {
152        entity_type: EntityType::SendGridApiKey,
153        regex: r"\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}",
154        score: 0.95,
155    },
156    SecretPattern {
157        entity_type: EntityType::TwilioApiKey,
158        regex: r"\bSK[0-9a-fA-F]{32}\b",
159        score: 0.85,
160    },
161    SecretPattern {
162        entity_type: EntityType::TelegramBotToken,
163        regex: r"\b\d{8,10}:AA[A-Za-z0-9_-]{33}",
164        score: 0.95,
165    },
166    SecretPattern {
167        entity_type: EntityType::HashicorpVaultToken,
168        regex: r"\bhv[sbr]\.[A-Za-z0-9_-]{24,}",
169        score: 0.95,
170    },
171    SecretPattern {
172        entity_type: EntityType::DatabaseConnectionString,
173        // The trailing path/query are part of the connection string, so they
174        // are captured too. Without them a redaction leaves `/dbname` and any
175        // query parameters dangling after the placeholder.
176        regex: r"\b(?:mongodb(?:\+srv)?|postgres(?:ql)?|mysql|mariadb|redis|amqp|mssql)://[^:@/\s]+:[^@/\s]+@[^\s/]+(?:/[^\s?#]*)?(?:\?[^\s#]*)?",
177        score: 0.90,
178    },
179    SecretPattern {
180        entity_type: EntityType::HuggingFaceToken,
181        regex: r"\b(?:hf_|api_org_)[A-Za-z]{34}\b",
182        score: 0.95,
183    },
184    SecretPattern {
185        entity_type: EntityType::DatabricksToken,
186        regex: r"\bdapi[0-9a-fA-F]{32}(?:-\d)?\b",
187        score: 0.95,
188    },
189    SecretPattern {
190        entity_type: EntityType::DigitalOceanToken,
191        regex: r"\bdo[por]_v1_[0-9a-fA-F]{64}\b",
192        score: 0.95,
193    },
194    SecretPattern {
195        entity_type: EntityType::NotionApiKey,
196        regex: r"\bntn_[0-9]{11}[A-Za-z0-9]{35}\b",
197        score: 0.95,
198    },
199    SecretPattern {
200        entity_type: EntityType::PerplexityApiKey,
201        regex: r"\bpplx-[A-Za-z0-9]{48}\b",
202        score: 0.95,
203    },
204    SecretPattern {
205        entity_type: EntityType::HttpBasicAuth,
206        // The rust `regex` crate has no lookaround. Padding `=` is not a word
207        // character, so a trailing `\b` misses canonically padded tokens.
208        regex: r"(?i)\bBasic\s+([A-Za-z0-9+/]+={0,2})(?:\s|$|[^A-Za-z0-9+/=])",
209        score: 0.95,
210    },
211];
212
213/// Pattern-based recognizer using regex
214#[derive(Debug, Clone)]
215pub struct PatternRecognizer {
216    name: String,
217    patterns: HashMap<EntityType, Vec<CompiledPattern>>,
218    min_score: f32,
219}
220
221#[derive(Debug, Clone)]
222struct CompiledPattern {
223    regex: Regex,
224    score: f32,
225    context_words: Vec<String>,
226}
227
228impl PatternRecognizer {
229    /// Create a new pattern recognizer with default patterns
230    pub fn new() -> Self {
231        let mut recognizer = Self {
232            name: "PatternRecognizer".to_string(),
233            patterns: HashMap::new(),
234            min_score: 0.5,
235        };
236        recognizer.load_default_patterns();
237        recognizer
238    }
239
240    /// Create a new pattern recognizer with custom name
241    pub fn with_name(name: impl Into<String>) -> Self {
242        let mut recognizer = Self::new();
243        recognizer.name = name.into();
244        recognizer
245    }
246
247    /// Set minimum confidence score
248    pub fn with_min_score(mut self, min_score: f32) -> Self {
249        self.min_score = min_score;
250        self
251    }
252
253    /// Add a custom pattern for an entity type
254    pub fn add_pattern(
255        &mut self,
256        entity_type: EntityType,
257        pattern: &str,
258        score: f32,
259    ) -> Result<()> {
260        let regex = Regex::new(pattern)?;
261        let compiled = CompiledPattern {
262            regex,
263            score,
264            context_words: vec![],
265        };
266        self.patterns.entry(entity_type).or_default().push(compiled);
267        Ok(())
268    }
269
270    /// Add a pattern with context words for score boosting
271    pub fn add_pattern_with_context(
272        &mut self,
273        entity_type: EntityType,
274        pattern: &str,
275        score: f32,
276        context_words: Vec<String>,
277    ) -> Result<()> {
278        let regex = Regex::new(pattern)?;
279        let compiled = CompiledPattern {
280            regex,
281            score,
282            context_words,
283        };
284        self.patterns.entry(entity_type).or_default().push(compiled);
285        Ok(())
286    }
287
288    /// Load default patterns for common PII types
289    fn load_default_patterns(&mut self) {
290        // Email addresses
291        let _ = self.add_pattern(
292            EntityType::EmailAddress,
293            r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b",
294            0.8,
295        );
296
297        // Phone numbers (US/international format with separators)
298        // Requires at least one separator or parentheses to avoid matching
299        // consecutive digits in credit cards, ISBNs, etc.
300        // Matches: (555) 123-4567, 555-123-4567, 555.123.4567, 555 123 4567
301        // Does NOT match: 5551234567 (no separators - too prone to false positives)
302        let _ = self.add_pattern(
303            EntityType::PhoneNumber,
304            r"\(\d{3}\)[-.\s]?\d{3}[-.\s]?\d{4}\b|\b\d{3}[-.\s]\d{3}[-.\s]?\d{4}\b",
305            0.7,
306        );
307
308        // Credit cards (4 groups of 4 digits)
309        let _ = self.add_pattern(
310            EntityType::CreditCard,
311            r"\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12})\b",
312            0.9,
313        );
314
315        // US SSN (simplified pattern - Rust regex doesn't support lookahead)
316        // Pattern matches XXX-XX-XXXX format
317        let _ = self.add_pattern(EntityType::UsSsn, r"\b\d{3}-\d{2}-\d{4}\b", 0.9);
318
319        // IP Address (IPv4)
320        let _ = self.add_pattern(
321            EntityType::IpAddress,
322            r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b",
323            0.8,
324        );
325
326        // URL
327        let _ = self.add_pattern(
328            EntityType::Url,
329            r"\b(?:https?://|www\.)[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z0-9][-a-zA-Z0-9]*)+(?:/[^\s]*)?\b",
330            0.7,
331        );
332
333        // Domain name (standalone, without protocol - avoid overlapping URL)
334        let _ = self.add_pattern(
335            EntityType::DomainName,
336            r"\b(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,}\b",
337            0.7,
338        );
339
340        // GUID/UUID
341        let _ = self.add_pattern(
342            EntityType::Guid,
343            r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b",
344            0.9,
345        );
346
347        // MAC Address
348        let _ = self.add_pattern(
349            EntityType::MacAddress,
350            r"\b(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}\b",
351            0.9,
352        );
353
354        // UK NHS Number
355        let _ = self.add_pattern_with_context(
356            EntityType::UkNhs,
357            r"\b(?:\d{3}\s?\d{3}\s?\d{4}|\d{10})\b",
358            0.6,
359            vec![
360                "NHS".to_string(),
361                "patient".to_string(),
362                "health".to_string(),
363            ],
364        );
365
366        // UK National Insurance Number
367        let _ = self.add_pattern(
368            EntityType::UkNino,
369            r"\b[A-CEGHJ-PR-TW-Z]{1}[A-CEGHJ-NPR-TW-Z]{1}\d{6}[A-D]{1}\b",
370            0.85,
371        );
372
373        // UK Postcode
374        let _ = self.add_pattern(
375            EntityType::UkPostcode,
376            r"\b[A-Z]{1,2}\d[A-Z\d]?\s?\d[A-Z]{2}\b",
377            0.75,
378        );
379
380        // UK Sort Code
381        let _ = self.add_pattern(EntityType::UkSortCode, r"\b\d{2}-\d{2}-\d{2}\b", 0.7);
382
383        // IBAN
384        let _ = self.add_pattern(
385            EntityType::IbanCode,
386            r"\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b",
387            0.75,
388        );
389
390        // Bitcoin Address
391        let _ = self.add_pattern(
392            EntityType::BtcAddress,
393            r"\b(?:bc1|[13])[a-zA-HJ-NP-Z0-9]{25,62}\b",
394            0.85,
395        );
396
397        // Ethereum Address
398        let _ = self.add_pattern(EntityType::EthAddress, r"\b0x[a-fA-F0-9]{40}\b", 0.9);
399
400        // MD5 Hash
401        let _ = self.add_pattern(EntityType::Md5Hash, r"\b[a-fA-F0-9]{32}\b", 0.6);
402
403        // SHA1 Hash
404        let _ = self.add_pattern(EntityType::Sha1Hash, r"\b[a-fA-F0-9]{40}\b", 0.6);
405
406        // SHA256 Hash
407        let _ = self.add_pattern(EntityType::Sha256Hash, r"\b[a-fA-F0-9]{64}\b", 0.6);
408
409        // US ZIP Code (5 digits or ZIP+4 format)
410        let _ = self.add_pattern(
411            EntityType::UsZipCode,
412            r"\b\d{5}(?:-\d{4})?\b",
413            0.6, // Lower confidence as could be other 5-digit numbers
414        );
415
416        // PO Box
417        let _ = self.add_pattern_with_context(
418            EntityType::PoBox,
419            r"\b(?:P\.?\s?O\.?|POST\s+OFFICE)\s*BOX\s+\d+\b",
420            0.85,
421            vec![
422                "address".to_string(),
423                "mail".to_string(),
424                "ship".to_string(),
425            ],
426        );
427
428        // ISBN (10 or 13 digit formats)
429        let _ = self.add_pattern(
430            EntityType::Isbn,
431            r"\b(?:ISBN(?:-1[03])?:?\s*)?(?:\d{9}[\dX]|\d{13})\b",
432            0.8,
433        );
434
435        // Generic Passport Number (alphanumeric, 6-9 characters)
436        let _ = self.add_pattern_with_context(
437            EntityType::PassportNumber,
438            r"\b[A-Z]{1,2}\d{6,9}\b",
439            0.7,
440            vec!["passport".to_string(), "travel".to_string()],
441        );
442
443        // Medical Record Number (various formats with MRN context)
444        let _ = self.add_pattern_with_context(
445            EntityType::MedicalRecordNumber,
446            r"\b(?:MRN|Medical\s*Record|Patient\s*ID):?\s*[A-Z0-9]{6,12}\b",
447            0.85,
448            vec![
449                "patient".to_string(),
450                "medical".to_string(),
451                "hospital".to_string(),
452            ],
453        );
454
455        // Age (with context)
456        let _ = self.add_pattern_with_context(
457            EntityType::Age,
458            r"\b(?:age|aged|years old):?\s*(\d{1,3})\b",
459            0.8,
460            vec!["years".to_string(), "old".to_string(), "age".to_string()],
461        );
462
463        // Date/Time (ISO format and common variants)
464        let _ = self.add_pattern(
465            EntityType::DateTime,
466            r"\b\d{4}-\d{2}-\d{2}(?:[T\s]\d{2}:\d{2}(?::\d{2})?)?\b",
467            0.5,
468        );
469
470        // US Driver's License (varies by state, common formats)
471        // More specific patterns to avoid false positives:
472        // - Letter prefix followed by 6-8 digits (most states)
473        // - State-specific format with dashes
474        // Base score is low (0.4) - requires context to reach min_score
475        let _ = self.add_pattern_with_context(
476            EntityType::UsDriverLicense,
477            r"\b[A-Z]\d{6,8}\b|\b[A-Z]\d{3}-\d{4}-\d{4}\b",
478            0.4,
479            vec![
480                "driver".to_string(),
481                "license".to_string(),
482                "DL".to_string(),
483                "DMV".to_string(),
484            ],
485        );
486
487        // US Passport Number (9 digits, sometimes with letter prefix)
488        // Base score is low - requires context
489        let _ = self.add_pattern_with_context(
490            EntityType::UsPassport,
491            r"\b[A-Z]?\d{9}\b",
492            0.4,
493            vec![
494                "passport".to_string(),
495                "travel".to_string(),
496                "state department".to_string(),
497            ],
498        );
499
500        // US Bank Account Number (typically 8-17 digits)
501        // Very low base score - highly dependent on context
502        let _ = self.add_pattern_with_context(
503            EntityType::UsBankNumber,
504            r"\b\d{8,17}\b",
505            0.3,
506            vec![
507                "account".to_string(),
508                "bank".to_string(),
509                "routing".to_string(),
510                "checking".to_string(),
511                "savings".to_string(),
512            ],
513        );
514
515        // UK Driver's License (DVLA format: 5 letters + 6 digits + 2 letters + 3 digits + 2 letters)
516        // Example: MORGA753116SM9IJ 35
517        let _ = self.add_pattern(
518            EntityType::UkDriverLicense,
519            r"\b[A-Z]{5}\d{6}[A-Z0-9]{2}\d[A-Z]{2}\s?\d{2}\b",
520            0.85,
521        );
522
523        // UK Passport Number (9 digits)
524        // Low base score - requires context to avoid matching random 9-digit numbers
525        let _ = self.add_pattern_with_context(
526            EntityType::UkPassportNumber,
527            r"\b\d{9}\b",
528            0.3,
529            vec![
530                "passport".to_string(),
531                "travel".to_string(),
532                "HMPO".to_string(),
533            ],
534        );
535
536        // UK Phone Number (landline: 01/02/03 prefix)
537        let _ = self.add_pattern(
538            EntityType::UkPhoneNumber,
539            r"\b(?:0[1-3]\d{2,3}\s?\d{3}\s?\d{4}|0[1-3]\d{2,3}\s?\d{6,7})\b",
540            0.75,
541        );
542
543        // UK Mobile Number (07 prefix)
544        let _ = self.add_pattern(
545            EntityType::UkMobileNumber,
546            r"\b07\d{3}\s?\d{3}\s?\d{3}\b",
547            0.8,
548        );
549
550        // UK Company Number (Companies House: 8 digits or 2 letters + 6 digits)
551        // Low base score - requires context to avoid matching random 8-digit numbers
552        let _ = self.add_pattern_with_context(
553            EntityType::UkCompanyNumber,
554            r"\b(?:\d{8}|[A-Z]{2}\d{6})\b",
555            0.3,
556            vec![
557                "company".to_string(),
558                "companies house".to_string(),
559                "registration".to_string(),
560                "CRN".to_string(),
561            ],
562        );
563
564        // Medical License Number (various formats with context)
565        let _ = self.add_pattern_with_context(
566            EntityType::MedicalLicense,
567            r"\b(?:MD|DO|NP|PA|RN|LPN)[-\s]?\d{5,10}\b",
568            0.8,
569            vec![
570                "license".to_string(),
571                "medical".to_string(),
572                "physician".to_string(),
573                "doctor".to_string(),
574                "nurse".to_string(),
575            ],
576        );
577
578        // Generic Crypto Wallet (covers various formats beyond BTC/ETH)
579        // Matches Litecoin (L/M/3), Ripple (r), etc.
580        let _ = self.add_pattern_with_context(
581            EntityType::CryptoWallet,
582            r"\b[LMr3][a-km-zA-HJ-NP-Z1-9]{25,34}\b",
583            0.75,
584            vec![
585                "wallet".to_string(),
586                "crypto".to_string(),
587                "address".to_string(),
588                "coin".to_string(),
589            ],
590        );
591
592        // Secrets and credentials - loaded from the flat data table above.
593        // Panic on compile failure: `let _ = add_pattern` previously swallowed
594        // invalid regexes (the rust `regex` crate has no lookaround).
595        for p in SECRET_PATTERNS {
596            self.add_pattern(p.entity_type.clone(), p.regex, p.score)
597                .unwrap_or_else(|e| {
598                    panic!(
599                        "SECRET_PATTERNS regex failed to compile for {:?}: {e}",
600                        p.entity_type
601                    )
602                });
603        }
604    }
605
606    /// Check context words around a match to boost confidence
607    fn check_context(&self, text: &str, start: usize, end: usize, context_words: &[String]) -> f32 {
608        if context_words.is_empty() {
609            return 0.0;
610        }
611
612        let (context_start, context_end) = context_window(text, start, end, CONTEXT_CHARS);
613        let context = text
614            .get(context_start..context_end)
615            .unwrap_or("")
616            .to_lowercase();
617
618        // Count matching context words
619        let matches = context_words
620            .iter()
621            .filter(|word| context.contains(&word.to_lowercase()))
622            .count();
623
624        // Boost score based on context matches (up to +0.3)
625        (matches as f32 / context_words.len() as f32) * 0.3
626    }
627}
628
629/// Context radius in Unicode scalar values, so non-ASCII text gets the same reach as ASCII.
630const CONTEXT_CHARS: usize = 50;
631
632/// Byte range covering `radius` chars before `start` and after `end`.
633///
634/// Both ends are char boundaries. On ASCII text this is identical to
635/// `start.saturating_sub(radius)..(end + radius).min(text.len())`.
636fn context_window(text: &str, start: usize, end: usize, radius: usize) -> (usize, usize) {
637    let start = crate::floor_char_boundary(text, start);
638    let end = crate::ceil_char_boundary(text, end.max(start));
639    let window_start = text[..start]
640        .char_indices()
641        .rev()
642        .take(radius)
643        .last()
644        .map_or(start, |(index, _)| index);
645    let window_end = text[end..]
646        .char_indices()
647        .nth(radius)
648        .map_or(text.len(), |(index, _)| end + index);
649    (window_start, window_end)
650}
651
652impl Default for PatternRecognizer {
653    fn default() -> Self {
654        Self::new()
655    }
656}
657
658impl Recognizer for PatternRecognizer {
659    fn name(&self) -> &str {
660        &self.name
661    }
662
663    fn supported_entities(&self) -> &[EntityType] {
664        lazy_static! {
665            static ref SUPPORTED: Vec<EntityType> = vec![
666                // Contact information
667                EntityType::EmailAddress,
668                EntityType::PhoneNumber,
669                EntityType::IpAddress,
670                EntityType::Url,
671                EntityType::DomainName,
672                // Financial
673                EntityType::CreditCard,
674                EntityType::IbanCode,
675                EntityType::UsBankNumber,
676                // US-specific
677                EntityType::UsSsn,
678                EntityType::UsDriverLicense,
679                EntityType::UsPassport,
680                EntityType::UsZipCode,
681                // UK-specific
682                EntityType::UkNhs,
683                EntityType::UkNino,
684                EntityType::UkPostcode,
685                EntityType::UkSortCode,
686                EntityType::UkDriverLicense,
687                EntityType::UkPassportNumber,
688                EntityType::UkPhoneNumber,
689                EntityType::UkMobileNumber,
690                EntityType::UkCompanyNumber,
691                // Healthcare
692                EntityType::MedicalLicense,
693                EntityType::MedicalRecordNumber,
694                // Generic identifiers
695                EntityType::PassportNumber,
696                EntityType::Age,
697                EntityType::Isbn,
698                EntityType::PoBox,
699                EntityType::DateTime,
700                // Crypto
701                EntityType::CryptoWallet,
702                EntityType::BtcAddress,
703                EntityType::EthAddress,
704                // Technical
705                EntityType::Guid,
706                EntityType::MacAddress,
707                EntityType::Md5Hash,
708                EntityType::Sha1Hash,
709                EntityType::Sha256Hash,
710                // Secrets and credentials
711                EntityType::PrivateKey,
712                EntityType::JwtToken,
713                EntityType::AwsAccessKey,
714                EntityType::GithubToken,
715                EntityType::GitlabToken,
716                EntityType::SlackToken,
717                EntityType::SlackWebhook,
718                EntityType::StripeApiKey,
719                EntityType::GoogleApiKey,
720                EntityType::OpenAiApiKey,
721                EntityType::AnthropicApiKey,
722                EntityType::NpmToken,
723                EntityType::PyPiToken,
724                EntityType::SendGridApiKey,
725                EntityType::TwilioApiKey,
726                EntityType::TelegramBotToken,
727                EntityType::HashicorpVaultToken,
728                EntityType::DatabaseConnectionString,
729                EntityType::HuggingFaceToken,
730                EntityType::DatabricksToken,
731                EntityType::DigitalOceanToken,
732                EntityType::NotionApiKey,
733                EntityType::PerplexityApiKey,
734                EntityType::HttpBasicAuth,
735            ];
736        }
737        &SUPPORTED
738    }
739
740    fn analyze(&self, text: &str, _language: &str) -> Result<Vec<RecognizerResult>> {
741        let _span = crate::operations_enabled()
742            .then(|| tracing::info_span!("redact.gateway.detect.patterns").entered());
743        let mut results = Vec::new();
744
745        for (entity_type, patterns) in &self.patterns {
746            for pattern in patterns {
747                for capture in pattern.regex.captures_iter(text) {
748                    // Prefer group 1 only for patterns that capture a value-only
749                    // span (HTTP Basic credentials; padded AWS Bedrock keys).
750                    // PII patterns such as AGE also have a group 1 — using it
751                    // globally would shrink those spans to the digits alone.
752                    if let Some(matched) = match entity_type {
753                        EntityType::HttpBasicAuth | EntityType::AwsAccessKey => {
754                            capture.get(1).or_else(|| capture.get(0))
755                        }
756                        _ => capture.get(0),
757                    } {
758                        let start = matched.start();
759                        let end = matched.end();
760                        let matched_text = matched.as_str();
761
762                        // Base score from pattern
763                        let mut score = pattern.score;
764
765                        // Boost score based on context if context words are provided
766                        if !pattern.context_words.is_empty() {
767                            score += self.check_context(text, start, end, &pattern.context_words);
768                            score = score.min(1.0); // Cap at 1.0
769                        }
770
771                        // Apply validation (checksum, format validation)
772                        // This can reduce or zero out the score for invalid matches
773                        let validation_factor = validate_entity(entity_type, matched_text);
774                        score *= validation_factor;
775
776                        if score >= self.min_score {
777                            results.push(
778                                RecognizerResult::new(
779                                    entity_type.clone(),
780                                    start,
781                                    end,
782                                    score,
783                                    self.name(),
784                                )
785                                .with_text(text),
786                            );
787                        }
788                    }
789                }
790            }
791        }
792
793        Ok(results)
794    }
795
796    fn min_score(&self) -> f32 {
797        self.min_score
798    }
799}
800
801#[cfg(test)]
802mod tests {
803    use super::*;
804
805    #[test]
806    fn context_window_matches_legacy_byte_window_on_ascii() {
807        let text: String = (0..300).map(|i| (b'a' + (i % 26) as u8) as char).collect();
808        for start in 0..text.len() {
809            for end in start..(start + 20).min(text.len()) {
810                assert_eq!(
811                    context_window(&text, start, end, 50),
812                    (start.saturating_sub(50), (end + 50).min(text.len())),
813                    "{start}..{end}"
814                );
815            }
816        }
817    }
818
819    #[test]
820    fn context_window_counts_vietnamese_chars_not_bytes() {
821        let text = format!("bank {}841234567890", "ạ ".repeat(20));
822        let start = text.find('8').unwrap();
823        let (window_start, window_end) = context_window(&text, start, start + 12, 50);
824        assert_eq!(text[window_start..start].chars().count(), 45);
825        assert_eq!(window_end, text.len());
826        assert!(text[window_start..window_end].contains("bank"));
827    }
828
829    #[test]
830    fn check_context_does_not_panic_on_multibyte_boundaries() {
831        let recognizer = PatternRecognizer::new();
832        for text in [
833            format!("841234567890{}ó", " ".repeat(49)),
834            format!("ử{}841234567890", " ".repeat(49)),
835            format!("841234567890{}📄", " ".repeat(48)),
836            "Mã đơn hàng 20260916 đã được xử lý thành công. Tổng cộng: 1.250.000 đồng.".to_string(),
837        ] {
838            recognizer.analyze(&text, "en").expect("analyze");
839        }
840    }
841
842    #[test]
843    fn test_age_span_includes_label() {
844        let recognizer = PatternRecognizer::new();
845        let text = "age: 42";
846        let results = recognizer.analyze(text, "en").unwrap();
847        let age = results
848            .iter()
849            .find(|r| r.entity_type == EntityType::Age)
850            .expect("AGE");
851        assert_eq!(&text[age.start..age.end], "age: 42");
852    }
853
854    #[test]
855    fn test_email_detection() {
856        let recognizer = PatternRecognizer::new();
857        let text = "Contact me at john.doe@example.com for details";
858        let results = recognizer.analyze(text, "en").unwrap();
859
860        let email_results: Vec<_> = results
861            .iter()
862            .filter(|r| r.entity_type == EntityType::EmailAddress)
863            .collect();
864        assert_eq!(email_results.len(), 1);
865        assert_eq!(
866            email_results[0].text,
867            Some("john.doe@example.com".to_string())
868        );
869        assert!(email_results[0].score >= 0.8);
870    }
871
872    #[test]
873    fn test_phone_detection() {
874        let recognizer = PatternRecognizer::new();
875        let text = "Call me at (555) 123-4567";
876        let results = recognizer.analyze(text, "en").unwrap();
877
878        assert!(!results.is_empty());
879        let phone_result = results
880            .iter()
881            .find(|r| r.entity_type == EntityType::PhoneNumber);
882        assert!(phone_result.is_some());
883    }
884
885    #[test]
886    fn test_credit_card_detection() {
887        let recognizer = PatternRecognizer::new();
888        let text = "Card number: 4532015112830366";
889        let results = recognizer.analyze(text, "en").unwrap();
890
891        assert!(!results.is_empty());
892        let cc_result = results
893            .iter()
894            .find(|r| r.entity_type == EntityType::CreditCard);
895        assert!(cc_result.is_some());
896    }
897
898    #[test]
899    fn test_ssn_detection() {
900        let recognizer = PatternRecognizer::new();
901        let text = "SSN: 123-45-6789";
902        let results = recognizer.analyze(text, "en").unwrap();
903
904        assert!(!results.is_empty());
905        let ssn_result = results.iter().find(|r| r.entity_type == EntityType::UsSsn);
906        assert!(ssn_result.is_some());
907    }
908
909    #[test]
910    fn test_uk_nhs_with_context() {
911        let recognizer = PatternRecognizer::new();
912        // Use a valid NHS number that passes mod-11 checksum: 943 476 5919
913        // Checksum: 9*10 + 4*9 + 3*8 + 4*7 + 7*6 + 6*5 + 5*4 + 9*3 + 1*2 = 220
914        // 11 - (220 % 11) = 11 - 0 = 11 -> 0, but last digit is 9, so let's use a known valid one
915        // Valid NHS: 401 023 2137 (checksum verified)
916        let text = "NHS patient number is 401 023 2137";
917        let results = recognizer.analyze(text, "en").unwrap();
918
919        assert!(!results.is_empty());
920        let nhs_result = results.iter().find(|r| r.entity_type == EntityType::UkNhs);
921        assert!(
922            nhs_result.is_some(),
923            "Should detect NHS number with context"
924        );
925        // Score should be boosted due to "NHS" context
926        if let Some(result) = nhs_result {
927            assert!(result.score > 0.6);
928        }
929    }
930
931    #[test]
932    fn test_uk_nino_detection() {
933        let recognizer = PatternRecognizer::new();
934        let text = "NINO: AB123456C";
935        let results = recognizer.analyze(text, "en").unwrap();
936
937        assert!(!results.is_empty());
938        let nino_result = results.iter().find(|r| r.entity_type == EntityType::UkNino);
939        assert!(nino_result.is_some());
940    }
941
942    #[test]
943    fn test_multiple_entities() {
944        let recognizer = PatternRecognizer::new();
945        let text = "Email john@example.com, phone (555) 123-4567, SSN 123-45-6789";
946        let results = recognizer.analyze(text, "en").unwrap();
947
948        assert!(results.len() >= 3);
949        assert!(results
950            .iter()
951            .any(|r| r.entity_type == EntityType::EmailAddress));
952        assert!(results
953            .iter()
954            .any(|r| r.entity_type == EntityType::PhoneNumber));
955        assert!(results.iter().any(|r| r.entity_type == EntityType::UsSsn));
956    }
957
958    #[test]
959    fn test_custom_pattern() {
960        let mut recognizer = PatternRecognizer::new();
961        recognizer
962            .add_pattern(
963                EntityType::Custom("CUSTOM_ID".to_string()),
964                r"\bCID-\d{6}\b",
965                0.9,
966            )
967            .unwrap();
968
969        let text = "Your customer ID is CID-123456";
970        let results = recognizer.analyze(text, "en").unwrap();
971
972        let custom_result = results
973            .iter()
974            .find(|r| matches!(r.entity_type, EntityType::Custom(_)));
975        assert!(custom_result.is_some());
976    }
977
978    #[test]
979    fn test_min_score_filtering() {
980        let recognizer = PatternRecognizer::new().with_min_score(0.9);
981        let text = "Date: 2024-01-15"; // Date has score 0.5
982        let results = recognizer.analyze(text, "en").unwrap();
983
984        // Date should be filtered out due to min_score
985        let date_results = results
986            .iter()
987            .filter(|r| r.entity_type == EntityType::DateTime)
988            .count();
989        assert_eq!(date_results, 0);
990    }
991
992    #[test]
993    fn test_uk_driver_license_detection() {
994        let recognizer = PatternRecognizer::new();
995        let text = "UK DL: MORGA753116SM9IJ 35";
996        let results = recognizer.analyze(text, "en").unwrap();
997
998        let dl_result = results
999            .iter()
1000            .find(|r| r.entity_type == EntityType::UkDriverLicense);
1001        assert!(dl_result.is_some(), "Should detect UK driver's license");
1002    }
1003
1004    #[test]
1005    fn test_uk_mobile_detection() {
1006        let recognizer = PatternRecognizer::new();
1007        let text = "Call me on 07700 900123";
1008        let results = recognizer.analyze(text, "en").unwrap();
1009
1010        let mobile_result = results
1011            .iter()
1012            .find(|r| r.entity_type == EntityType::UkMobileNumber);
1013        assert!(mobile_result.is_some(), "Should detect UK mobile number");
1014    }
1015
1016    #[test]
1017    fn test_uk_phone_detection() {
1018        let recognizer = PatternRecognizer::new();
1019        let text = "Office: 0207 123 4567";
1020        let results = recognizer.analyze(text, "en").unwrap();
1021
1022        let phone_result = results
1023            .iter()
1024            .find(|r| r.entity_type == EntityType::UkPhoneNumber);
1025        assert!(phone_result.is_some(), "Should detect UK phone number");
1026    }
1027
1028    #[test]
1029    fn test_medical_license_detection() {
1030        let recognizer = PatternRecognizer::new();
1031        let text = "Medical license: MD-123456789";
1032        let results = recognizer.analyze(text, "en").unwrap();
1033
1034        let license_result = results
1035            .iter()
1036            .find(|r| r.entity_type == EntityType::MedicalLicense);
1037        assert!(license_result.is_some(), "Should detect medical license");
1038    }
1039
1040    #[test]
1041    fn test_supported_entities_count() {
1042        let recognizer = PatternRecognizer::new();
1043        let supported = recognizer.supported_entities();
1044        // 36 original PII + 18 Phase 1 secrets + 6 Phase 2 named types
1045        assert_eq!(
1046            supported.len(),
1047            60,
1048            "Should support 60 pattern-based entity types, got {}",
1049            supported.len()
1050        );
1051    }
1052}
redact_core/recognizers/pattern.rs

redact_core/recognizers/
pattern.rs
