1use std::collections::{BTreeMap, HashMap};
46use std::fmt;
47
48use regex::Regex;
49
50use crate::document::{InlineRun, Node, Table};
51use crate::tree::{ItemTree, TreeKind};
52use crate::DoclingDocument;
53
54#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
59pub enum PiiKind {
60 Email,
61 CreditCard,
62 Iban,
63 NationalId,
64 IpAddress,
65 UrlCredentials,
66 Phone,
67 Person,
68 Organization,
69 Location,
70 Address,
71 Custom,
73}
74
75impl PiiKind {
76 pub const ALL: [PiiKind; 12] = [
78 PiiKind::Email,
79 PiiKind::CreditCard,
80 PiiKind::Iban,
81 PiiKind::NationalId,
82 PiiKind::IpAddress,
83 PiiKind::UrlCredentials,
84 PiiKind::Phone,
85 PiiKind::Person,
86 PiiKind::Organization,
87 PiiKind::Location,
88 PiiKind::Address,
89 PiiKind::Custom,
90 ];
91
92 pub const PATTERN: [PiiKind; 7] = [
94 PiiKind::Email,
95 PiiKind::CreditCard,
96 PiiKind::Iban,
97 PiiKind::NationalId,
98 PiiKind::IpAddress,
99 PiiKind::UrlCredentials,
100 PiiKind::Phone,
101 ];
102
103 pub const NER: [PiiKind; 4] = [
105 PiiKind::Person,
106 PiiKind::Organization,
107 PiiKind::Location,
108 PiiKind::Address,
109 ];
110
111 pub fn label(self) -> &'static str {
113 match self {
114 PiiKind::Email => "EMAIL",
115 PiiKind::Phone => "PHONE",
116 PiiKind::CreditCard => "CREDIT_CARD",
117 PiiKind::Iban => "IBAN",
118 PiiKind::IpAddress => "IP",
119 PiiKind::UrlCredentials => "CREDENTIALS",
120 PiiKind::NationalId => "NATIONAL_ID",
121 PiiKind::Person => "PERSON",
122 PiiKind::Organization => "ORG",
123 PiiKind::Location => "LOCATION",
124 PiiKind::Address => "ADDRESS",
125 PiiKind::Custom => "REDACTED",
126 }
127 }
128
129 pub fn name(self) -> &'static str {
131 match self {
132 PiiKind::Email => "email",
133 PiiKind::Phone => "phone",
134 PiiKind::CreditCard => "credit_card",
135 PiiKind::Iban => "iban",
136 PiiKind::IpAddress => "ip_address",
137 PiiKind::UrlCredentials => "url_credentials",
138 PiiKind::NationalId => "national_id",
139 PiiKind::Person => "person",
140 PiiKind::Organization => "organization",
141 PiiKind::Location => "location",
142 PiiKind::Address => "address",
143 PiiKind::Custom => "custom",
144 }
145 }
146
147 pub fn parse(s: &str) -> Option<Self> {
150 let s = s.trim().to_ascii_lowercase().replace('-', "_");
151 match s.as_str() {
152 "ip" => return Some(PiiKind::IpAddress),
153 "org" => return Some(PiiKind::Organization),
154 "credentials" | "url_credential" => return Some(PiiKind::UrlCredentials),
155 _ => {}
156 }
157 PiiKind::ALL.into_iter().find(|k| k.name() == s)
158 }
159
160 pub const ACCEPTED: &'static str = "email, phone, credit_card, iban, ip_address, \
162 url_credentials, national_id, person, organization, location, address, custom";
163}
164
165#[derive(Debug, Clone, PartialEq, Eq, Default)]
167pub enum Replacement {
168 #[default]
170 Label,
171 Pseudonym,
176 Fixed(String),
178}
179
180impl Replacement {
181 pub fn parse(s: &str) -> Option<Self> {
183 let t = s.trim();
184 if t.eq_ignore_ascii_case("label") {
185 return Some(Replacement::Label);
186 }
187 if t.eq_ignore_ascii_case("pseudonym") {
188 return Some(Replacement::Pseudonym);
189 }
190 let lower = t.to_ascii_lowercase();
191 if let Some(rest) = lower.strip_prefix("fixed:") {
192 let text = &t[t.len() - rest.len()..];
194 return Some(Replacement::Fixed(text.to_string()));
195 }
196 None
197 }
198
199 pub const ACCEPTED: &'static str = "label, pseudonym, fixed:<text>";
200}
201
202#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
206pub enum ImageRedaction {
207 #[default]
210 Drop,
211 BoxOut,
216 Keep,
218}
219
220impl ImageRedaction {
221 pub fn parse(s: &str) -> Option<Self> {
223 match s.trim().to_ascii_lowercase().replace('-', "_").as_str() {
224 "drop" => Some(ImageRedaction::Drop),
225 "box_out" | "boxout" | "box" => Some(ImageRedaction::BoxOut),
226 "keep" => Some(ImageRedaction::Keep),
227 _ => None,
228 }
229 }
230
231 pub const ACCEPTED: &'static str = "drop, box_out, keep";
232}
233
234#[derive(Debug, Clone, PartialEq, Eq)]
239pub struct CustomPattern {
240 pub name: String,
241 pub regex: String,
242}
243
244impl CustomPattern {
245 pub fn parse(s: &str) -> Option<Self> {
248 let s = s.trim();
249 if s.is_empty() {
250 return None;
251 }
252 let (name, regex) = match s.split_once('=') {
253 Some((n, r))
256 if !n.is_empty()
257 && n.chars()
258 .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') =>
259 {
260 (n.to_string(), r.to_string())
261 }
262 _ => ("custom".to_string(), s.to_string()),
263 };
264 Some(CustomPattern { name, regex })
265 }
266}
267
268#[derive(Debug, Clone, PartialEq)]
271pub struct RedactionOptions {
272 pub kinds: Vec<PiiKind>,
275 pub replacement: Replacement,
276 pub custom_patterns: Vec<CustomPattern>,
278 pub deny_terms: Vec<String>,
281 pub allow_terms: Vec<String>,
284 pub allow_patterns: Vec<String>,
289 pub ner_min_score: f32,
293 pub images: ImageRedaction,
294 pub return_mapping: bool,
298}
299
300impl Default for RedactionOptions {
301 fn default() -> Self {
302 Self {
303 kinds: Vec::new(),
304 replacement: Replacement::Label,
305 custom_patterns: Vec::new(),
306 deny_terms: Vec::new(),
307 allow_terms: Vec::new(),
308 allow_patterns: Vec::new(),
309 ner_min_score: 0.85,
310 images: ImageRedaction::Drop,
311 return_mapping: false,
312 }
313 }
314}
315
316impl RedactionOptions {
317 pub fn wants(&self, kind: PiiKind) -> bool {
319 self.kinds.is_empty() || self.kinds.contains(&kind)
320 }
321
322 pub fn wants_ner(&self) -> bool {
324 PiiKind::NER.iter().any(|&k| self.wants(k))
325 }
326}
327
328#[derive(Debug, Clone, PartialEq)]
332pub struct Span {
333 pub start: usize,
334 pub end: usize,
335 pub kind: PiiKind,
336 pub score: f32,
337 pub name: Option<String>,
338}
339
340impl Span {
341 pub fn new(start: usize, end: usize, kind: PiiKind) -> Self {
342 Span {
343 start,
344 end,
345 kind,
346 score: 1.0,
347 name: None,
348 }
349 }
350}
351
352pub trait PiiDetector {
356 fn detect(&self, text: &str) -> Vec<Span>;
357}
358
359pub struct CompositeDetector(pub Vec<Box<dyn PiiDetector + Send + Sync>>);
362
363impl PiiDetector for CompositeDetector {
364 fn detect(&self, text: &str) -> Vec<Span> {
365 self.0.iter().flat_map(|d| d.detect(text)).collect()
366 }
367}
368
369#[derive(Debug, Clone, PartialEq, Eq, Default)]
375pub struct RedactionReport {
376 pub counts: BTreeMap<String, usize>,
377 pub total: usize,
378 pub mapping: Option<Vec<(String, String)>>,
379}
380
381impl RedactionReport {
382 pub fn counts_json(&self) -> serde_json::Value {
385 serde_json::Value::Object(
386 self.counts
387 .iter()
388 .map(|(k, v)| (k.clone(), serde_json::json!(v)))
389 .collect(),
390 )
391 }
392}
393
394#[derive(Debug, Clone, PartialEq, Eq)]
396pub struct RedactionError(pub String);
397
398impl fmt::Display for RedactionError {
399 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
400 f.write_str(&self.0)
401 }
402}
403
404impl std::error::Error for RedactionError {}
405
406macro_rules! re {
411 ($pat:expr) => {{
412 static RE: std::sync::OnceLock<Regex> = std::sync::OnceLock::new();
413 RE.get_or_init(|| Regex::new($pat).expect("static regex"))
414 }};
415}
416
417#[derive(Debug, Default)]
422pub struct AllowList {
423 terms: Vec<String>,
424 patterns: Vec<Regex>,
425}
426
427impl AllowList {
428 pub fn new(opts: &RedactionOptions) -> Result<Self, RedactionError> {
431 let terms = opts
432 .allow_terms
433 .iter()
434 .map(|t| t.trim().to_lowercase())
435 .filter(|t| !t.is_empty())
436 .collect();
437 let mut patterns = Vec::new();
438 for p in opts
439 .allow_patterns
440 .iter()
441 .map(|p| p.trim())
442 .filter(|p| !p.is_empty())
443 {
444 let re = Regex::new(&format!("^(?:{p})$"))
445 .map_err(|e| RedactionError(format!("redact allow pattern {p:?}: {e}")))?;
446 patterns.push(re);
447 }
448 Ok(Self { terms, patterns })
449 }
450
451 pub fn allows(&self, text: &str) -> bool {
453 (!self.terms.is_empty() && self.terms.iter().any(|a| a == &text.to_lowercase()))
454 || self.patterns.iter().any(|re| re.is_match(text))
455 }
456}
457
458pub struct PatternDetector {
462 kinds: Vec<PiiKind>,
463 custom: Vec<(String, Regex)>,
464 deny: Vec<Regex>,
465 allow: AllowList,
466}
467
468impl PatternDetector {
469 pub fn new(opts: &RedactionOptions) -> Result<Self, RedactionError> {
473 let kinds = PiiKind::PATTERN
474 .into_iter()
475 .filter(|&k| opts.wants(k))
476 .collect();
477 let mut custom = Vec::new();
478 for p in &opts.custom_patterns {
479 let re = Regex::new(&p.regex)
480 .map_err(|e| RedactionError(format!("redact pattern {:?}: {e}", p.name)))?;
481 custom.push((p.name.trim().to_ascii_uppercase().replace('-', "_"), re));
482 }
483 let deny = opts
484 .deny_terms
485 .iter()
486 .filter(|t| !t.trim().is_empty())
487 .map(|t| {
488 Regex::new(&format!(r"(?i)\b{}\b", regex::escape(t.trim()))).expect("escaped term")
489 })
490 .collect();
491 let allow = AllowList::new(opts)?;
492 Ok(Self {
493 kinds,
494 custom,
495 deny,
496 allow,
497 })
498 }
499
500 fn wants(&self, kind: PiiKind) -> bool {
501 self.kinds.contains(&kind)
502 }
503
504 pub fn allowed(&self, text: &str) -> bool {
507 self.allow.allows(text)
508 }
509}
510
511impl PiiDetector for PatternDetector {
512 fn detect(&self, text: &str) -> Vec<Span> {
513 let mut out = Vec::new();
514 if text.is_empty() {
515 return out;
516 }
517 let mut credentials: Vec<(usize, usize)> = Vec::new();
521 if self.wants(PiiKind::UrlCredentials) {
522 for c in re!(r"(?i)[a-z][a-z0-9+.\-]*://([^\s/@:]+:[^\s/@]+)@").captures_iter(text) {
523 let m = c.get(1).expect("group 1");
524 credentials.push((m.start(), m.end()));
525 out.push(Span::new(m.start(), m.end(), PiiKind::UrlCredentials));
526 }
527 }
528 if self.wants(PiiKind::Email) {
529 for m in re!(r"(?i)[a-z0-9._%+\-\\]+@[a-z0-9](?:[a-z0-9\-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9\-]*[a-z0-9])?)*\.[a-z]{2,}").find_iter(text) {
530 let (mut s, mut e) = (m.start(), m.end());
533 while e > s && text.as_bytes()[e - 1] == b'.' {
534 e -= 1;
535 }
536 while s < e && text.as_bytes()[s] == b'\\' {
537 s += 1;
538 }
539 if s < e && !credentials.iter().any(|&(cs, ce)| s < ce && cs < e) {
540 out.push(Span::new(s, e, PiiKind::Email));
541 }
542 }
543 }
544 if self.wants(PiiKind::CreditCard) {
545 for m in re!(r"\b\d(?:[ \-]?\d){12,18}\b").find_iter(text) {
546 let digits: String = m.as_str().chars().filter(|c| c.is_ascii_digit()).collect();
547 if !maximal_digit_run(text, m.start(), m.end())
550 || !token_bounded(text, m.start(), m.end())
551 {
552 continue;
553 }
554 if card_brand(&digits).is_some()
558 && card_grouping_ok(m.as_str())
559 && luhn_valid(&digits)
560 {
561 out.push(Span::new(m.start(), m.end(), PiiKind::CreditCard));
562 }
563 }
564 }
565 if self.wants(PiiKind::Iban) {
566 for m in
567 re!(r"\b[A-Z]{2}\d{2}(?: ?[A-Z0-9]{4}){2,7}(?: ?[A-Z0-9]{1,4})?\b").find_iter(text)
568 {
569 let compact: String = m.as_str().chars().filter(|c| *c != ' ').collect();
570 if iban_valid(&compact) {
571 out.push(Span::new(m.start(), m.end(), PiiKind::Iban));
572 }
573 }
574 }
575 if self.wants(PiiKind::IpAddress) {
576 for m in re!(r"\b(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}\b").find_iter(text) {
577 let before = text[..m.start()].chars().next_back();
579 let after = text[m.end()..].chars().next();
580 if before == Some('.') || after == Some('.') {
581 continue;
582 }
583 if matches!(before, Some('v') | Some('V')) {
585 continue;
586 }
587 out.push(Span::new(m.start(), m.end(), PiiKind::IpAddress));
588 }
589 for m in re!(r"[0-9A-Fa-f:]{3,45}").find_iter(text) {
590 let s = m.as_str();
591 if s.matches(':').count() < 2 || !s.chars().any(|c| c.is_ascii_hexdigit()) {
592 continue;
593 }
594 if s.parse::<std::net::Ipv6Addr>().is_ok() {
595 out.push(Span::new(m.start(), m.end(), PiiKind::IpAddress));
596 }
597 }
598 }
599 if self.wants(PiiKind::NationalId) {
600 for m in re!(r"\b\d{3}-\d{2}-\d{4}\b").find_iter(text) {
604 let s = m.as_str();
605 let area = &s[0..3];
606 if area == "000" || area == "666" || area.starts_with('9') {
607 continue;
608 }
609 if &s[4..6] == "00" || &s[7..11] == "0000" {
610 continue;
611 }
612 if !token_bounded(text, m.start(), m.end()) || citation_before(text, m.start()) {
613 continue;
614 }
615 out.push(Span::new(m.start(), m.end(), PiiKind::NationalId));
616 }
617 for m in re!(
621 r"\b[A-CEGHJ-PR-TW-Z][A-CEGHJ-NPR-TW-Z](?:\d{6}[A-D]| \d{2} \d{2} \d{2} [A-D])\b"
622 )
623 .find_iter(text)
624 {
625 let prefix = &m.as_str()[0..2];
626 if ["BG", "GB", "NK", "KN", "TN", "NT", "ZZ"].contains(&prefix) {
627 continue;
628 }
629 if !token_bounded(text, m.start(), m.end()) {
630 continue;
631 }
632 out.push(Span::new(m.start(), m.end(), PiiKind::NationalId));
633 }
634 for m in re!(r"\b[2-9]\d{3}([ \-]?)\d{4}([ \-]?)\d{4}\b").captures_iter(text) {
639 let whole = m.get(0).expect("match");
640 let (sep1, sep2) = (
641 m.get(1).map_or("", |g| g.as_str()),
642 m.get(2).map_or("", |g| g.as_str()),
643 );
644 if sep1 != sep2 || !token_bounded(text, whole.start(), whole.end()) {
645 continue;
646 }
647 let s = whole.as_str();
648 if !sep1.is_empty() && (s.starts_with("19") || s.starts_with("20")) {
649 continue;
650 }
651 let digits: String = s.chars().filter(|c| c.is_ascii_digit()).collect();
652 if verhoeff_valid(&digits) {
653 out.push(Span::new(whole.start(), whole.end(), PiiKind::NationalId));
654 }
655 }
656 }
657 if self.wants(PiiKind::Phone) {
658 let ssn_shape = re!(r"^\d{3}-\d{2}-\d{4}$");
659 for m in re!(r"(?:\+\d{1,3}[ .\-]?)?(?:\(\d{1,4}\)[ .\-]?)?\d{2,4}(?:[ .\-]\d{2,4}){1,4}\b|\+\d{7,15}\b").find_iter(text) {
660 let s = m.as_str();
661 let digits = s.chars().filter(|c| c.is_ascii_digit()).count();
662 if !(7..=15).contains(&digits) || looks_like_date_or_version(s) {
663 continue;
664 }
665 if ssn_shape.is_match(s) {
667 continue;
668 }
669 if !token_bounded(text, m.start(), m.end()) {
673 continue;
674 }
675 if !phone_shape(s, digits) && !phone_keyword_before(text, m.start()) {
682 continue;
683 }
684 out.push(Span::new(m.start(), m.end(), PiiKind::Phone));
685 }
686 }
687 for (name, re) in &self.custom {
688 for c in re.captures_iter(text) {
689 let m = c.get(1).or_else(|| c.get(0)).expect("match");
690 if m.start() == m.end() {
691 continue;
692 }
693 out.push(Span {
694 start: m.start(),
695 end: m.end(),
696 kind: PiiKind::Custom,
697 score: 1.0,
698 name: Some(name.clone()),
699 });
700 }
701 }
702 for re in &self.deny {
703 for m in re.find_iter(text) {
704 out.push(Span::new(m.start(), m.end(), PiiKind::Custom));
705 }
706 }
707 out.retain(|s| !self.allowed(&text[s.start..s.end]));
708 out
709 }
710}
711
712fn maximal_digit_run(text: &str, start: usize, end: usize) -> bool {
715 let bytes = text.as_bytes();
716 let before =
717 start >= 2 && matches!(bytes[start - 1], b' ' | b'-') && bytes[start - 2].is_ascii_digit();
718 let after = end + 1 < bytes.len()
719 && matches!(bytes[end], b' ' | b'-')
720 && bytes[end + 1].is_ascii_digit();
721 !before && !after
722}
723
724fn token_bounded(text: &str, start: usize, end: usize) -> bool {
728 let before = text[..start].chars().next_back();
729 let after = text[end..].chars().next();
730 let opens_itself = text[start..].starts_with(['+', '(']);
734 let left_ok = match before {
735 Some(c) if opens_itself => !c.is_ascii_digit(),
736 Some(c) => !c.is_alphanumeric(),
737 None => true,
738 };
739 left_ok && !after.is_some_and(|c| c.is_alphanumeric())
740}
741
742fn phone_shape(s: &str, digits: usize) -> bool {
748 if s.starts_with('+') || s.contains('(') {
749 return true;
750 }
751 let groups: Vec<usize> = s
752 .split(|c: char| !c.is_ascii_digit())
753 .filter(|g| !g.is_empty())
754 .map(str::len)
755 .collect();
756 groups == [3, 3, 4]
757 || groups == [1, 3, 3, 4]
758 || (groups.len() >= 3
759 && ((10..=11).contains(&digits) || (s.starts_with('0') && digits <= 13)))
760}
761
762fn phone_keyword_before(text: &str, pos: usize) -> bool {
765 let head = &text[..pos];
766 let from = head.char_indices().rev().nth(30).map_or(0, |(i, _)| i);
767 re!(r"(?i)\b(?:tel|tel\.|telephone|phone|fax|mobile|cell|call|contact|whatsapp|hotline|dial|sms|text)\b[^\d]*$")
768 .is_match(&head[from..])
769}
770
771fn citation_before(text: &str, pos: usize) -> bool {
775 let head = text[..pos].trim_end();
776 let from = head.char_indices().rev().nth(16).map_or(0, |(i, _)| i);
777 re!(r"(?i)(?:\b(?:rule|rules|section|sections|sec|code|article|art|para|paragraph|clause|chapter|title|regulation|reg|order|statute|part|no|number)\.?|§+)\s*(?:no\.?\s*)?$")
778 .is_match(&head[from..])
779}
780
781fn card_brand(digits: &str) -> Option<&'static str> {
784 let n = digits.len();
785 let p = |k: usize| digits[..k.min(n)].parse::<u32>().unwrap_or(0);
786 let (two, three, four) = (p(2), p(3), p(4));
787 if digits.starts_with('4') && matches!(n, 13 | 16 | 19) {
788 return Some("visa");
789 }
790 if ((51..=55).contains(&two) || (2221..=2720).contains(&four)) && n == 16 {
791 return Some("mastercard");
792 }
793 if (two == 34 || two == 37) && n == 15 {
794 return Some("amex");
795 }
796 if (four == 6011 || two == 65 || (644..=649).contains(&three)) && (16..=19).contains(&n) {
797 return Some("discover");
798 }
799 if ((300..=305).contains(&three) || two == 36 || two == 38 || two == 39)
800 && (14..=19).contains(&n)
801 {
802 return Some("diners");
803 }
804 if (3528..=3589).contains(&four) && (16..=19).contains(&n) {
805 return Some("jcb");
806 }
807 if two == 62 && (16..=19).contains(&n) {
808 return Some("unionpay");
809 }
810 if matches!(
811 four,
812 5018 | 5020 | 5038 | 5893 | 6304 | 6759 | 6761 | 6762 | 6763
813 ) && (12..=19).contains(&n)
814 {
815 return Some("maestro");
816 }
817 None
818}
819
820fn card_grouping_ok(s: &str) -> bool {
824 let groups: Vec<usize> = s
825 .split(|c: char| !c.is_ascii_digit())
826 .filter(|g| !g.is_empty())
827 .map(str::len)
828 .collect();
829 if groups.len() == 1 {
830 return true;
831 }
832 let (last, head) = groups.split_last().expect("non-empty");
833 (head.iter().all(|&g| g == 4) && (1..=4).contains(last))
834 || groups == [4, 6, 5]
835 || groups == [4, 6, 4]
836}
837
838pub fn luhn_valid(digits: &str) -> bool {
840 let mut sum = 0u32;
841 let mut double = false;
842 for c in digits.chars().rev() {
843 let Some(d) = c.to_digit(10) else {
844 return false;
845 };
846 let d = if double {
847 let x = d * 2;
848 if x > 9 {
849 x - 9
850 } else {
851 x
852 }
853 } else {
854 d
855 };
856 sum += d;
857 double = !double;
858 }
859 digits.len() >= 2 && sum % 10 == 0
860}
861
862fn iban_length(country: &str) -> Option<usize> {
866 Some(match country {
867 "AL" | "AZ" | "CY" | "DO" | "GT" | "HU" | "LB" | "PL" | "BY" | "SV" | "NI" => 28,
868 "AD" | "CZ" | "MD" | "PK" | "RO" | "SA" | "SK" | "ES" | "SE" | "TN" | "VG" => 24,
869 "AT" | "BA" | "EE" | "KZ" | "LT" | "LU" | "XK" | "MN" => 20,
870 "BH" | "BG" | "CR" | "GE" | "DE" | "IE" | "ME" | "RS" | "GB" | "VA" => 22,
871 "BE" | "BI" => 16,
872 "BR" | "PS" | "QA" | "UA" | "EG" => 29,
873 "HR" | "LI" | "CH" => 21,
874 "DK" | "FO" | "FI" | "GL" | "NL" | "SD" | "FK" => 18,
875 "FR" | "GR" | "IT" | "MR" | "MC" | "SM" | "DJ" => 27,
876 "GI" | "IL" | "AE" | "IQ" | "TL" | "SO" | "OM" => 23,
877 "IS" => 26,
878 "JO" | "KW" | "MU" | "YE" => 30,
879 "LV" => 21,
880 "MK" | "SI" => 19,
881 "MT" => 31,
882 "NO" => 15,
883 "PT" | "ST" | "LY" => 25,
884 "TR" => 26,
885 "LC" => 32,
886 "RU" => 33,
887 _ => return None,
888 })
889}
890
891pub fn iban_valid(iban: &str) -> bool {
893 if iban.len() < 15 || iban.len() > 34 || !iban.is_ascii() {
894 return false;
895 }
896 let country = &iban[0..2];
897 if iban_length(country) != Some(iban.len()) {
898 return false;
899 }
900 let rearranged = format!("{}{}", &iban[4..], &iban[..4]);
901 let mut rem = 0u32;
902 for c in rearranged.chars() {
903 let v = match c {
904 '0'..='9' => c as u32 - '0' as u32,
905 'A'..='Z' => c as u32 - 'A' as u32 + 10,
906 _ => return false,
907 };
908 rem = if v >= 10 {
909 (rem * 100 + v) % 97
910 } else {
911 (rem * 10 + v) % 97
912 };
913 }
914 rem == 1
915}
916
917pub fn verhoeff_valid(digits: &str) -> bool {
919 const D: [[u8; 10]; 10] = [
920 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
921 [1, 2, 3, 4, 0, 6, 7, 8, 9, 5],
922 [2, 3, 4, 0, 1, 7, 8, 9, 5, 6],
923 [3, 4, 0, 1, 2, 8, 9, 5, 6, 7],
924 [4, 0, 1, 2, 3, 9, 5, 6, 7, 8],
925 [5, 9, 8, 7, 6, 0, 4, 3, 2, 1],
926 [6, 5, 9, 8, 7, 1, 0, 4, 3, 2],
927 [7, 6, 5, 9, 8, 2, 1, 0, 4, 3],
928 [8, 7, 6, 5, 9, 3, 2, 1, 0, 4],
929 [9, 8, 7, 6, 5, 4, 3, 2, 1, 0],
930 ];
931 const P: [[u8; 10]; 8] = [
932 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
933 [1, 5, 7, 6, 2, 8, 3, 0, 9, 4],
934 [5, 8, 0, 3, 7, 9, 6, 1, 4, 2],
935 [8, 9, 1, 6, 0, 4, 3, 5, 2, 7],
936 [9, 4, 5, 3, 1, 2, 6, 8, 7, 0],
937 [4, 2, 8, 6, 5, 7, 3, 9, 0, 1],
938 [2, 7, 9, 3, 8, 0, 6, 4, 1, 5],
939 [7, 0, 4, 6, 9, 1, 3, 2, 5, 8],
940 ];
941 let mut c = 0u8;
942 for (i, ch) in digits.chars().rev().enumerate() {
943 let Some(d) = ch.to_digit(10) else {
944 return false;
945 };
946 c = D[c as usize][P[i % 8][d as usize] as usize];
947 }
948 !digits.is_empty() && c == 0
949}
950
951fn looks_like_date_or_version(s: &str) -> bool {
954 let s = s.trim();
955 re!(r"^\d{4}[\-./]\d{1,2}[\-./]\d{1,2}$").is_match(s)
956 || re!(r"^\d{1,2}[\-./]\d{1,2}[\-./]\d{2,4}$").is_match(s)
957 || (s.contains('.') && !s.contains(' ') && !s.contains('-') && !s.starts_with('+'))
959}
960
961pub struct Redactor<'a> {
970 opts: &'a RedactionOptions,
971 detector: &'a dyn PiiDetector,
972 allow: AllowList,
976 numbers: HashMap<(String, String), usize>,
978 next: HashMap<String, usize>,
979 report: RedactionReport,
980 counting: bool,
984}
985
986impl<'a> Redactor<'a> {
987 pub fn new(opts: &'a RedactionOptions, detector: &'a dyn PiiDetector) -> Self {
988 Redactor {
989 opts,
990 detector,
991 allow: AllowList::new(opts).unwrap_or_default(),
992 numbers: HashMap::new(),
993 next: HashMap::new(),
994 report: RedactionReport {
995 mapping: opts.return_mapping.then(Vec::new),
996 ..Default::default()
997 },
998 counting: true,
999 }
1000 }
1001
1002 pub fn report(&self) -> &RedactionReport {
1004 &self.report
1005 }
1006
1007 pub fn finish(self) -> RedactionReport {
1008 self.report
1009 }
1010
1011 fn spans(&self, text: &str) -> Vec<Span> {
1016 let mut spans: Vec<Span> = self
1017 .detector
1018 .detect(text)
1019 .into_iter()
1020 .filter(|s| s.start < s.end && s.end <= text.len())
1021 .filter(|s| text.is_char_boundary(s.start) && text.is_char_boundary(s.end))
1022 .filter(|s| self.opts.wants(s.kind))
1023 .filter(|s| !PiiKind::NER.contains(&s.kind) || s.score >= self.opts.ner_min_score)
1024 .filter(|s| !self.allow.allows(&text[s.start..s.end]))
1025 .collect();
1026 spans.sort_by(|a, b| {
1027 a.start
1028 .cmp(&b.start)
1029 .then((b.end - b.start).cmp(&(a.end - a.start)))
1030 .then(a.kind.cmp(&b.kind))
1031 });
1032 let mut kept: Vec<Span> = Vec::with_capacity(spans.len());
1033 for s in spans {
1034 match kept.last() {
1035 Some(last) if s.start < last.end => {
1036 if s.end - s.start > last.end - last.start {
1038 kept.pop();
1039 kept.push(s);
1040 }
1041 }
1042 _ => kept.push(s),
1043 }
1044 }
1045 kept
1046 }
1047
1048 pub fn has_pii(&self, text: &str) -> bool {
1051 !self.spans(text).is_empty()
1052 }
1053
1054 fn label_of(span: &Span) -> String {
1055 span.name
1056 .clone()
1057 .unwrap_or_else(|| span.kind.label().to_string())
1058 }
1059
1060 fn placeholder(&mut self, span: &Span, original: &str) -> String {
1062 let label = Self::label_of(span);
1063 if self.counting {
1064 *self.report.counts.entry(label.clone()).or_insert(0) += 1;
1065 self.report.total += 1;
1066 }
1067 let out = match &self.opts.replacement {
1068 Replacement::Label => format!("[{label}]"),
1069 Replacement::Fixed(s) => s.clone(),
1070 Replacement::Pseudonym => {
1071 let key = normalize(original);
1074 let n = match self.numbers.get(&(label.clone(), key.clone())) {
1075 Some(n) => *n,
1076 None => {
1077 let next = self.next.entry(label.clone()).or_insert(0);
1078 *next += 1;
1079 self.numbers.insert((label.clone(), key), *next);
1080 *next
1081 }
1082 };
1083 format!("[{label}_{n}]")
1084 }
1085 };
1086 if let Some(map) = self.report.mapping.as_mut() {
1087 if !map.iter().any(|(o, _)| o == original) {
1088 map.push((original.to_string(), out.clone()));
1089 }
1090 }
1091 out
1092 }
1093
1094 pub fn redact_text(&mut self, text: &mut String) {
1096 let spans = self.spans(text);
1097 if spans.is_empty() {
1098 return;
1099 }
1100 let mut out = String::with_capacity(text.len());
1101 let mut pos = 0;
1102 for span in &spans {
1103 out.push_str(&text[pos..span.start]);
1104 let original = text[span.start..span.end].to_string();
1105 out.push_str(&self.placeholder(span, &original));
1106 pos = span.end;
1107 }
1108 out.push_str(&text[pos..]);
1109 *text = out;
1110 }
1111
1112 fn redact_opt(&mut self, text: &mut Option<String>) {
1113 if let Some(t) = text.as_mut() {
1114 self.redact_text(t);
1115 }
1116 }
1117
1118 pub fn redact_runs(&mut self, runs: &mut [InlineRun]) {
1122 let joined: String = runs.iter().map(|r| r.text.as_str()).collect();
1123 let spans = self.spans(&joined);
1124 if spans.is_empty() {
1125 return;
1126 }
1127 let mut starts = Vec::with_capacity(runs.len());
1129 let mut acc = 0;
1130 for r in runs.iter() {
1131 starts.push(acc);
1132 acc += r.text.len();
1133 }
1134 let mut placeholders: Vec<String> = Vec::with_capacity(spans.len());
1138 for span in &spans {
1139 let original = joined[span.start..span.end].to_string();
1140 placeholders.push(self.placeholder(span, &original));
1141 }
1142 for (i, run) in runs.iter_mut().enumerate() {
1143 let (rs, re) = (starts[i], starts[i] + run.text.len());
1144 let mut out = String::new();
1145 let mut pos = rs;
1146 for (span, ph) in spans.iter().zip(&placeholders) {
1147 if span.end <= rs || span.start >= re {
1148 continue;
1149 }
1150 let s = span.start.max(rs);
1151 out.push_str(&joined[pos..s]);
1152 if span.start >= rs {
1153 out.push_str(ph);
1154 }
1155 pos = span.end.min(re);
1156 }
1157 out.push_str(&joined[pos..re]);
1158 run.text = out;
1159 }
1160 }
1161
1162 fn redact_table(&mut self, table: &mut Table) {
1163 for row in table.rows.iter_mut() {
1164 for cell in row.iter_mut() {
1165 self.redact_text(cell);
1166 }
1167 }
1168 if let Some(cells) = table.cells.as_mut() {
1169 for c in cells.iter_mut() {
1170 self.redact_text(&mut c.text);
1171 }
1172 }
1173 if let Some(blocks) = table.cell_blocks.as_mut() {
1174 for row in blocks.iter_mut() {
1175 for cell in row.iter_mut() {
1176 self.redact_nodes(cell);
1177 }
1178 }
1179 }
1180 self.redact_opt(&mut table.caption);
1181 }
1182
1183 fn redact_fields(&mut self, items: &mut [crate::FieldItem]) {
1184 for f in items.iter_mut() {
1185 self.redact_opt(&mut f.marker);
1186 self.redact_opt(&mut f.key);
1187 self.redact_opt(&mut f.value);
1188 }
1189 }
1190
1191 fn redact_graph(&mut self, cells: &mut [crate::GraphCell]) {
1192 for c in cells.iter_mut() {
1193 self.redact_text(&mut c.text);
1194 self.redact_text(&mut c.orig);
1195 }
1196 }
1197
1198 fn redact_image(&mut self, image: &mut Option<crate::PictureImage>) {
1199 if self.opts.images == ImageRedaction::Drop {
1200 *image = None;
1201 }
1202 }
1203
1204 pub fn redact_nodes(&mut self, nodes: &mut [Node]) {
1206 for node in nodes.iter_mut() {
1207 self.redact_node(node);
1208 }
1209 }
1210
1211 fn redact_node(&mut self, node: &mut Node) {
1212 if let Node::ScriptText { inner, .. } = node {
1216 let plain = std::mem::replace(inner.as_mut(), Node::PageBreak);
1217 *node = plain;
1218 }
1219 match node {
1220 Node::Heading { text, .. }
1221 | Node::Paragraph { text }
1222 | Node::ListText { text, .. }
1223 | Node::CheckboxItem { text, .. }
1224 | Node::PageFurniture { text, .. }
1225 | Node::FurnitureText { text, .. }
1226 | Node::TextDump(text) => self.redact_text(text),
1227 Node::ListItem {
1228 text,
1229 marker,
1230 dclx,
1231 href,
1232 ..
1233 } => {
1234 self.redact_text(text);
1235 self.redact_opt(marker);
1236 self.redact_opt(href);
1237 if let Some(d) = dclx.as_mut() {
1238 self.redact_text(&mut d.text);
1239 self.redact_opt(&mut d.marker);
1240 self.redact_runs(&mut d.runs);
1241 }
1242 }
1243 Node::Code {
1244 text, orig, pretty, ..
1245 } => {
1246 self.redact_text(text);
1247 self.redact_opt(orig);
1248 self.redact_opt(pretty);
1249 }
1250 Node::Table(t) => self.redact_table(t),
1251 Node::Picture {
1252 caption,
1253 caption_href,
1254 image,
1255 description,
1256 ..
1257 } => {
1258 self.redact_opt(caption);
1259 self.redact_opt(caption_href);
1260 if let Some(d) = description.as_mut() {
1263 self.redact_text(&mut d.text);
1264 }
1265 self.redact_image(image);
1266 }
1267 Node::Formula { orig, .. } => self.redact_text(orig),
1270 Node::Caption { text, href } | Node::LabeledText { text, href, .. } => {
1271 self.redact_text(text);
1272 self.redact_opt(href);
1273 }
1274 Node::Chart { table, caption, .. } => {
1275 self.redact_table(table);
1276 self.redact_opt(caption);
1277 }
1278 Node::Group { name, children, .. } => {
1279 self.redact_opt(name);
1280 self.redact_nodes(children);
1281 }
1282 Node::FieldRegion { items } => self.redact_fields(items),
1283 Node::KeyValueGraph { cells, .. } => self.redact_graph(cells),
1284 Node::InlineGroup { runs, md_text, .. } => {
1285 self.redact_runs(runs);
1286 self.redact_text(md_text);
1287 }
1288 Node::ScriptText { .. } => {}
1290 Node::Furniture { inner, .. }
1291 | Node::Commented { inner, .. }
1292 | Node::Located { inner, .. }
1293 | Node::Prov { inner, .. }
1294 | Node::DoclangOnly(inner) => self.redact_node(inner),
1295 Node::Track { track, cue, inner } => {
1300 self.redact_opt(&mut track.voice);
1301 let original = cue.clone();
1302 self.redact_text(cue);
1303 match inner.as_mut() {
1304 Node::Paragraph { text, .. } if text.ends_with(&original) => {
1305 let prefix_len = text.len() - original.len();
1306 text.truncate(prefix_len);
1307 text.push_str(cue);
1308 }
1309 other => {
1310 let counting = std::mem::replace(&mut self.counting, false);
1311 self.redact_node(other);
1312 self.counting = counting;
1313 }
1314 }
1315 }
1316 Node::CommentSection { name, text, .. } => {
1317 self.redact_text(name);
1318 self.redact_text(text);
1319 }
1320 Node::PictureChildren(children) => self.redact_nodes(children),
1321 Node::PageBreak | Node::PageInfo { .. } | Node::Anchor { .. } => {}
1322 }
1323 }
1324
1325 pub fn redact_tree(&mut self, tree: &mut ItemTree) {
1330 let counting = std::mem::replace(&mut self.counting, false);
1331 self.redact_tree_items(tree);
1332 self.counting = counting;
1333 }
1334
1335 fn redact_tree_items(&mut self, tree: &mut ItemTree) {
1336 for item in tree.items.iter_mut() {
1337 match &mut item.kind {
1338 TreeKind::Text {
1339 text,
1340 orig,
1341 hyperlink,
1342 list,
1343 ..
1344 } => {
1345 self.redact_text(text);
1346 self.redact_opt(orig);
1347 self.redact_opt(hyperlink);
1348 if let Some(l) = list.as_mut() {
1349 self.redact_text(&mut l.marker);
1350 }
1351 }
1352 TreeKind::Code {
1353 text,
1354 orig,
1355 hyperlink,
1356 ..
1357 } => {
1358 self.redact_text(text);
1359 self.redact_opt(orig);
1360 self.redact_opt(hyperlink);
1361 }
1362 TreeKind::Group { name, .. } => self.redact_text(name),
1363 TreeKind::Table { table, .. } => self.redact_table(table),
1364 TreeKind::Picture {
1365 image,
1366 chart,
1367 description,
1368 ..
1369 } => {
1370 self.redact_image(image);
1371 if let Some(d) = description.as_mut() {
1372 self.redact_text(&mut d.text);
1373 }
1374 if let Some(t) = chart.as_mut() {
1375 self.redact_table(t);
1376 }
1377 }
1378 TreeKind::FieldRegion { items } => self.redact_fields(items),
1379 TreeKind::KeyValueGraph { cells, .. } => self.redact_graph(cells),
1380 }
1381 if let Some(track) = item.source.as_mut() {
1382 self.redact_opt(&mut track.voice);
1383 }
1384 for note in item.notes.iter_mut() {
1385 self.redact_text(&mut note.text);
1386 }
1387 }
1388 }
1389
1390 pub fn redact_links(&mut self, links: &mut [(String, String)]) {
1392 for (text, href) in links.iter_mut() {
1393 self.redact_text(text);
1394 self.redact_text(href);
1395 }
1396 }
1397
1398 pub fn redact_document(&mut self, doc: &mut DoclingDocument) {
1400 self.redact_nodes(&mut doc.nodes);
1401 if let Some(tree) = doc.tree.as_mut() {
1402 self.redact_tree(tree);
1403 }
1404 self.redact_links(&mut doc.links);
1405 self.redact_metadata(&mut doc.metadata);
1406 if self.opts.images == ImageRedaction::Drop {
1407 doc.page_images.clear();
1408 }
1409 }
1410
1411 pub fn redact_metadata(&mut self, m: &mut crate::metadata::DocumentMetadata) {
1421 if self.opts.wants(PiiKind::Person) {
1422 for a in m.authors.iter_mut() {
1423 self.redact_whole(a, PiiKind::Person);
1424 }
1425 if let Some(by) = m.last_modified_by.as_mut() {
1426 self.redact_whole(by, PiiKind::Person);
1427 }
1428 for key in ["Artist", "XPAuthor"] {
1429 if let Some(v) = m.exif.get_mut(key) {
1430 self.redact_whole(v, PiiKind::Person);
1431 }
1432 }
1433 for key in ["artist", "album_artist", "composer", "performer"] {
1434 if let Some(v) = m.audio.get_mut(key) {
1435 self.redact_whole(v, PiiKind::Person);
1436 }
1437 }
1438 }
1439 if self.opts.wants(PiiKind::Location) {
1440 let had_gps = m.gps.take().is_some();
1441 let before = m.exif.len();
1442 m.exif.retain(|k, _| !k.starts_with("GPS"));
1443 if had_gps || m.exif.len() != before {
1444 self.count(PiiKind::Location.label());
1445 }
1446 }
1447 self.redact_opt(&mut m.title);
1448 self.redact_opt(&mut m.subject);
1449 self.redact_opt(&mut m.description);
1450 for k in m.keywords.iter_mut() {
1451 self.redact_text(k);
1452 }
1453 self.redact_opt(&mut m.publisher);
1454 self.redact_opt(&mut m.rights);
1455 self.redact_opt(&mut m.identifier);
1456 self.redact_opt(&mut m.creator_tool);
1457 for v in m.properties.values_mut() {
1458 self.redact_text(v);
1459 }
1460 for (k, v) in m.exif.iter_mut() {
1461 if k != "Artist" && k != "XPAuthor" {
1462 self.redact_text(v);
1463 }
1464 }
1465 for (k, v) in m.audio.iter_mut() {
1466 if !matches!(
1467 k.as_str(),
1468 "artist" | "album_artist" | "composer" | "performer"
1469 ) {
1470 self.redact_text(v);
1471 }
1472 }
1473 }
1474
1475 fn redact_whole(&mut self, text: &mut String, kind: PiiKind) {
1478 if text.trim().is_empty() || self.allow.allows(text) {
1479 return;
1480 }
1481 let span = Span::new(0, text.len(), kind);
1482 let original = text.clone();
1483 *text = self.placeholder(&span, &original);
1484 }
1485
1486 fn count(&mut self, label: &str) {
1488 if self.counting {
1489 *self.report.counts.entry(label.to_string()).or_insert(0) += 1;
1490 self.report.total += 1;
1491 }
1492 }
1493}
1494
1495fn normalize(s: &str) -> String {
1499 s.to_lowercase()
1500 .chars()
1501 .filter(|c| !c.is_whitespace() && *c != '-' && *c != '.' && *c != '(' && *c != ')')
1502 .collect()
1503}
1504
1505impl DoclingDocument {
1506 pub fn redact(
1510 &mut self,
1511 opts: &RedactionOptions,
1512 detector: &dyn PiiDetector,
1513 ) -> RedactionReport {
1514 let mut r = Redactor::new(opts, detector);
1515 r.redact_document(self);
1516 r.finish()
1517 }
1518}
1519
1520#[cfg(test)]
1521mod tests {
1522 use super::*;
1523
1524 fn detector(opts: &RedactionOptions) -> PatternDetector {
1525 PatternDetector::new(opts).unwrap()
1526 }
1527
1528 fn redact(text: &str) -> String {
1529 let opts = RedactionOptions::default();
1530 let d = detector(&opts);
1531 let mut s = text.to_string();
1532 Redactor::new(&opts, &d).redact_text(&mut s);
1533 s
1534 }
1535
1536 fn kinds(text: &str) -> Vec<PiiKind> {
1537 let opts = RedactionOptions::default();
1538 let d = detector(&opts);
1539 Redactor::new(&opts, &d)
1540 .spans(text)
1541 .into_iter()
1542 .map(|s| s.kind)
1543 .collect()
1544 }
1545
1546 #[test]
1547 fn emails_including_markdown_links_and_escapes() {
1548 assert_eq!(
1549 redact("Write to [john.doe@example.com](mailto:john.doe@example.com)."),
1550 "Write to [[EMAIL]](mailto:[EMAIL])."
1551 );
1552 assert_eq!(
1554 redact(r"jane\_doe@example.org, please."),
1555 "[EMAIL], please."
1556 );
1557 assert_eq!(redact("a@b"), "a@b");
1558 }
1559
1560 #[test]
1561 fn cards_need_luhn_and_ibans_need_mod97() {
1562 assert_eq!(kinds("card 4111 1111 1111 1111"), vec![PiiKind::CreditCard]);
1563 assert_eq!(kinds("card 4111-1111-1111-1111"), vec![PiiKind::CreditCard]);
1564 assert_eq!(kinds("order 4111 1111 1111 1112"), vec![]);
1565 assert_eq!(
1566 kinds("IBAN DE89 3704 0044 0532 0130 00"),
1567 vec![PiiKind::Iban]
1568 );
1569 assert_eq!(kinds("IBAN GB82WEST12345698765432"), vec![PiiKind::Iban]);
1570 assert_eq!(kinds("IBAN DE89 3704 0044 0532 0130 01"), vec![]);
1571 assert_eq!(kinds("code XY12 ABCD 1234 EFGH 5678"), vec![]);
1572 }
1573
1574 #[test]
1575 fn phones_but_not_dates_or_versions() {
1576 assert_eq!(kinds("call +1 (555) 123-4567"), vec![PiiKind::Phone]);
1577 assert_eq!(kinds("call 555-123-4567 today"), vec![PiiKind::Phone]);
1578 assert_eq!(kinds("tel +44 20 7946 0958"), vec![PiiKind::Phone]);
1579 assert_eq!(kinds("on 2024-01-15 and 15.01.2024 and 01/15/24"), vec![]);
1580 assert_eq!(kinds("version 1.2.3 and v10.0.1 and v2.0.0.1"), vec![]);
1581 assert_eq!(kinds("ISBN 978-3-16-148410-0"), vec![]);
1582 }
1583
1584 #[test]
1585 fn ip_addresses_and_url_credentials() {
1586 assert_eq!(kinds("host 192.168.1.10 up"), vec![PiiKind::IpAddress]);
1587 assert_eq!(kinds("v 1.2.3.4.5"), vec![]);
1588 assert_eq!(
1589 kinds("addr 2001:db8::1 and ::1"),
1590 vec![PiiKind::IpAddress, PiiKind::IpAddress]
1591 );
1592 assert_eq!(kinds("at 12:30:45 today"), vec![]);
1593 assert_eq!(kinds("mac aa:bb:cc:dd:ee:ff"), vec![]);
1594 assert_eq!(
1595 redact("see https://alice:s3cret@db.example.com/x"),
1596 "see https://[CREDENTIALS]@db.example.com/x"
1597 );
1598 }
1599
1600 #[test]
1601 fn national_ids_with_their_rules() {
1602 assert_eq!(kinds("SSN 123-45-6789"), vec![PiiKind::NationalId]);
1603 assert_eq!(
1604 kinds("not 000-45-6789 nor 123-00-6789 nor 900-45-6789"),
1605 vec![]
1606 );
1607 assert_eq!(kinds("NINO AB 12 34 56 C"), vec![PiiKind::NationalId]);
1608 assert_eq!(kinds("NINO QQ 12 34 56 C"), vec![]);
1609 assert_eq!(kinds("NINO BG 12 34 56 C"), vec![]);
1610 let base = "22345678901";
1613 let ok = (0..10)
1614 .map(|d| format!("{base}{d}"))
1615 .find(|s| verhoeff_valid(s))
1616 .unwrap();
1617 assert_eq!(kinds(&format!("Aadhaar {ok}")), vec![PiiKind::NationalId]);
1618 let bad = (0..10)
1619 .map(|d| format!("{base}{d}"))
1620 .find(|s| !verhoeff_valid(s))
1621 .unwrap();
1622 assert_eq!(kinds(&format!("Aadhaar {bad}")), vec![]);
1623 }
1624
1625 #[test]
1626 fn replacement_modes_and_pseudonym_consistency() {
1627 let opts = RedactionOptions {
1628 replacement: Replacement::Pseudonym,
1629 return_mapping: true,
1630 ..Default::default()
1631 };
1632 let d = detector(&opts);
1633 let mut r = Redactor::new(&opts, &d);
1634 let mut a = "a@x.com wrote to b@x.com; A@X.COM again".to_string();
1635 r.redact_text(&mut a);
1636 assert_eq!(a, "[EMAIL_1] wrote to [EMAIL_2]; [EMAIL_1] again");
1637 let mut b = "call 555-123-4567 or 555 123 4567".to_string();
1638 r.redact_text(&mut b);
1639 assert_eq!(b, "call [PHONE_1] or [PHONE_1]");
1640 let report = r.finish();
1641 assert_eq!(report.counts["EMAIL"], 3);
1642 assert_eq!(report.counts["PHONE"], 2);
1643 assert_eq!(report.total, 5);
1644 let map = report.mapping.unwrap();
1645 assert!(map.contains(&("a@x.com".into(), "[EMAIL_1]".into())));
1646 assert_eq!(map.len(), 5);
1647
1648 let opts = RedactionOptions {
1649 replacement: Replacement::Fixed("***".into()),
1650 ..Default::default()
1651 };
1652 let d = detector(&opts);
1653 let mut s = "a@x.com".to_string();
1654 Redactor::new(&opts, &d).redact_text(&mut s);
1655 assert_eq!(s, "***");
1656 }
1657
1658 #[test]
1659 fn kinds_filter_custom_patterns_deny_and_allow_terms() {
1660 let opts = RedactionOptions {
1661 kinds: vec![PiiKind::Phone],
1662 ..Default::default()
1663 };
1664 let d = detector(&opts);
1665 let mut s = "a@x.com 555-123-4567".to_string();
1666 Redactor::new(&opts, &d).redact_text(&mut s);
1667 assert_eq!(s, "a@x.com [PHONE]");
1668
1669 let opts = RedactionOptions {
1670 custom_patterns: vec![CustomPattern::parse("case_id=CASE-\\d{4}").unwrap()],
1671 deny_terms: vec!["Project Falcon".into()],
1672 allow_terms: vec!["support@example.com".into()],
1673 ..Default::default()
1674 };
1675 let d = detector(&opts);
1676 let mut s = "CASE-0042 by project falcon: support@example.com / x@example.com".to_string();
1677 Redactor::new(&opts, &d).redact_text(&mut s);
1678 assert_eq!(s, "[CASE_ID] by [REDACTED]: support@example.com / [EMAIL]");
1679 assert!(PatternDetector::new(&RedactionOptions {
1680 custom_patterns: vec![CustomPattern {
1681 name: "bad".into(),
1682 regex: "(".into()
1683 }],
1684 ..Default::default()
1685 })
1686 .is_err());
1687 }
1688
1689 #[test]
1692 fn postal_codes_citations_record_ids_and_part_numbers_are_not_pii() {
1693 let opts = RedactionOptions {
1694 kinds: vec![
1695 PiiKind::Phone,
1696 PiiKind::CreditCard,
1697 PiiKind::Iban,
1698 PiiKind::NationalId,
1699 PiiKind::IpAddress,
1700 ],
1701 ..Default::default()
1702 };
1703 let d = detector(&opts);
1704 let untouched = [
1705 "Ship to: Example Corp, 100 Main Street, Springfield, IL 62704-1234.",
1706 "Send comments to Department 55A, Mail Station 1234 5600 Elm Road.",
1707 "Leave accrual is defined in rule 412-18-205 of the state code.",
1708 "Leave accrual is defined in rule 412-18-0150 of the state code.",
1709 "The input file contains rows such as 60601 0001 12500 00350 and 60602 0002 9800 00120.",
1710 "Replace the bracket with part number TR482913 B before assembly.",
1711 "/XY123456 A configuration file is required for every job.",
1712 "Position created in 2019 95753514 for the regional office.",
1713 "Employee 88061052 76665755 Clerk Grade 4",
1714 ];
1715 for text in untouched {
1716 let mut s = text.to_string();
1717 Redactor::new(&opts, &d).redact_text(&mut s);
1718 assert_eq!(s, text);
1719 }
1720 let mut s = "Call the help desk at +1 555 010 0199 for assistance.".to_string();
1721 Redactor::new(&opts, &d).redact_text(&mut s);
1722 assert_eq!(s, "Call the help desk at [PHONE] for assistance.");
1723 }
1724
1725 #[test]
1728 fn real_values_are_still_redacted() {
1729 assert_eq!(kinds("+1 555 010 0199"), vec![PiiKind::Phone]);
1730 assert_eq!(kinds("(555) 010-0199"), vec![PiiKind::Phone]);
1731 assert_eq!(kinds("555-010-0199"), vec![PiiKind::Phone]);
1732 assert_eq!(kinds("Tel: 020 7946 0958"), vec![PiiKind::Phone]);
1733 assert_eq!(kinds("020 7946 0958"), vec![PiiKind::Phone]);
1734 assert_eq!(kinds("mobile 0151 496 0000"), vec![PiiKind::Phone]);
1735 assert_eq!(kinds("phone: 1234 5600"), vec![PiiKind::Phone]);
1736 assert_eq!(kinds("Mail Station 1234 5600"), vec![]);
1737 assert_eq!(kinds("card 4111 1111 1111 1111"), vec![PiiKind::CreditCard]);
1738 assert_eq!(kinds("amex 378282246310005"), vec![PiiKind::CreditCard]);
1739 assert_eq!(kinds("amex 3782 822463 10005"), vec![PiiKind::CreditCard]);
1740 assert_eq!(kinds("mc 5555 5555 5555 4444"), vec![PiiKind::CreditCard]);
1741 assert_eq!(kinds("mc 2223 0031 2200 3222"), vec![PiiKind::CreditCard]);
1742 assert_eq!(kinds("order 8806105276665755"), vec![]);
1744 assert_eq!(
1746 kinds(
1747 "4111 1111 1111 1111"
1748 .replace(' ', "-")
1749 .replace("1111-1111-1111", "11111111-1111")
1750 .as_str()
1751 ),
1752 vec![]
1753 );
1754 assert_eq!(kinds("NINO AB123456C"), vec![PiiKind::NationalId]);
1755 assert_eq!(kinds("NINO AB 12 34 56 C"), vec![PiiKind::NationalId]);
1756 assert_eq!(kinds("NINO AB123456 C"), vec![]);
1757 assert_eq!(kinds("NINO AB 123456C"), vec![]);
1758 assert_eq!(kinds("SSN 123-45-6789"), vec![PiiKind::NationalId]);
1759 assert_eq!(kinds("ID 412-18-0150 on file"), vec![PiiKind::NationalId]);
1760 assert_eq!(kinds("see § 412-18-0150 and section 412-18-0151"), vec![]);
1761 assert_eq!(
1762 kinds("IBAN GB82 WEST 1234 5698 7654 32"),
1763 vec![PiiKind::Iban]
1764 );
1765 let base = "22345678901";
1767 let ok = (0..10)
1768 .map(|d| format!("{base}{d}"))
1769 .find(|s| verhoeff_valid(s))
1770 .unwrap();
1771 let spaced = format!("{} {} {}", &ok[..4], &ok[4..8], &ok[8..]);
1772 assert_eq!(
1773 kinds(&format!("Aadhaar {spaced}")),
1774 vec![PiiKind::NationalId]
1775 );
1776 assert_eq!(kinds(&format!("Aadhaar {} {}", &ok[..4], &ok[4..])), vec![]);
1777 let year_base = "20199575351";
1778 let year_ok = (0..10)
1779 .map(|d| format!("{year_base}{d}"))
1780 .find(|s| verhoeff_valid(s))
1781 .unwrap();
1782 assert_eq!(
1783 kinds(&format!(
1784 "in {} {} {}",
1785 &year_ok[..4],
1786 &year_ok[4..8],
1787 &year_ok[8..]
1788 )),
1789 vec![]
1790 );
1791 assert_eq!(
1792 kinds(&format!("Aadhaar {year_ok}")),
1793 vec![PiiKind::NationalId]
1794 );
1795 }
1796
1797 #[test]
1799 fn matches_are_whole_tokens() {
1800 assert_eq!(redact("IL 62704-1234 stays"), "IL 62704-1234 stays");
1801 assert_eq!(
1802 redact("row 60602 0002 9800 00120 stays"),
1803 "row 60602 0002 9800 00120 stays"
1804 );
1805 assert_eq!(redact("id x555-010-0199"), "id x555-010-0199");
1806 assert_eq!(redact("id 555-010-0199x"), "id 555-010-0199x");
1807 assert_eq!(redact("ref 4111111111111111a"), "ref 4111111111111111a");
1808 assert_eq!(redact("ref AB123456CD"), "ref AB123456CD");
1809 assert_eq!(redact("call 555-010-0199."), "call [PHONE].");
1810 }
1811
1812 #[test]
1814 fn allow_patterns_exempt_matching_spans() {
1815 let opts = RedactionOptions {
1816 allow_patterns: vec![r"\+1 555 010 \d{4}".into(), r"[a-z]+@example\.com".into()],
1817 ..Default::default()
1818 };
1819 let d = detector(&opts);
1820 let mut s =
1821 "+1 555 010 0199, +1 555 010 0200, +1 555 011 0200, bob@example.com, bob@example.org"
1822 .to_string();
1823 Redactor::new(&opts, &d).redact_text(&mut s);
1824 assert_eq!(
1825 s,
1826 "+1 555 010 0199, +1 555 010 0200, [PHONE], bob@example.com, [EMAIL]"
1827 );
1828 assert!(PatternDetector::new(&RedactionOptions {
1829 allow_patterns: vec!["(".into()],
1830 ..Default::default()
1831 })
1832 .is_err());
1833 }
1834
1835 #[test]
1839 fn transcript_timings_are_not_phones_and_cues_count_once() {
1840 fn track(start: f64, end: f64, cue: &str) -> Node {
1841 Node::Track {
1842 track: crate::tree::TreeTrack {
1843 start_time: start,
1844 end_time: end,
1845 identifier: None,
1846 voice: None,
1847 },
1848 cue: cue.to_string(),
1849 inner: Box::new(Node::Paragraph {
1850 text: format!("[time: {start}-{end}] {cue}"),
1851 }),
1852 }
1853 }
1854 let opts = RedactionOptions {
1855 replacement: Replacement::Pseudonym,
1856 ..Default::default()
1857 };
1858 let d = detector(&opts);
1859 let mut doc = DoclingDocument::new("t");
1860 doc.nodes = vec![
1861 track(1234.5, 1240.8, "Shakespeare on scenery by Oscar Wilde."),
1862 track(
1863 1240.8,
1864 1251.25,
1865 "Call me on 555-010-0199 or 555-010-0199 again.",
1866 ),
1867 ];
1868 let report = doc.redact(&opts, &d);
1869 assert_eq!(report.counts.get("PHONE"), Some(&2));
1870 assert_eq!(report.total, 2);
1871 let texts: Vec<(String, String)> = doc
1872 .nodes
1873 .iter()
1874 .map(|n| match n {
1875 Node::Track { cue, inner, .. } => match inner.as_ref() {
1876 Node::Paragraph { text, .. } => (cue.clone(), text.clone()),
1877 _ => unreachable!(),
1878 },
1879 _ => unreachable!(),
1880 })
1881 .collect();
1882 assert_eq!(
1883 texts[0],
1884 (
1885 "Shakespeare on scenery by Oscar Wilde.".to_string(),
1886 "[time: 1234.5-1240.8] Shakespeare on scenery by Oscar Wilde.".to_string()
1887 )
1888 );
1889 assert_eq!(
1890 texts[1],
1891 (
1892 "Call me on [PHONE_1] or [PHONE_1] again.".to_string(),
1893 "[time: 1240.8-1251.25] Call me on [PHONE_1] or [PHONE_1] again.".to_string()
1894 )
1895 );
1896 assert_eq!(kinds("[time: 1234.5-1240.8]"), vec![]);
1898 }
1899
1900 #[test]
1901 fn overlapping_spans_keep_the_longest() {
1902 assert_eq!(kinds("4111 1111 1111 1111"), vec![PiiKind::CreditCard]);
1904 let opts = RedactionOptions {
1906 custom_patterns: vec![CustomPattern::parse("contact=Contact: \\S+").unwrap()],
1907 ..Default::default()
1908 };
1909 let d = detector(&opts);
1910 let mut s = "Contact: a@x.com now".to_string();
1911 Redactor::new(&opts, &d).redact_text(&mut s);
1912 assert_eq!(s, "[CONTACT] now");
1913 }
1914
1915 #[test]
1916 fn inline_runs_are_matched_on_the_joined_text() {
1917 let opts = RedactionOptions::default();
1918 let d = detector(&opts);
1919 let mut runs = vec![
1920 InlineRun {
1921 text: "Mail john".into(),
1922 bold: true,
1923 ..Default::default()
1924 },
1925 InlineRun {
1926 text: "@example".into(),
1927 ..Default::default()
1928 },
1929 InlineRun {
1930 text: ".com today, or 555-".into(),
1931 italic: true,
1932 ..Default::default()
1933 },
1934 InlineRun {
1935 text: "123-4567.".into(),
1936 ..Default::default()
1937 },
1938 ];
1939 Redactor::new(&opts, &d).redact_runs(&mut runs);
1940 let texts: Vec<&str> = runs.iter().map(|r| r.text.as_str()).collect();
1941 assert_eq!(texts, vec!["Mail [EMAIL]", "", " today, or [PHONE]", "."]);
1942 assert!(runs[0].bold && runs[2].italic);
1943 }
1944
1945 #[test]
1946 fn the_document_walk_reaches_every_string() {
1947 use crate::tree::TreeKind;
1948 let mut doc = DoclingDocument::new("file.docx");
1949 doc.push(Node::Heading {
1950 level: 1,
1951 text: "Re: a@x.com".into(),
1952 });
1953 doc.push(Node::Group {
1954 label: "section".into(),
1955 name: Some("b@x.com".into()),
1956 layer: None,
1957 children: vec![Node::Furniture {
1958 layer: crate::ContentLayer::Furniture,
1959 inner: Box::new(Node::Paragraph {
1960 text: "c@x.com".into(),
1961 }),
1962 }],
1963 });
1964 let mut table = Table {
1965 rows: vec![vec!["d@x.com".into()]],
1966 ..Default::default()
1967 };
1968 table.cell_blocks = Some(vec![vec![vec![Node::Paragraph {
1969 text: "e@x.com".into(),
1970 }]]]);
1971 doc.push(Node::Table(table));
1972 doc.push(Node::Picture {
1973 caption: Some("f@x.com".into()),
1974 caption_href: Some("mailto:f@x.com".into()),
1975 image: Some(crate::PictureImage {
1976 mimetype: "image/png".into(),
1977 width: 1,
1978 height: 1,
1979 data: vec![1],
1980 dpi: 72,
1981 display_size: None,
1982 }),
1983 classification: None,
1984 description: Some(crate::PictureDescription {
1985 text: "ocr j@x.com".into(),
1986 provenance: "ppocr".into(),
1987 }),
1988 caption_parent: Default::default(),
1989 caption_location: None,
1990 });
1991 doc.push(Node::Formula {
1992 latex: "x@y.com".into(),
1993 orig: "g@x.com".into(),
1994 location: None,
1995 });
1996 doc.links.push(("h@x.com".into(), "mailto:h@x.com".into()));
1997 let mut tree = ItemTree::default();
1998 tree.add(
1999 None,
2000 None,
2001 TreeKind::Text {
2002 label: "text".into(),
2003 text: "i@x.com".into(),
2004 orig: Some("i@x.com".into()),
2005 formatting: None,
2006 hyperlink: Some("mailto:i@x.com".into()),
2007 level: None,
2008 list: None,
2009 },
2010 );
2011 doc.tree = Some(tree);
2012 doc.page_images.insert(
2013 1,
2014 crate::PictureImage {
2015 mimetype: "image/png".into(),
2016 width: 1,
2017 height: 1,
2018 data: vec![1],
2019 dpi: 72,
2020 display_size: None,
2021 },
2022 );
2023
2024 let opts = RedactionOptions::default();
2025 let d = detector(&opts);
2026 let report = doc.redact(&opts, &d);
2027 assert_eq!(report.counts["EMAIL"], 11);
2029 let json = doc.export_to_json();
2030 let md = doc.export_to_markdown();
2031 let dl = doc.export_to_doclang();
2032 for out in [&json, &md, &dl] {
2033 assert!(!out.contains("@x.com"), "{out}");
2034 }
2035 assert_eq!(doc.name, "file.docx", "the file name is not touched");
2036 assert!(doc.page_images.is_empty());
2037 assert!(matches!(&doc.nodes[3], Node::Picture { image: None, .. }));
2038 assert!(matches!(&doc.nodes[4], Node::Formula { latex, .. } if latex == "x@y.com"));
2040
2041 let mut doc2 = DoclingDocument::new("t");
2043 doc2.page_images.insert(
2044 1,
2045 crate::PictureImage {
2046 mimetype: "image/png".into(),
2047 width: 1,
2048 height: 1,
2049 data: vec![1],
2050 dpi: 72,
2051 display_size: None,
2052 },
2053 );
2054 let opts = RedactionOptions {
2055 images: ImageRedaction::Keep,
2056 ..Default::default()
2057 };
2058 let d = detector(&opts);
2059 doc2.redact(&opts, &d);
2060 assert_eq!(doc2.page_images.len(), 1);
2061 }
2062
2063 #[test]
2064 fn wire_spellings_parse() {
2065 assert_eq!(PiiKind::parse("Credit-Card"), Some(PiiKind::CreditCard));
2066 assert_eq!(PiiKind::parse("ip"), Some(PiiKind::IpAddress));
2067 assert_eq!(PiiKind::parse("nope"), None);
2068 assert_eq!(
2069 Replacement::parse("fixed:XXX"),
2070 Some(Replacement::Fixed("XXX".into()))
2071 );
2072 assert_eq!(
2073 Replacement::parse("Pseudonym"),
2074 Some(Replacement::Pseudonym)
2075 );
2076 assert_eq!(
2077 ImageRedaction::parse("box-out"),
2078 Some(ImageRedaction::BoxOut)
2079 );
2080 let p = CustomPattern::parse("case_id=CASE-\\d+").unwrap();
2081 assert_eq!(
2082 (p.name.as_str(), p.regex.as_str()),
2083 ("case_id", "CASE-\\d+")
2084 );
2085 assert_eq!(CustomPattern::parse("(?i)x=y").unwrap().name, "custom");
2086 }
2087}