Skip to main content

docling_core/
redact.rs

1//! Opt-in PII redaction of a [`DoclingDocument`] (#621).
2//!
3//! A converted document feeds search and RAG pipelines — Markdown, chunks,
4//! embeddings, vector stores, LLM prompts — and personal data in the source
5//! flows into every one of them. Redacting each export format after the
6//! fact misses the text that lives elsewhere in the model: the item tree
7//! behind the JSON export, the link table, a code block's `orig`, the
8//! key-value graph's cells, a caption's hyperlink, the pixels of an embedded
9//! image. This pass runs **once, on the model, before any serializer,
10//! chunker or stream reads it**, replaces every detected span in place with
11//! a placeholder and returns a [`RedactionReport`] of counts — never the
12//! original values (unless the caller asks for the mapping explicitly).
13//!
14//! Detection is pluggable through [`PiiDetector`]. The built-in
15//! [`PatternDetector`] is pure Rust and compiles for wasm: e-mail, phone,
16//! Luhn-validated card numbers, mod-97-validated IBANs, IPv4/IPv6, URL
17//! credentials and a few national IDs with their own check digits (US SSN,
18//! UK NINO, Indian Aadhaar — Verhoeff), plus the caller's own patterns and
19//! deny terms. A name/organization/location detector (NER) lives in the
20//! `docling` crate behind its `ner` feature and plugs into the same trait;
21//! several detectors compose through [`CompositeDetector`], overlapping
22//! spans resolved longest-first.
23//!
24//! Python docling has no redaction stage, so this is a docling.rs extension
25//! (recorded in `docs/MIGRATION.md`); it is off by default and, unused,
26//! changes nothing.
27//!
28//! What is walked: every user-visible string of the flat nodes, recursively
29//! (groups, furniture wrappers, picture children, rich table cells), the
30//! item tree (`tree`), the link table, code text and `orig`, a formula's
31//! `orig` (the LaTeX is left alone — it is a rendering, not text, and the
32//! pipeline's own `orig` is what a reader could recover PII from), tables
33//! (grid, first-class cells, rich-cell blocks), field regions, key-value
34//! cells, comments, hrefs, VTT cue voices. An inline group is matched on
35//! the concatenation of its runs and the spans mapped back onto the runs.
36//! The document's `name` (the file name) is not touched. Image handling is
37//! [`ImageRedaction`]: `Drop` removes every embedded image and page render
38//! here; `BoxOut` needs the OCR models and is the `docling` crate's job
39//! (this pass then leaves the pixels alone, as `Keep` does).
40//!
41//! Non-goals: no compliance guarantee (GDPR/HIPAA/PCI), no rewriting of the
42//! source file, no cross-node entity detection, no pseudonym persistence
43//! across documents.
44
45use std::collections::{BTreeMap, HashMap};
46use std::fmt;
47
48use regex::Regex;
49
50use crate::document::{InlineRun, Node, Table};
51use crate::tree::{ItemTree, TreeKind};
52use crate::DoclingDocument;
53
54/// The kinds of personal data a detector can report. The order is the
55/// precedence when two detectors' spans overlap at equal length — the
56/// check-digit formats (card, IBAN, national ID) over a phone number, whose
57/// digit groups they also look like.
58#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
59pub enum PiiKind {
60    Email,
61    CreditCard,
62    Iban,
63    NationalId,
64    IpAddress,
65    UrlCredentials,
66    Phone,
67    Person,
68    Organization,
69    Location,
70    Address,
71    /// A caller-supplied pattern ([`CustomPattern`]) or deny term.
72    Custom,
73}
74
75impl PiiKind {
76    /// Every kind, in precedence order.
77    pub const ALL: [PiiKind; 12] = [
78        PiiKind::Email,
79        PiiKind::CreditCard,
80        PiiKind::Iban,
81        PiiKind::NationalId,
82        PiiKind::IpAddress,
83        PiiKind::UrlCredentials,
84        PiiKind::Phone,
85        PiiKind::Person,
86        PiiKind::Organization,
87        PiiKind::Location,
88        PiiKind::Address,
89        PiiKind::Custom,
90    ];
91
92    /// The kinds the pattern detector handles itself (no model needed).
93    pub const PATTERN: [PiiKind; 7] = [
94        PiiKind::Email,
95        PiiKind::CreditCard,
96        PiiKind::Iban,
97        PiiKind::NationalId,
98        PiiKind::IpAddress,
99        PiiKind::UrlCredentials,
100        PiiKind::Phone,
101    ];
102
103    /// The kinds only a named-entity model finds.
104    pub const NER: [PiiKind; 4] = [
105        PiiKind::Person,
106        PiiKind::Organization,
107        PiiKind::Location,
108        PiiKind::Address,
109    ];
110
111    /// The placeholder label: `[EMAIL]`, `[PERSON_2]`.
112    pub fn label(self) -> &'static str {
113        match self {
114            PiiKind::Email => "EMAIL",
115            PiiKind::Phone => "PHONE",
116            PiiKind::CreditCard => "CREDIT_CARD",
117            PiiKind::Iban => "IBAN",
118            PiiKind::IpAddress => "IP",
119            PiiKind::UrlCredentials => "CREDENTIALS",
120            PiiKind::NationalId => "NATIONAL_ID",
121            PiiKind::Person => "PERSON",
122            PiiKind::Organization => "ORG",
123            PiiKind::Location => "LOCATION",
124            PiiKind::Address => "ADDRESS",
125            PiiKind::Custom => "REDACTED",
126        }
127    }
128
129    /// The wire / CLI spelling (`credit_card`, `ip_address`, …).
130    pub fn name(self) -> &'static str {
131        match self {
132            PiiKind::Email => "email",
133            PiiKind::Phone => "phone",
134            PiiKind::CreditCard => "credit_card",
135            PiiKind::Iban => "iban",
136            PiiKind::IpAddress => "ip_address",
137            PiiKind::UrlCredentials => "url_credentials",
138            PiiKind::NationalId => "national_id",
139            PiiKind::Person => "person",
140            PiiKind::Organization => "organization",
141            PiiKind::Location => "location",
142            PiiKind::Address => "address",
143            PiiKind::Custom => "custom",
144        }
145    }
146
147    /// Parse a wire spelling (case-insensitive; `-` as `_`; `ip` and `org`
148    /// accepted as shorthands).
149    pub fn parse(s: &str) -> Option<Self> {
150        let s = s.trim().to_ascii_lowercase().replace('-', "_");
151        match s.as_str() {
152            "ip" => return Some(PiiKind::IpAddress),
153            "org" => return Some(PiiKind::Organization),
154            "credentials" | "url_credential" => return Some(PiiKind::UrlCredentials),
155            _ => {}
156        }
157        PiiKind::ALL.into_iter().find(|k| k.name() == s)
158    }
159
160    /// Every spelling [`parse`](Self::parse) accepts, for error messages.
161    pub const ACCEPTED: &'static str = "email, phone, credit_card, iban, ip_address, \
162        url_credentials, national_id, person, organization, location, address, custom";
163}
164
165/// What a detected span becomes.
166#[derive(Debug, Clone, PartialEq, Eq, Default)]
167pub enum Replacement {
168    /// `[EMAIL]`, `[PERSON]` — the kind's label (the default).
169    #[default]
170    Label,
171    /// `[EMAIL_1]`, `[EMAIL_2]`: a number per distinct value within the
172    /// document, so the same e-mail reads as the same placeholder wherever it
173    /// recurs and two different ones stay apart — what a downstream reader
174    /// (or an LLM) needs to follow "who did what" with the names gone.
175    Pseudonym,
176    /// One fixed string for everything, e.g. `█████` or `***`.
177    Fixed(String),
178}
179
180impl Replacement {
181    /// Parse the wire spelling: `label` | `pseudonym` | `fixed:<text>`.
182    pub fn parse(s: &str) -> Option<Self> {
183        let t = s.trim();
184        if t.eq_ignore_ascii_case("label") {
185            return Some(Replacement::Label);
186        }
187        if t.eq_ignore_ascii_case("pseudonym") {
188            return Some(Replacement::Pseudonym);
189        }
190        let lower = t.to_ascii_lowercase();
191        if let Some(rest) = lower.strip_prefix("fixed:") {
192            // Keep the caller's own casing of the text.
193            let text = &t[t.len() - rest.len()..];
194            return Some(Replacement::Fixed(text.to_string()));
195        }
196        None
197    }
198
199    pub const ACCEPTED: &'static str = "label, pseudonym, fixed:<text>";
200}
201
202/// What happens to embedded images (pictures, page renders) when the pass
203/// runs. An image can carry anything the text does — a scanned letter, a
204/// screenshot of a profile — and the pass cannot read it without OCR.
205#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
206pub enum ImageRedaction {
207    /// Remove every embedded image and page render (the default): nothing
208    /// unread leaves the document.
209    #[default]
210    Drop,
211    /// OCR each image and paint solid boxes over the lines that carry a
212    /// detected span; keep the rest of the pixels. Needs the OCR models (the
213    /// `docling` crate does this; without the models it degrades to `Drop`
214    /// with a warning).
215    BoxOut,
216    /// Leave the images as they are.
217    Keep,
218}
219
220impl ImageRedaction {
221    /// Parse the wire spelling: `drop` | `box_out` | `keep`.
222    pub fn parse(s: &str) -> Option<Self> {
223        match s.trim().to_ascii_lowercase().replace('-', "_").as_str() {
224            "drop" => Some(ImageRedaction::Drop),
225            "box_out" | "boxout" | "box" => Some(ImageRedaction::BoxOut),
226            "keep" => Some(ImageRedaction::Keep),
227            _ => None,
228        }
229    }
230
231    pub const ACCEPTED: &'static str = "drop, box_out, keep";
232}
233
234/// A caller-supplied pattern: `name` is the placeholder label (`[CASE_ID]`
235/// for `case_id`), `regex` the expression (the `regex` crate's syntax —
236/// no look-around); the whole match is the span, or capture group 1 when
237/// the expression has one.
238#[derive(Debug, Clone, PartialEq, Eq)]
239pub struct CustomPattern {
240    pub name: String,
241    pub regex: String,
242}
243
244impl CustomPattern {
245    /// Parse the wire spelling `NAME=REGEX` (a bare `REGEX` gets the label
246    /// `CUSTOM`).
247    pub fn parse(s: &str) -> Option<Self> {
248        let s = s.trim();
249        if s.is_empty() {
250            return None;
251        }
252        let (name, regex) = match s.split_once('=') {
253            // `NAME=…` only when the name looks like an identifier — a regex
254            // itself may start with `(?i)x=`.
255            Some((n, r))
256                if !n.is_empty()
257                    && n.chars()
258                        .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') =>
259            {
260                (n.to_string(), r.to_string())
261            }
262            _ => ("custom".to_string(), s.to_string()),
263        };
264        Some(CustomPattern { name, regex })
265    }
266}
267
268/// The pass's settings. `Default` = every kind, labels, no custom patterns,
269/// images dropped, no mapping.
270#[derive(Debug, Clone, PartialEq)]
271pub struct RedactionOptions {
272    /// Which kinds to redact; empty = every kind the available detectors
273    /// find (the pattern kinds, and the NER kinds when a model runs).
274    pub kinds: Vec<PiiKind>,
275    pub replacement: Replacement,
276    /// Extra patterns, redacted as [`PiiKind::Custom`] under their own label.
277    pub custom_patterns: Vec<CustomPattern>,
278    /// Literal terms (case-insensitive, whole words) always redacted, as
279    /// `[REDACTED]` — a project code name, a client's name the NER misses.
280    pub deny_terms: Vec<String>,
281    /// Literal terms (case-insensitive) never redacted even when a detector
282    /// flags them — `support@example.com`, the company's own name.
283    pub allow_terms: Vec<String>,
284    /// Regexes a flagged span must not match in full to be redacted (#694)
285    /// — the escape hatch for a document's own number shapes: part numbers
286    /// (`[A-Z]{2}\d{6}`), record IDs, a ticket format. Applied to every
287    /// detector's spans, the NER's included.
288    pub allow_patterns: Vec<String>,
289    /// The NER detector's minimum confidence for a span (0–1; 0.85 by
290    /// default — a lone common word in a table header reads as a location
291    /// at ~0.8, a real name at 0.99+).
292    pub ner_min_score: f32,
293    pub images: ImageRedaction,
294    /// Keep the `original → placeholder` pairs in the report. Off by default
295    /// — the report then carries counts only and can be logged or returned
296    /// over HTTP without leaking what was removed.
297    pub return_mapping: bool,
298}
299
300impl Default for RedactionOptions {
301    fn default() -> Self {
302        Self {
303            kinds: Vec::new(),
304            replacement: Replacement::Label,
305            custom_patterns: Vec::new(),
306            deny_terms: Vec::new(),
307            allow_terms: Vec::new(),
308            allow_patterns: Vec::new(),
309            ner_min_score: 0.85,
310            images: ImageRedaction::Drop,
311            return_mapping: false,
312        }
313    }
314}
315
316impl RedactionOptions {
317    /// Whether `kind` is in scope (an empty `kinds` means all).
318    pub fn wants(&self, kind: PiiKind) -> bool {
319        self.kinds.is_empty() || self.kinds.contains(&kind)
320    }
321
322    /// Whether any NER-only kind is in scope.
323    pub fn wants_ner(&self) -> bool {
324        PiiKind::NER.iter().any(|&k| self.wants(k))
325    }
326}
327
328/// One detected span: byte offsets into the text it was detected on, the
329/// kind, the detector's confidence (1.0 for a pattern) and, for a custom
330/// pattern, its name (the label).
331#[derive(Debug, Clone, PartialEq)]
332pub struct Span {
333    pub start: usize,
334    pub end: usize,
335    pub kind: PiiKind,
336    pub score: f32,
337    pub name: Option<String>,
338}
339
340impl Span {
341    pub fn new(start: usize, end: usize, kind: PiiKind) -> Self {
342        Span {
343            start,
344            end,
345            kind,
346            score: 1.0,
347            name: None,
348        }
349    }
350}
351
352/// A span detector. `detect` returns the spans of one text; the pass calls
353/// it once per string of the document, so a detector keeps no per-document
354/// state (pseudonym numbering is the pass's own).
355pub trait PiiDetector {
356    fn detect(&self, text: &str) -> Vec<Span>;
357}
358
359/// Several detectors as one; their spans are merged by the pass's overlap
360/// rule (longest first, then kind precedence).
361pub struct CompositeDetector(pub Vec<Box<dyn PiiDetector + Send + Sync>>);
362
363impl PiiDetector for CompositeDetector {
364    fn detect(&self, text: &str) -> Vec<Span> {
365        self.0.iter().flat_map(|d| d.detect(text)).collect()
366    }
367}
368
369/// What was redacted: one count per label (`EMAIL`, `PHONE`, a custom
370/// pattern's name, `REDACTED` for deny terms), the total, and — only with
371/// [`RedactionOptions::return_mapping`] — the `original → placeholder`
372/// pairs. A backend that also builds docling's item tree (DOCX, HTML) has
373/// its text redacted in both representations but counted once.
374#[derive(Debug, Clone, PartialEq, Eq, Default)]
375pub struct RedactionReport {
376    pub counts: BTreeMap<String, usize>,
377    pub total: usize,
378    pub mapping: Option<Vec<(String, String)>>,
379}
380
381impl RedactionReport {
382    /// The counts as a JSON object (`{"EMAIL": 2, "PHONE": 1}`), what the
383    /// HTTP surfaces answer with — never the mapping.
384    pub fn counts_json(&self) -> serde_json::Value {
385        serde_json::Value::Object(
386            self.counts
387                .iter()
388                .map(|(k, v)| (k.clone(), serde_json::json!(v)))
389                .collect(),
390        )
391    }
392}
393
394/// A rejected option (an invalid custom regex).
395#[derive(Debug, Clone, PartialEq, Eq)]
396pub struct RedactionError(pub String);
397
398impl fmt::Display for RedactionError {
399    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
400        f.write_str(&self.0)
401    }
402}
403
404impl std::error::Error for RedactionError {}
405
406// ---------------------------------------------------------------------------
407// The pattern detector
408// ---------------------------------------------------------------------------
409
410macro_rules! re {
411    ($pat:expr) => {{
412        static RE: std::sync::OnceLock<Regex> = std::sync::OnceLock::new();
413        RE.get_or_init(|| Regex::new($pat).expect("static regex"))
414    }};
415}
416
417/// What is never redacted: the allow terms (case-insensitive, the whole
418/// span) and the allow patterns (a regex the whole span matches). Built
419/// once per pass from the options and asked about every span, whichever
420/// detector produced it.
421#[derive(Debug, Default)]
422pub struct AllowList {
423    terms: Vec<String>,
424    patterns: Vec<Regex>,
425}
426
427impl AllowList {
428    /// Compile the options' allow terms and patterns; an invalid pattern is
429    /// the error.
430    pub fn new(opts: &RedactionOptions) -> Result<Self, RedactionError> {
431        let terms = opts
432            .allow_terms
433            .iter()
434            .map(|t| t.trim().to_lowercase())
435            .filter(|t| !t.is_empty())
436            .collect();
437        let mut patterns = Vec::new();
438        for p in opts
439            .allow_patterns
440            .iter()
441            .map(|p| p.trim())
442            .filter(|p| !p.is_empty())
443        {
444            let re = Regex::new(&format!("^(?:{p})$"))
445                .map_err(|e| RedactionError(format!("redact allow pattern {p:?}: {e}")))?;
446            patterns.push(re);
447        }
448        Ok(Self { terms, patterns })
449    }
450
451    /// Whether a span's text is allowed through.
452    pub fn allows(&self, text: &str) -> bool {
453        (!self.terms.is_empty() && self.terms.iter().any(|a| a == &text.to_lowercase()))
454            || self.patterns.iter().any(|re| re.is_match(text))
455    }
456}
457
458/// The built-in detector: patterns with check-digit validation where the
459/// format has one, so a random 16-digit order number is not a card and
460/// `2024-01-15` is not a phone.
461pub struct PatternDetector {
462    kinds: Vec<PiiKind>,
463    custom: Vec<(String, Regex)>,
464    deny: Vec<Regex>,
465    allow: AllowList,
466}
467
468impl PatternDetector {
469    /// Build from the options: the pattern kinds in scope, the custom
470    /// patterns compiled (an invalid one is the error), the deny terms as
471    /// whole-word case-insensitive matches.
472    pub fn new(opts: &RedactionOptions) -> Result<Self, RedactionError> {
473        let kinds = PiiKind::PATTERN
474            .into_iter()
475            .filter(|&k| opts.wants(k))
476            .collect();
477        let mut custom = Vec::new();
478        for p in &opts.custom_patterns {
479            let re = Regex::new(&p.regex)
480                .map_err(|e| RedactionError(format!("redact pattern {:?}: {e}", p.name)))?;
481            custom.push((p.name.trim().to_ascii_uppercase().replace('-', "_"), re));
482        }
483        let deny = opts
484            .deny_terms
485            .iter()
486            .filter(|t| !t.trim().is_empty())
487            .map(|t| {
488                Regex::new(&format!(r"(?i)\b{}\b", regex::escape(t.trim()))).expect("escaped term")
489            })
490            .collect();
491        let allow = AllowList::new(opts)?;
492        Ok(Self {
493            kinds,
494            custom,
495            deny,
496            allow,
497        })
498    }
499
500    fn wants(&self, kind: PiiKind) -> bool {
501        self.kinds.contains(&kind)
502    }
503
504    /// Whether the span's text is an allowed term or matches an allow
505    /// pattern.
506    pub fn allowed(&self, text: &str) -> bool {
507        self.allow.allows(text)
508    }
509}
510
511impl PiiDetector for PatternDetector {
512    fn detect(&self, text: &str) -> Vec<Span> {
513        let mut out = Vec::new();
514        if text.is_empty() {
515            return out;
516        }
517        // `scheme://user:password@host` — the `user:password` part. Found
518        // first: `password@host.tld` also looks like an e-mail address, and
519        // the credentials span is the one to keep.
520        let mut credentials: Vec<(usize, usize)> = Vec::new();
521        if self.wants(PiiKind::UrlCredentials) {
522            for c in re!(r"(?i)[a-z][a-z0-9+.\-]*://([^\s/@:]+:[^\s/@]+)@").captures_iter(text) {
523                let m = c.get(1).expect("group 1");
524                credentials.push((m.start(), m.end()));
525                out.push(Span::new(m.start(), m.end(), PiiKind::UrlCredentials));
526            }
527        }
528        if self.wants(PiiKind::Email) {
529            for m in re!(r"(?i)[a-z0-9._%+\-\\]+@[a-z0-9](?:[a-z0-9\-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9\-]*[a-z0-9])?)*\.[a-z]{2,}").find_iter(text) {
530                // A trailing `.` belongs to the sentence; a leading `\` to
531                // a Markdown escape of the character after it.
532                let (mut s, mut e) = (m.start(), m.end());
533                while e > s && text.as_bytes()[e - 1] == b'.' {
534                    e -= 1;
535                }
536                while s < e && text.as_bytes()[s] == b'\\' {
537                    s += 1;
538                }
539                if s < e && !credentials.iter().any(|&(cs, ce)| s < ce && cs < e) {
540                    out.push(Span::new(s, e, PiiKind::Email));
541                }
542            }
543        }
544        if self.wants(PiiKind::CreditCard) {
545            for m in re!(r"\b\d(?:[ \-]?\d){12,18}\b").find_iter(text) {
546                let digits: String = m.as_str().chars().filter(|c| c.is_ascii_digit()).collect();
547                // The whole number run, not a Luhn-valid suffix of a longer
548                // one (`4111 1111 1111 1112` is not a card).
549                if !maximal_digit_run(text, m.start(), m.end())
550                    || !token_bounded(text, m.start(), m.end())
551                {
552                    continue;
553                }
554                // Luhn alone passes a tenth of random digit pairs (#694):
555                // the number must also open with a brand's IIN at that
556                // brand's length, and be grouped the way cards are printed.
557                if card_brand(&digits).is_some()
558                    && card_grouping_ok(m.as_str())
559                    && luhn_valid(&digits)
560                {
561                    out.push(Span::new(m.start(), m.end(), PiiKind::CreditCard));
562                }
563            }
564        }
565        if self.wants(PiiKind::Iban) {
566            for m in
567                re!(r"\b[A-Z]{2}\d{2}(?: ?[A-Z0-9]{4}){2,7}(?: ?[A-Z0-9]{1,4})?\b").find_iter(text)
568            {
569                let compact: String = m.as_str().chars().filter(|c| *c != ' ').collect();
570                if iban_valid(&compact) {
571                    out.push(Span::new(m.start(), m.end(), PiiKind::Iban));
572                }
573            }
574        }
575        if self.wants(PiiKind::IpAddress) {
576            for m in re!(r"\b(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}\b").find_iter(text) {
577                // Not the middle of a longer dotted run (`1.2.3.4.5`).
578                let before = text[..m.start()].chars().next_back();
579                let after = text[m.end()..].chars().next();
580                if before == Some('.') || after == Some('.') {
581                    continue;
582                }
583                // `v2.0.0.1` is a version, not an address.
584                if matches!(before, Some('v') | Some('V')) {
585                    continue;
586                }
587                out.push(Span::new(m.start(), m.end(), PiiKind::IpAddress));
588            }
589            for m in re!(r"[0-9A-Fa-f:]{3,45}").find_iter(text) {
590                let s = m.as_str();
591                if s.matches(':').count() < 2 || !s.chars().any(|c| c.is_ascii_hexdigit()) {
592                    continue;
593                }
594                if s.parse::<std::net::Ipv6Addr>().is_ok() {
595                    out.push(Span::new(m.start(), m.end(), PiiKind::IpAddress));
596                }
597            }
598        }
599        if self.wants(PiiKind::NationalId) {
600            // US SSN: AAA-GG-SSSS with the SSA's never-issued ranges excluded
601            // — and not a legal citation (`rule 412-18-0150`, `§ 310-20-1000`,
602            // #694): a citation word right before it says what it is.
603            for m in re!(r"\b\d{3}-\d{2}-\d{4}\b").find_iter(text) {
604                let s = m.as_str();
605                let area = &s[0..3];
606                if area == "000" || area == "666" || area.starts_with('9') {
607                    continue;
608                }
609                if &s[4..6] == "00" || &s[7..11] == "0000" {
610                    continue;
611                }
612                if !token_bounded(text, m.start(), m.end()) || citation_before(text, m.start()) {
613                    continue;
614                }
615                out.push(Span::new(m.start(), m.end(), PiiKind::NationalId));
616            }
617            // UK National Insurance number, written compact (`AB123456C`)
618            // or fully spaced (`AB 12 34 56 C`) — never half and half, which
619            // is how a part number and the next word line up (`TR482913 B`).
620            for m in re!(
621                r"\b[A-CEGHJ-PR-TW-Z][A-CEGHJ-NPR-TW-Z](?:\d{6}[A-D]| \d{2} \d{2} \d{2} [A-D])\b"
622            )
623            .find_iter(text)
624            {
625                let prefix = &m.as_str()[0..2];
626                if ["BG", "GB", "NK", "KN", "TN", "NT", "ZZ"].contains(&prefix) {
627                    continue;
628                }
629                if !token_bounded(text, m.start(), m.end()) {
630                    continue;
631                }
632                out.push(Span::new(m.start(), m.end(), PiiKind::NationalId));
633            }
634            // Indian Aadhaar: 12 digits, first 2–9, grouped 4-4-4 with one
635            // separator throughout (or none), the first group not a year
636            // (`2019 95753514` is a date and a record number), Verhoeff
637            // check digit.
638            for m in re!(r"\b[2-9]\d{3}([ \-]?)\d{4}([ \-]?)\d{4}\b").captures_iter(text) {
639                let whole = m.get(0).expect("match");
640                let (sep1, sep2) = (
641                    m.get(1).map_or("", |g| g.as_str()),
642                    m.get(2).map_or("", |g| g.as_str()),
643                );
644                if sep1 != sep2 || !token_bounded(text, whole.start(), whole.end()) {
645                    continue;
646                }
647                let s = whole.as_str();
648                if !sep1.is_empty() && (s.starts_with("19") || s.starts_with("20")) {
649                    continue;
650                }
651                let digits: String = s.chars().filter(|c| c.is_ascii_digit()).collect();
652                if verhoeff_valid(&digits) {
653                    out.push(Span::new(whole.start(), whole.end(), PiiKind::NationalId));
654                }
655            }
656        }
657        if self.wants(PiiKind::Phone) {
658            let ssn_shape = re!(r"^\d{3}-\d{2}-\d{4}$");
659            for m in re!(r"(?:\+\d{1,3}[ .\-]?)?(?:\(\d{1,4}\)[ .\-]?)?\d{2,4}(?:[ .\-]\d{2,4}){1,4}\b|\+\d{7,15}\b").find_iter(text) {
660                let s = m.as_str();
661                let digits = s.chars().filter(|c| c.is_ascii_digit()).count();
662                if !(7..=15).contains(&digits) || looks_like_date_or_version(s) {
663                    continue;
664                }
665                // `AAA-GG-SSSS` is an SSN shape, valid or not — never a phone.
666                if ssn_shape.is_match(s) {
667                    continue;
668                }
669                // Whole tokens only (#694): `IL 62704-1234` must not yield
670                // `2704-1234`, nor `60601 0001 12500` a run that starts
671                // inside a number.
672                if !token_bounded(text, m.start(), m.end()) {
673                    continue;
674                }
675                // Digit groups alone are not a phone — a ZIP+4, a rule
676                // number, two record IDs side by side all look like one.
677                // It takes a shape phones are written in: a `+` country
678                // code, a parenthesized area code, NANP's 3-3-4, a national
679                // number of ten or more digits in three or more groups, or
680                // a word nearby that says it is a phone.
681                if !phone_shape(s, digits) && !phone_keyword_before(text, m.start()) {
682                    continue;
683                }
684                out.push(Span::new(m.start(), m.end(), PiiKind::Phone));
685            }
686        }
687        for (name, re) in &self.custom {
688            for c in re.captures_iter(text) {
689                let m = c.get(1).or_else(|| c.get(0)).expect("match");
690                if m.start() == m.end() {
691                    continue;
692                }
693                out.push(Span {
694                    start: m.start(),
695                    end: m.end(),
696                    kind: PiiKind::Custom,
697                    score: 1.0,
698                    name: Some(name.clone()),
699                });
700            }
701        }
702        for re in &self.deny {
703            for m in re.find_iter(text) {
704                out.push(Span::new(m.start(), m.end(), PiiKind::Custom));
705            }
706        }
707        out.retain(|s| !self.allowed(&text[s.start..s.end]));
708        out
709    }
710}
711
712/// Whether `text[start..end]` is a whole run of digit groups: not preceded
713/// or followed by another digit group through a space or dash.
714fn maximal_digit_run(text: &str, start: usize, end: usize) -> bool {
715    let bytes = text.as_bytes();
716    let before =
717        start >= 2 && matches!(bytes[start - 1], b' ' | b'-') && bytes[start - 2].is_ascii_digit();
718    let after = end + 1 < bytes.len()
719        && matches!(bytes[end], b' ' | b'-')
720        && bytes[end + 1].is_ascii_digit();
721    !before && !after
722}
723
724/// Whether `text[start..end]` begins and ends at a token boundary: the
725/// character before and the one after are neither letters nor digits
726/// (#694 — no match may start or end inside a longer alphanumeric token).
727fn token_bounded(text: &str, start: usize, end: usize) -> bool {
728    let before = text[..start].chars().next_back();
729    let after = text[end..].chars().next();
730    // A match that opens with `+` or `(` is its own left edge: the runs of
731    // an inline paragraph are joined without the whitespace between them
732    // (`or` + `+1 (555) …`), and a letter before a `+` is no digit run.
733    let opens_itself = text[start..].starts_with(['+', '(']);
734    let left_ok = match before {
735        Some(c) if opens_itself => !c.is_ascii_digit(),
736        Some(c) => !c.is_alphanumeric(),
737        None => true,
738    };
739    left_ok && !after.is_some_and(|c| c.is_alphanumeric())
740}
741
742/// The shapes a phone number is written in (the digit groups alone are
743/// not one): `+` country code; `(area)`; NANP `3-3-4` or `1-3-3-4`; a
744/// national number of ten or eleven digits in three or more groups, or
745/// one with a trunk `0` of up to thirteen (`020 7946 0958`, `0151 496
746/// 0000`) — a `4-4-4` of twelve digits is a record number.
747fn phone_shape(s: &str, digits: usize) -> bool {
748    if s.starts_with('+') || s.contains('(') {
749        return true;
750    }
751    let groups: Vec<usize> = s
752        .split(|c: char| !c.is_ascii_digit())
753        .filter(|g| !g.is_empty())
754        .map(str::len)
755        .collect();
756    groups == [3, 3, 4]
757        || groups == [1, 3, 3, 4]
758        || (groups.len() >= 3
759            && ((10..=11).contains(&digits) || (s.starts_with('0') && digits <= 13)))
760}
761
762/// Whether a word that announces a phone number (`tel`, `phone`, `fax`,
763/// `mobile`, `call`, …) stands within a few words before `pos`.
764fn phone_keyword_before(text: &str, pos: usize) -> bool {
765    let head = &text[..pos];
766    let from = head.char_indices().rev().nth(30).map_or(0, |(i, _)| i);
767    re!(r"(?i)\b(?:tel|tel\.|telephone|phone|fax|mobile|cell|call|contact|whatsapp|hotline|dial|sms|text)\b[^\d]*$")
768        .is_match(&head[from..])
769}
770
771/// Whether a legal-citation word stands right before `pos`: `rule`,
772/// `section`, `sec.`, `§`, `code`, `article`, `art.`, `para.`, `clause`,
773/// `chapter`, `title`, `regulation`, `reg.`, `order`, `statute`, `part`.
774fn citation_before(text: &str, pos: usize) -> bool {
775    let head = text[..pos].trim_end();
776    let from = head.char_indices().rev().nth(16).map_or(0, |(i, _)| i);
777    re!(r"(?i)(?:\b(?:rule|rules|section|sections|sec|code|article|art|para|paragraph|clause|chapter|title|regulation|reg|order|statute|part|no|number)\.?|§+)\s*(?:no\.?\s*)?$")
778        .is_match(&head[from..])
779}
780
781/// The card brand a digit string opens with, at a length that brand issues
782/// (ISO/IEC 7812 IIN ranges): `None` for a number no brand would print.
783fn card_brand(digits: &str) -> Option<&'static str> {
784    let n = digits.len();
785    let p = |k: usize| digits[..k.min(n)].parse::<u32>().unwrap_or(0);
786    let (two, three, four) = (p(2), p(3), p(4));
787    if digits.starts_with('4') && matches!(n, 13 | 16 | 19) {
788        return Some("visa");
789    }
790    if ((51..=55).contains(&two) || (2221..=2720).contains(&four)) && n == 16 {
791        return Some("mastercard");
792    }
793    if (two == 34 || two == 37) && n == 15 {
794        return Some("amex");
795    }
796    if (four == 6011 || two == 65 || (644..=649).contains(&three)) && (16..=19).contains(&n) {
797        return Some("discover");
798    }
799    if ((300..=305).contains(&three) || two == 36 || two == 38 || two == 39)
800        && (14..=19).contains(&n)
801    {
802        return Some("diners");
803    }
804    if (3528..=3589).contains(&four) && (16..=19).contains(&n) {
805        return Some("jcb");
806    }
807    if two == 62 && (16..=19).contains(&n) {
808        return Some("unionpay");
809    }
810    if matches!(
811        four,
812        5018 | 5020 | 5038 | 5893 | 6304 | 6759 | 6761 | 6762 | 6763
813    ) && (12..=19).contains(&n)
814    {
815        return Some("maestro");
816    }
817    None
818}
819
820/// Whether a card number's separators fall where cards print them: none,
821/// groups of four (a last group of one to four), or Amex's 4-6-5 / Diners'
822/// 4-6-4. `88061052 76665755`, two record IDs, is 8-8.
823fn card_grouping_ok(s: &str) -> bool {
824    let groups: Vec<usize> = s
825        .split(|c: char| !c.is_ascii_digit())
826        .filter(|g| !g.is_empty())
827        .map(str::len)
828        .collect();
829    if groups.len() == 1 {
830        return true;
831    }
832    let (last, head) = groups.split_last().expect("non-empty");
833    (head.iter().all(|&g| g == 4) && (1..=4).contains(last))
834        || groups == [4, 6, 5]
835        || groups == [4, 6, 4]
836}
837
838/// Luhn (ISO/IEC 7812-1) check over a digit string.
839pub fn luhn_valid(digits: &str) -> bool {
840    let mut sum = 0u32;
841    let mut double = false;
842    for c in digits.chars().rev() {
843        let Some(d) = c.to_digit(10) else {
844            return false;
845        };
846        let d = if double {
847            let x = d * 2;
848            if x > 9 {
849                x - 9
850            } else {
851                x
852            }
853        } else {
854            d
855        };
856        sum += d;
857        double = !double;
858    }
859    digits.len() >= 2 && sum % 10 == 0
860}
861
862/// IBAN length per country (the SWIFT registry); an unknown country is not
863/// an IBAN for this detector, which keeps random uppercase-plus-digit tokens
864/// (`XY12…`) from matching on mod-97 luck alone.
865fn iban_length(country: &str) -> Option<usize> {
866    Some(match country {
867        "AL" | "AZ" | "CY" | "DO" | "GT" | "HU" | "LB" | "PL" | "BY" | "SV" | "NI" => 28,
868        "AD" | "CZ" | "MD" | "PK" | "RO" | "SA" | "SK" | "ES" | "SE" | "TN" | "VG" => 24,
869        "AT" | "BA" | "EE" | "KZ" | "LT" | "LU" | "XK" | "MN" => 20,
870        "BH" | "BG" | "CR" | "GE" | "DE" | "IE" | "ME" | "RS" | "GB" | "VA" => 22,
871        "BE" | "BI" => 16,
872        "BR" | "PS" | "QA" | "UA" | "EG" => 29,
873        "HR" | "LI" | "CH" => 21,
874        "DK" | "FO" | "FI" | "GL" | "NL" | "SD" | "FK" => 18,
875        "FR" | "GR" | "IT" | "MR" | "MC" | "SM" | "DJ" => 27,
876        "GI" | "IL" | "AE" | "IQ" | "TL" | "SO" | "OM" => 23,
877        "IS" => 26,
878        "JO" | "KW" | "MU" | "YE" => 30,
879        "LV" => 21,
880        "MK" | "SI" => 19,
881        "MT" => 31,
882        "NO" => 15,
883        "PT" | "ST" | "LY" => 25,
884        "TR" => 26,
885        "LC" => 32,
886        "RU" => 33,
887        _ => return None,
888    })
889}
890
891/// ISO 13616 mod-97 check over a compact (no spaces) IBAN.
892pub fn iban_valid(iban: &str) -> bool {
893    if iban.len() < 15 || iban.len() > 34 || !iban.is_ascii() {
894        return false;
895    }
896    let country = &iban[0..2];
897    if iban_length(country) != Some(iban.len()) {
898        return false;
899    }
900    let rearranged = format!("{}{}", &iban[4..], &iban[..4]);
901    let mut rem = 0u32;
902    for c in rearranged.chars() {
903        let v = match c {
904            '0'..='9' => c as u32 - '0' as u32,
905            'A'..='Z' => c as u32 - 'A' as u32 + 10,
906            _ => return false,
907        };
908        rem = if v >= 10 {
909            (rem * 100 + v) % 97
910        } else {
911            (rem * 10 + v) % 97
912        };
913    }
914    rem == 1
915}
916
917/// Verhoeff check (the Aadhaar check digit).
918pub fn verhoeff_valid(digits: &str) -> bool {
919    const D: [[u8; 10]; 10] = [
920        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
921        [1, 2, 3, 4, 0, 6, 7, 8, 9, 5],
922        [2, 3, 4, 0, 1, 7, 8, 9, 5, 6],
923        [3, 4, 0, 1, 2, 8, 9, 5, 6, 7],
924        [4, 0, 1, 2, 3, 9, 5, 6, 7, 8],
925        [5, 9, 8, 7, 6, 0, 4, 3, 2, 1],
926        [6, 5, 9, 8, 7, 1, 0, 4, 3, 2],
927        [7, 6, 5, 9, 8, 2, 1, 0, 4, 3],
928        [8, 7, 6, 5, 9, 3, 2, 1, 0, 4],
929        [9, 8, 7, 6, 5, 4, 3, 2, 1, 0],
930    ];
931    const P: [[u8; 10]; 8] = [
932        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
933        [1, 5, 7, 6, 2, 8, 3, 0, 9, 4],
934        [5, 8, 0, 3, 7, 9, 6, 1, 4, 2],
935        [8, 9, 1, 6, 0, 4, 3, 5, 2, 7],
936        [9, 4, 5, 3, 1, 2, 6, 8, 7, 0],
937        [4, 2, 8, 6, 5, 7, 3, 9, 0, 1],
938        [2, 7, 9, 3, 8, 0, 6, 4, 1, 5],
939        [7, 0, 4, 6, 9, 1, 3, 2, 5, 8],
940    ];
941    let mut c = 0u8;
942    for (i, ch) in digits.chars().rev().enumerate() {
943        let Some(d) = ch.to_digit(10) else {
944            return false;
945        };
946        c = D[c as usize][P[i % 8][d as usize] as usize];
947    }
948    !digits.is_empty() && c == 0
949}
950
951/// `2024-01-15`, `15.01.2024`, `01/15/24`, `1.2.3`, `10.0.1` — digit groups
952/// that are a date or a version number, not a phone.
953fn looks_like_date_or_version(s: &str) -> bool {
954    let s = s.trim();
955    re!(r"^\d{4}[\-./]\d{1,2}[\-./]\d{1,2}$").is_match(s)
956        || re!(r"^\d{1,2}[\-./]\d{1,2}[\-./]\d{2,4}$").is_match(s)
957        // Dotted groups with no other separator: a version string.
958        || (s.contains('.') && !s.contains(' ') && !s.contains('-') && !s.starts_with('+'))
959}
960
961// ---------------------------------------------------------------------------
962// The pass
963// ---------------------------------------------------------------------------
964
965/// The pass's state across one document: the pseudonym numbering and the
966/// report. [`DoclingDocument::redact`] drives it over a whole document; the
967/// streaming converter drives it over each page batch so the Markdown it
968/// emits matches the buffered export byte for byte.
969pub struct Redactor<'a> {
970    opts: &'a RedactionOptions,
971    detector: &'a dyn PiiDetector,
972    /// The allow terms and patterns, asked about every span whichever
973    /// detector produced it (an invalid pattern was rejected by the
974    /// options; here it is simply absent).
975    allow: AllowList,
976    /// `(label, original) → number`, first-seen order per label.
977    numbers: HashMap<(String, String), usize>,
978    next: HashMap<String, usize>,
979    report: RedactionReport,
980    /// Off while the item tree is walked: a DOCX / HTML document holds its
981    /// text twice (the flat nodes and the tree the JSON export reads), and
982    /// the report counts a value once, not per representation.
983    counting: bool,
984}
985
986impl<'a> Redactor<'a> {
987    pub fn new(opts: &'a RedactionOptions, detector: &'a dyn PiiDetector) -> Self {
988        Redactor {
989            opts,
990            detector,
991            allow: AllowList::new(opts).unwrap_or_default(),
992            numbers: HashMap::new(),
993            next: HashMap::new(),
994            report: RedactionReport {
995                mapping: opts.return_mapping.then(Vec::new),
996                ..Default::default()
997            },
998            counting: true,
999        }
1000    }
1001
1002    /// The report so far (the final one after the last batch).
1003    pub fn report(&self) -> &RedactionReport {
1004        &self.report
1005    }
1006
1007    pub fn finish(self) -> RedactionReport {
1008        self.report
1009    }
1010
1011    /// The spans of `text` the options keep, non-overlapping, in order:
1012    /// a longer span wins over a shorter one it overlaps, equal lengths by
1013    /// kind precedence; NER spans under the score floor and out-of-scope
1014    /// kinds are dropped.
1015    fn spans(&self, text: &str) -> Vec<Span> {
1016        let mut spans: Vec<Span> = self
1017            .detector
1018            .detect(text)
1019            .into_iter()
1020            .filter(|s| s.start < s.end && s.end <= text.len())
1021            .filter(|s| text.is_char_boundary(s.start) && text.is_char_boundary(s.end))
1022            .filter(|s| self.opts.wants(s.kind))
1023            .filter(|s| !PiiKind::NER.contains(&s.kind) || s.score >= self.opts.ner_min_score)
1024            .filter(|s| !self.allow.allows(&text[s.start..s.end]))
1025            .collect();
1026        spans.sort_by(|a, b| {
1027            a.start
1028                .cmp(&b.start)
1029                .then((b.end - b.start).cmp(&(a.end - a.start)))
1030                .then(a.kind.cmp(&b.kind))
1031        });
1032        let mut kept: Vec<Span> = Vec::with_capacity(spans.len());
1033        for s in spans {
1034            match kept.last() {
1035                Some(last) if s.start < last.end => {
1036                    // Overlap: keep the longer (or, equal, the earlier kind).
1037                    if s.end - s.start > last.end - last.start {
1038                        kept.pop();
1039                        kept.push(s);
1040                    }
1041                }
1042                _ => kept.push(s),
1043            }
1044        }
1045        kept
1046    }
1047
1048    /// Whether `text` carries any span the options keep — what the image
1049    /// box-out asks per OCR line.
1050    pub fn has_pii(&self, text: &str) -> bool {
1051        !self.spans(text).is_empty()
1052    }
1053
1054    fn label_of(span: &Span) -> String {
1055        span.name
1056            .clone()
1057            .unwrap_or_else(|| span.kind.label().to_string())
1058    }
1059
1060    /// The placeholder for one span of `original`.
1061    fn placeholder(&mut self, span: &Span, original: &str) -> String {
1062        let label = Self::label_of(span);
1063        if self.counting {
1064            *self.report.counts.entry(label.clone()).or_insert(0) += 1;
1065            self.report.total += 1;
1066        }
1067        let out = match &self.opts.replacement {
1068            Replacement::Label => format!("[{label}]"),
1069            Replacement::Fixed(s) => s.clone(),
1070            Replacement::Pseudonym => {
1071                // The same value — case-folded for e-mails and names,
1072                // whitespace-folded for numbers — gets the same number.
1073                let key = normalize(original);
1074                let n = match self.numbers.get(&(label.clone(), key.clone())) {
1075                    Some(n) => *n,
1076                    None => {
1077                        let next = self.next.entry(label.clone()).or_insert(0);
1078                        *next += 1;
1079                        self.numbers.insert((label.clone(), key), *next);
1080                        *next
1081                    }
1082                };
1083                format!("[{label}_{n}]")
1084            }
1085        };
1086        if let Some(map) = self.report.mapping.as_mut() {
1087            if !map.iter().any(|(o, _)| o == original) {
1088                map.push((original.to_string(), out.clone()));
1089            }
1090        }
1091        out
1092    }
1093
1094    /// Redact one string in place.
1095    pub fn redact_text(&mut self, text: &mut String) {
1096        let spans = self.spans(text);
1097        if spans.is_empty() {
1098            return;
1099        }
1100        let mut out = String::with_capacity(text.len());
1101        let mut pos = 0;
1102        for span in &spans {
1103            out.push_str(&text[pos..span.start]);
1104            let original = text[span.start..span.end].to_string();
1105            out.push_str(&self.placeholder(span, &original));
1106            pos = span.end;
1107        }
1108        out.push_str(&text[pos..]);
1109        *text = out;
1110    }
1111
1112    fn redact_opt(&mut self, text: &mut Option<String>) {
1113        if let Some(t) = text.as_mut() {
1114            self.redact_text(t);
1115        }
1116    }
1117
1118    /// Redact a run sequence as one text: spans are found on the joined
1119    /// runs and written back — the run the span starts in gets the
1120    /// placeholder, the runs it crosses lose the covered characters.
1121    pub fn redact_runs(&mut self, runs: &mut [InlineRun]) {
1122        let joined: String = runs.iter().map(|r| r.text.as_str()).collect();
1123        let spans = self.spans(&joined);
1124        if spans.is_empty() {
1125            return;
1126        }
1127        // Run byte ranges in the joined text.
1128        let mut starts = Vec::with_capacity(runs.len());
1129        let mut acc = 0;
1130        for r in runs.iter() {
1131            starts.push(acc);
1132            acc += r.text.len();
1133        }
1134        // Rebuild every run's text from the joined text with the spans
1135        // applied: for each run, walk its range and copy the pieces outside
1136        // spans; a span starting inside the run contributes its placeholder.
1137        let mut placeholders: Vec<String> = Vec::with_capacity(spans.len());
1138        for span in &spans {
1139            let original = joined[span.start..span.end].to_string();
1140            placeholders.push(self.placeholder(span, &original));
1141        }
1142        for (i, run) in runs.iter_mut().enumerate() {
1143            let (rs, re) = (starts[i], starts[i] + run.text.len());
1144            let mut out = String::new();
1145            let mut pos = rs;
1146            for (span, ph) in spans.iter().zip(&placeholders) {
1147                if span.end <= rs || span.start >= re {
1148                    continue;
1149                }
1150                let s = span.start.max(rs);
1151                out.push_str(&joined[pos..s]);
1152                if span.start >= rs {
1153                    out.push_str(ph);
1154                }
1155                pos = span.end.min(re);
1156            }
1157            out.push_str(&joined[pos..re]);
1158            run.text = out;
1159        }
1160    }
1161
1162    fn redact_table(&mut self, table: &mut Table) {
1163        for row in table.rows.iter_mut() {
1164            for cell in row.iter_mut() {
1165                self.redact_text(cell);
1166            }
1167        }
1168        if let Some(cells) = table.cells.as_mut() {
1169            for c in cells.iter_mut() {
1170                self.redact_text(&mut c.text);
1171            }
1172        }
1173        if let Some(blocks) = table.cell_blocks.as_mut() {
1174            for row in blocks.iter_mut() {
1175                for cell in row.iter_mut() {
1176                    self.redact_nodes(cell);
1177                }
1178            }
1179        }
1180        self.redact_opt(&mut table.caption);
1181    }
1182
1183    fn redact_fields(&mut self, items: &mut [crate::FieldItem]) {
1184        for f in items.iter_mut() {
1185            self.redact_opt(&mut f.marker);
1186            self.redact_opt(&mut f.key);
1187            self.redact_opt(&mut f.value);
1188        }
1189    }
1190
1191    fn redact_graph(&mut self, cells: &mut [crate::GraphCell]) {
1192        for c in cells.iter_mut() {
1193            self.redact_text(&mut c.text);
1194            self.redact_text(&mut c.orig);
1195        }
1196    }
1197
1198    fn redact_image(&mut self, image: &mut Option<crate::PictureImage>) {
1199        if self.opts.images == ImageRedaction::Drop {
1200            *image = None;
1201        }
1202    }
1203
1204    /// Redact every string of `nodes`, recursively.
1205    pub fn redact_nodes(&mut self, nodes: &mut [Node]) {
1206        for node in nodes.iter_mut() {
1207            self.redact_node(node);
1208        }
1209    }
1210
1211    fn redact_node(&mut self, node: &mut Node) {
1212        // A marked PDF item (#1005) is redacted as its plain item: the runs
1213        // split a name across a script boundary (`Smith²`) where a pattern
1214        // would miss it, and each value is counted once.
1215        if let Node::ScriptText { inner, .. } = node {
1216            let plain = std::mem::replace(inner.as_mut(), Node::PageBreak);
1217            *node = plain;
1218        }
1219        match node {
1220            Node::Heading { text, .. }
1221            | Node::Paragraph { text }
1222            | Node::ListText { text, .. }
1223            | Node::CheckboxItem { text, .. }
1224            | Node::PageFurniture { text, .. }
1225            | Node::FurnitureText { text, .. }
1226            | Node::TextDump(text) => self.redact_text(text),
1227            Node::ListItem {
1228                text,
1229                marker,
1230                dclx,
1231                split,
1232                href,
1233                ..
1234            } => {
1235                self.redact_text(text);
1236                self.redact_opt(marker);
1237                self.redact_opt(href);
1238                if let Some(s) = split.as_mut() {
1239                    self.redact_text(&mut s.orig);
1240                    self.redact_text(&mut s.text);
1241                    self.redact_text(&mut s.marker);
1242                }
1243                if let Some(d) = dclx.as_mut() {
1244                    self.redact_text(&mut d.text);
1245                    self.redact_opt(&mut d.marker);
1246                    self.redact_runs(&mut d.runs);
1247                }
1248            }
1249            Node::Code {
1250                text, orig, pretty, ..
1251            } => {
1252                self.redact_text(text);
1253                self.redact_opt(orig);
1254                self.redact_opt(pretty);
1255            }
1256            Node::Table(t) => self.redact_table(t),
1257            Node::Picture {
1258                caption,
1259                caption_href,
1260                image,
1261                description,
1262                ..
1263            } => {
1264                self.redact_opt(caption);
1265                self.redact_opt(caption_href);
1266                // The picture-OCR text (#645) is read off the image: redact
1267                // it like any other text.
1268                if let Some(d) = description.as_mut() {
1269                    self.redact_text(&mut d.text);
1270                }
1271                self.redact_image(image);
1272            }
1273            // The LaTeX is a rendering of the formula's glyphs; `orig` is
1274            // the extracted text a reader could recover a value from.
1275            Node::Formula { orig, .. } => self.redact_text(orig),
1276            Node::Caption { text, href } | Node::LabeledText { text, href, .. } => {
1277                self.redact_text(text);
1278                self.redact_opt(href);
1279            }
1280            Node::Chart { table, caption, .. } => {
1281                self.redact_table(table);
1282                self.redact_opt(caption);
1283            }
1284            Node::Group { name, children, .. } => {
1285                self.redact_opt(name);
1286                self.redact_nodes(children);
1287            }
1288            Node::FieldRegion { items } => self.redact_fields(items),
1289            Node::KeyValueGraph { cells, .. } => self.redact_graph(cells),
1290            Node::InlineGroup { runs, md_text, .. } => {
1291                self.redact_runs(runs);
1292                self.redact_text(md_text);
1293            }
1294            // Replaced by its plain item above.
1295            Node::ScriptText { .. } => {}
1296            Node::Furniture { inner, .. }
1297            | Node::Commented { inner, .. }
1298            | Node::Located { inner, .. }
1299            | Node::Prov { inner, .. }
1300            | Node::DoclangOnly(inner) => self.redact_node(inner),
1301            // A transcript cue (#694): the words are redacted once, in
1302            // `cue`, and spliced into the paragraph behind its `[time: …]`
1303            // prefix — the timing is never a phone number, and the value is
1304            // counted once, not per representation.
1305            Node::Track { track, cue, inner } => {
1306                self.redact_opt(&mut track.voice);
1307                let original = cue.clone();
1308                self.redact_text(cue);
1309                match inner.as_mut() {
1310                    Node::Paragraph { text, .. } if text.ends_with(&original) => {
1311                        let prefix_len = text.len() - original.len();
1312                        text.truncate(prefix_len);
1313                        text.push_str(cue);
1314                    }
1315                    other => {
1316                        let counting = std::mem::replace(&mut self.counting, false);
1317                        self.redact_node(other);
1318                        self.counting = counting;
1319                    }
1320                }
1321            }
1322            Node::CommentSection { name, text, .. } => {
1323                self.redact_text(name);
1324                self.redact_text(text);
1325            }
1326            Node::PictureChildren(children) => self.redact_nodes(children),
1327            Node::PageBreak | Node::PageInfo { .. } | Node::Anchor { .. } => {}
1328        }
1329    }
1330
1331    /// Redact the item tree (#621: the JSON export reads it, not the nodes,
1332    /// for the backends that build one). The same pseudonym numbering as
1333    /// the nodes got; the counts are not incremented again — the tree is
1334    /// the same text in docling's shape, not more of it.
1335    pub fn redact_tree(&mut self, tree: &mut ItemTree) {
1336        let counting = std::mem::replace(&mut self.counting, false);
1337        self.redact_tree_items(tree);
1338        self.counting = counting;
1339    }
1340
1341    fn redact_tree_items(&mut self, tree: &mut ItemTree) {
1342        for item in tree.items.iter_mut() {
1343            match &mut item.kind {
1344                TreeKind::Text {
1345                    text,
1346                    orig,
1347                    hyperlink,
1348                    list,
1349                    ..
1350                } => {
1351                    self.redact_text(text);
1352                    self.redact_opt(orig);
1353                    self.redact_opt(hyperlink);
1354                    if let Some(l) = list.as_mut() {
1355                        self.redact_text(&mut l.marker);
1356                    }
1357                }
1358                TreeKind::Code {
1359                    text,
1360                    orig,
1361                    hyperlink,
1362                    ..
1363                } => {
1364                    self.redact_text(text);
1365                    self.redact_opt(orig);
1366                    self.redact_opt(hyperlink);
1367                }
1368                TreeKind::Group { name, .. } => self.redact_text(name),
1369                TreeKind::Table { table, .. } => self.redact_table(table),
1370                TreeKind::Picture {
1371                    image,
1372                    chart,
1373                    description,
1374                    ..
1375                } => {
1376                    self.redact_image(image);
1377                    if let Some(d) = description.as_mut() {
1378                        self.redact_text(&mut d.text);
1379                    }
1380                    if let Some(t) = chart.as_mut() {
1381                        self.redact_table(t);
1382                    }
1383                }
1384                TreeKind::FieldRegion { items } => self.redact_fields(items),
1385                TreeKind::KeyValueGraph { cells, .. } => self.redact_graph(cells),
1386            }
1387            if let Some(track) = item.source.as_mut() {
1388                self.redact_opt(&mut track.voice);
1389            }
1390            for note in item.notes.iter_mut() {
1391                self.redact_text(&mut note.text);
1392            }
1393        }
1394    }
1395
1396    /// Redact the link table (`(text, href)` pairs).
1397    pub fn redact_links(&mut self, links: &mut [(String, String)]) {
1398        for (text, href) in links.iter_mut() {
1399            self.redact_text(text);
1400            self.redact_text(href);
1401        }
1402    }
1403
1404    /// The whole document: nodes, tree, links, page renders.
1405    pub fn redact_document(&mut self, doc: &mut DoclingDocument) {
1406        self.redact_nodes(&mut doc.nodes);
1407        if let Some(tree) = doc.tree.as_mut() {
1408            self.redact_tree(tree);
1409        }
1410        self.redact_links(&mut doc.links);
1411        self.redact_metadata(&mut doc.metadata);
1412        if self.opts.images == ImageRedaction::Drop {
1413            doc.page_images.clear();
1414        }
1415    }
1416
1417    /// The document's metadata (#679). The author fields — `authors`,
1418    /// `last_modified_by`, EXIF's `Artist` / `XPAuthor`, an audio file's
1419    /// `artist` / `album_artist` / `composer` / `performer` — are names by
1420    /// definition, so each is masked whole as a `PERSON` without asking a
1421    /// detector (an allow term still exempts it); the GPS position and the
1422    /// `GPS*` EXIF tags are dropped, counted as `LOCATION`; every other
1423    /// string goes through the detectors like a paragraph (an e-mail in a
1424    /// description, a phone number in a comment). The kinds filter applies:
1425    /// `kinds=email` leaves the names and the position alone.
1426    pub fn redact_metadata(&mut self, m: &mut crate::metadata::DocumentMetadata) {
1427        if self.opts.wants(PiiKind::Person) {
1428            for a in m.authors.iter_mut() {
1429                self.redact_whole(a, PiiKind::Person);
1430            }
1431            if let Some(by) = m.last_modified_by.as_mut() {
1432                self.redact_whole(by, PiiKind::Person);
1433            }
1434            for key in ["Artist", "XPAuthor"] {
1435                if let Some(v) = m.exif.get_mut(key) {
1436                    self.redact_whole(v, PiiKind::Person);
1437                }
1438            }
1439            for key in ["artist", "album_artist", "composer", "performer"] {
1440                if let Some(v) = m.audio.get_mut(key) {
1441                    self.redact_whole(v, PiiKind::Person);
1442                }
1443            }
1444        }
1445        if self.opts.wants(PiiKind::Location) {
1446            let had_gps = m.gps.take().is_some();
1447            let before = m.exif.len();
1448            m.exif.retain(|k, _| !k.starts_with("GPS"));
1449            if had_gps || m.exif.len() != before {
1450                self.count(PiiKind::Location.label());
1451            }
1452        }
1453        self.redact_opt(&mut m.title);
1454        self.redact_opt(&mut m.subject);
1455        self.redact_opt(&mut m.description);
1456        for k in m.keywords.iter_mut() {
1457            self.redact_text(k);
1458        }
1459        self.redact_opt(&mut m.publisher);
1460        self.redact_opt(&mut m.rights);
1461        self.redact_opt(&mut m.identifier);
1462        self.redact_opt(&mut m.creator_tool);
1463        for v in m.properties.values_mut() {
1464            self.redact_text(v);
1465        }
1466        for (k, v) in m.exif.iter_mut() {
1467            if k != "Artist" && k != "XPAuthor" {
1468                self.redact_text(v);
1469            }
1470        }
1471        for (k, v) in m.audio.iter_mut() {
1472            if !matches!(
1473                k.as_str(),
1474                "artist" | "album_artist" | "composer" | "performer"
1475            ) {
1476                self.redact_text(v);
1477            }
1478        }
1479    }
1480
1481    /// Replace a whole value with the placeholder of `kind` — a field that
1482    /// is a name by definition. An empty value and an allowed term stay.
1483    fn redact_whole(&mut self, text: &mut String, kind: PiiKind) {
1484        if text.trim().is_empty() || self.allow.allows(text) {
1485            return;
1486        }
1487        let span = Span::new(0, text.len(), kind);
1488        let original = text.clone();
1489        *text = self.placeholder(&span, &original);
1490    }
1491
1492    /// Count a removal that leaves no placeholder behind.
1493    fn count(&mut self, label: &str) {
1494        if self.counting {
1495            *self.report.counts.entry(label.to_string()).or_insert(0) += 1;
1496            self.report.total += 1;
1497        }
1498    }
1499}
1500
1501/// The pseudonym key of a value: case-folded, inner whitespace collapsed,
1502/// so `John Smith` / `john smith` / `JOHN  SMITH` share one number and
1503/// `+1 555 123 4567` / `+1-555-123-4567` do too.
1504fn normalize(s: &str) -> String {
1505    s.to_lowercase()
1506        .chars()
1507        .filter(|c| !c.is_whitespace() && *c != '-' && *c != '.' && *c != '(' && *c != ')')
1508        .collect()
1509}
1510
1511impl DoclingDocument {
1512    /// Redact the document in place with `detector` (#621) and return what
1513    /// was redacted. See the [module docs](self) for what is walked. The
1514    /// built-in [`PatternDetector`] is `PatternDetector::new(&opts)?`.
1515    pub fn redact(
1516        &mut self,
1517        opts: &RedactionOptions,
1518        detector: &dyn PiiDetector,
1519    ) -> RedactionReport {
1520        let mut r = Redactor::new(opts, detector);
1521        r.redact_document(self);
1522        r.finish()
1523    }
1524}
1525
1526#[cfg(test)]
1527mod tests {
1528    use super::*;
1529
1530    fn detector(opts: &RedactionOptions) -> PatternDetector {
1531        PatternDetector::new(opts).unwrap()
1532    }
1533
1534    fn redact(text: &str) -> String {
1535        let opts = RedactionOptions::default();
1536        let d = detector(&opts);
1537        let mut s = text.to_string();
1538        Redactor::new(&opts, &d).redact_text(&mut s);
1539        s
1540    }
1541
1542    fn kinds(text: &str) -> Vec<PiiKind> {
1543        let opts = RedactionOptions::default();
1544        let d = detector(&opts);
1545        Redactor::new(&opts, &d)
1546            .spans(text)
1547            .into_iter()
1548            .map(|s| s.kind)
1549            .collect()
1550    }
1551
1552    #[test]
1553    fn emails_including_markdown_links_and_escapes() {
1554        assert_eq!(
1555            redact("Write to [john.doe@example.com](mailto:john.doe@example.com)."),
1556            "Write to [[EMAIL]](mailto:[EMAIL])."
1557        );
1558        // docling escapes `_` in text; the escape is part of the address.
1559        assert_eq!(
1560            redact(r"jane\_doe@example.org, please."),
1561            "[EMAIL], please."
1562        );
1563        assert_eq!(redact("a@b"), "a@b");
1564    }
1565
1566    #[test]
1567    fn cards_need_luhn_and_ibans_need_mod97() {
1568        assert_eq!(kinds("card 4111 1111 1111 1111"), vec![PiiKind::CreditCard]);
1569        assert_eq!(kinds("card 4111-1111-1111-1111"), vec![PiiKind::CreditCard]);
1570        assert_eq!(kinds("order 4111 1111 1111 1112"), vec![]);
1571        assert_eq!(
1572            kinds("IBAN DE89 3704 0044 0532 0130 00"),
1573            vec![PiiKind::Iban]
1574        );
1575        assert_eq!(kinds("IBAN GB82WEST12345698765432"), vec![PiiKind::Iban]);
1576        assert_eq!(kinds("IBAN DE89 3704 0044 0532 0130 01"), vec![]);
1577        assert_eq!(kinds("code XY12 ABCD 1234 EFGH 5678"), vec![]);
1578    }
1579
1580    #[test]
1581    fn phones_but_not_dates_or_versions() {
1582        assert_eq!(kinds("call +1 (555) 123-4567"), vec![PiiKind::Phone]);
1583        assert_eq!(kinds("call 555-123-4567 today"), vec![PiiKind::Phone]);
1584        assert_eq!(kinds("tel +44 20 7946 0958"), vec![PiiKind::Phone]);
1585        assert_eq!(kinds("on 2024-01-15 and 15.01.2024 and 01/15/24"), vec![]);
1586        assert_eq!(kinds("version 1.2.3 and v10.0.1 and v2.0.0.1"), vec![]);
1587        assert_eq!(kinds("ISBN 978-3-16-148410-0"), vec![]);
1588    }
1589
1590    #[test]
1591    fn ip_addresses_and_url_credentials() {
1592        assert_eq!(kinds("host 192.168.1.10 up"), vec![PiiKind::IpAddress]);
1593        assert_eq!(kinds("v 1.2.3.4.5"), vec![]);
1594        assert_eq!(
1595            kinds("addr 2001:db8::1 and ::1"),
1596            vec![PiiKind::IpAddress, PiiKind::IpAddress]
1597        );
1598        assert_eq!(kinds("at 12:30:45 today"), vec![]);
1599        assert_eq!(kinds("mac aa:bb:cc:dd:ee:ff"), vec![]);
1600        assert_eq!(
1601            redact("see https://alice:s3cret@db.example.com/x"),
1602            "see https://[CREDENTIALS]@db.example.com/x"
1603        );
1604    }
1605
1606    #[test]
1607    fn national_ids_with_their_rules() {
1608        assert_eq!(kinds("SSN 123-45-6789"), vec![PiiKind::NationalId]);
1609        assert_eq!(
1610            kinds("not 000-45-6789 nor 123-00-6789 nor 900-45-6789"),
1611            vec![]
1612        );
1613        assert_eq!(kinds("NINO AB 12 34 56 C"), vec![PiiKind::NationalId]);
1614        assert_eq!(kinds("NINO QQ 12 34 56 C"), vec![]);
1615        assert_eq!(kinds("NINO BG 12 34 56 C"), vec![]);
1616        // Verhoeff: 2234 5678 9012 fails, the digit that passes is found by
1617        // the check itself.
1618        let base = "22345678901";
1619        let ok = (0..10)
1620            .map(|d| format!("{base}{d}"))
1621            .find(|s| verhoeff_valid(s))
1622            .unwrap();
1623        assert_eq!(kinds(&format!("Aadhaar {ok}")), vec![PiiKind::NationalId]);
1624        let bad = (0..10)
1625            .map(|d| format!("{base}{d}"))
1626            .find(|s| !verhoeff_valid(s))
1627            .unwrap();
1628        assert_eq!(kinds(&format!("Aadhaar {bad}")), vec![]);
1629    }
1630
1631    #[test]
1632    fn replacement_modes_and_pseudonym_consistency() {
1633        let opts = RedactionOptions {
1634            replacement: Replacement::Pseudonym,
1635            return_mapping: true,
1636            ..Default::default()
1637        };
1638        let d = detector(&opts);
1639        let mut r = Redactor::new(&opts, &d);
1640        let mut a = "a@x.com wrote to b@x.com; A@X.COM again".to_string();
1641        r.redact_text(&mut a);
1642        assert_eq!(a, "[EMAIL_1] wrote to [EMAIL_2]; [EMAIL_1] again");
1643        let mut b = "call 555-123-4567 or 555 123 4567".to_string();
1644        r.redact_text(&mut b);
1645        assert_eq!(b, "call [PHONE_1] or [PHONE_1]");
1646        let report = r.finish();
1647        assert_eq!(report.counts["EMAIL"], 3);
1648        assert_eq!(report.counts["PHONE"], 2);
1649        assert_eq!(report.total, 5);
1650        let map = report.mapping.unwrap();
1651        assert!(map.contains(&("a@x.com".into(), "[EMAIL_1]".into())));
1652        assert_eq!(map.len(), 5);
1653
1654        let opts = RedactionOptions {
1655            replacement: Replacement::Fixed("***".into()),
1656            ..Default::default()
1657        };
1658        let d = detector(&opts);
1659        let mut s = "a@x.com".to_string();
1660        Redactor::new(&opts, &d).redact_text(&mut s);
1661        assert_eq!(s, "***");
1662    }
1663
1664    #[test]
1665    fn kinds_filter_custom_patterns_deny_and_allow_terms() {
1666        let opts = RedactionOptions {
1667            kinds: vec![PiiKind::Phone],
1668            ..Default::default()
1669        };
1670        let d = detector(&opts);
1671        let mut s = "a@x.com 555-123-4567".to_string();
1672        Redactor::new(&opts, &d).redact_text(&mut s);
1673        assert_eq!(s, "a@x.com [PHONE]");
1674
1675        let opts = RedactionOptions {
1676            custom_patterns: vec![CustomPattern::parse("case_id=CASE-\\d{4}").unwrap()],
1677            deny_terms: vec!["Project Falcon".into()],
1678            allow_terms: vec!["support@example.com".into()],
1679            ..Default::default()
1680        };
1681        let d = detector(&opts);
1682        let mut s = "CASE-0042 by project falcon: support@example.com / x@example.com".to_string();
1683        Redactor::new(&opts, &d).redact_text(&mut s);
1684        assert_eq!(s, "[CASE_ID] by [REDACTED]: support@example.com / [EMAIL]");
1685        assert!(PatternDetector::new(&RedactionOptions {
1686            custom_patterns: vec![CustomPattern {
1687                name: "bad".into(),
1688                regex: "(".into()
1689            }],
1690            ..Default::default()
1691        })
1692        .is_err());
1693    }
1694
1695    /// #694: the nine misfires reported on real documents stay as they are
1696    /// with the five pattern kinds selected; the control phone goes.
1697    #[test]
1698    fn postal_codes_citations_record_ids_and_part_numbers_are_not_pii() {
1699        let opts = RedactionOptions {
1700            kinds: vec![
1701                PiiKind::Phone,
1702                PiiKind::CreditCard,
1703                PiiKind::Iban,
1704                PiiKind::NationalId,
1705                PiiKind::IpAddress,
1706            ],
1707            ..Default::default()
1708        };
1709        let d = detector(&opts);
1710        let untouched = [
1711            "Ship to: Example Corp, 100 Main Street, Springfield, IL 62704-1234.",
1712            "Send comments to Department 55A, Mail Station 1234 5600 Elm Road.",
1713            "Leave accrual is defined in rule 412-18-205 of the state code.",
1714            "Leave accrual is defined in rule 412-18-0150 of the state code.",
1715            "The input file contains rows such as 60601 0001 12500 00350 and 60602 0002 9800 00120.",
1716            "Replace the bracket with part number TR482913 B before assembly.",
1717            "/XY123456 A configuration file is required for every job.",
1718            "Position created in 2019 95753514 for the regional office.",
1719            "Employee 88061052 76665755 Clerk Grade 4",
1720        ];
1721        for text in untouched {
1722            let mut s = text.to_string();
1723            Redactor::new(&opts, &d).redact_text(&mut s);
1724            assert_eq!(s, text);
1725        }
1726        let mut s = "Call the help desk at +1 555 010 0199 for assistance.".to_string();
1727        Redactor::new(&opts, &d).redact_text(&mut s);
1728        assert_eq!(s, "Call the help desk at [PHONE] for assistance.");
1729    }
1730
1731    /// … while the real values keep being found, in every shape they are
1732    /// written in.
1733    #[test]
1734    fn real_values_are_still_redacted() {
1735        assert_eq!(kinds("+1 555 010 0199"), vec![PiiKind::Phone]);
1736        assert_eq!(kinds("(555) 010-0199"), vec![PiiKind::Phone]);
1737        assert_eq!(kinds("555-010-0199"), vec![PiiKind::Phone]);
1738        assert_eq!(kinds("Tel: 020 7946 0958"), vec![PiiKind::Phone]);
1739        assert_eq!(kinds("020 7946 0958"), vec![PiiKind::Phone]);
1740        assert_eq!(kinds("mobile 0151 496 0000"), vec![PiiKind::Phone]);
1741        assert_eq!(kinds("phone: 1234 5600"), vec![PiiKind::Phone]);
1742        assert_eq!(kinds("Mail Station 1234 5600"), vec![]);
1743        assert_eq!(kinds("card 4111 1111 1111 1111"), vec![PiiKind::CreditCard]);
1744        assert_eq!(kinds("amex 378282246310005"), vec![PiiKind::CreditCard]);
1745        assert_eq!(kinds("amex 3782 822463 10005"), vec![PiiKind::CreditCard]);
1746        assert_eq!(kinds("mc 5555 5555 5555 4444"), vec![PiiKind::CreditCard]);
1747        assert_eq!(kinds("mc 2223 0031 2200 3222"), vec![PiiKind::CreditCard]);
1748        // Luhn-valid, no brand: an order number.
1749        assert_eq!(kinds("order 8806105276665755"), vec![]);
1750        // A card number split as a phone is not a card.
1751        assert_eq!(
1752            kinds(
1753                "4111 1111 1111 1111"
1754                    .replace(' ', "-")
1755                    .replace("1111-1111-1111", "11111111-1111")
1756                    .as_str()
1757            ),
1758            vec![]
1759        );
1760        assert_eq!(kinds("NINO AB123456C"), vec![PiiKind::NationalId]);
1761        assert_eq!(kinds("NINO AB 12 34 56 C"), vec![PiiKind::NationalId]);
1762        assert_eq!(kinds("NINO AB123456 C"), vec![]);
1763        assert_eq!(kinds("NINO AB 123456C"), vec![]);
1764        assert_eq!(kinds("SSN 123-45-6789"), vec![PiiKind::NationalId]);
1765        assert_eq!(kinds("ID 412-18-0150 on file"), vec![PiiKind::NationalId]);
1766        assert_eq!(kinds("see § 412-18-0150 and section 412-18-0151"), vec![]);
1767        assert_eq!(
1768            kinds("IBAN GB82 WEST 1234 5698 7654 32"),
1769            vec![PiiKind::Iban]
1770        );
1771        // Aadhaar: consistent separators; a year first group is a date.
1772        let base = "22345678901";
1773        let ok = (0..10)
1774            .map(|d| format!("{base}{d}"))
1775            .find(|s| verhoeff_valid(s))
1776            .unwrap();
1777        let spaced = format!("{} {} {}", &ok[..4], &ok[4..8], &ok[8..]);
1778        assert_eq!(
1779            kinds(&format!("Aadhaar {spaced}")),
1780            vec![PiiKind::NationalId]
1781        );
1782        assert_eq!(kinds(&format!("Aadhaar {} {}", &ok[..4], &ok[4..])), vec![]);
1783        let year_base = "20199575351";
1784        let year_ok = (0..10)
1785            .map(|d| format!("{year_base}{d}"))
1786            .find(|s| verhoeff_valid(s))
1787            .unwrap();
1788        assert_eq!(
1789            kinds(&format!(
1790                "in {} {} {}",
1791                &year_ok[..4],
1792                &year_ok[4..8],
1793                &year_ok[8..]
1794            )),
1795            vec![]
1796        );
1797        assert_eq!(
1798            kinds(&format!("Aadhaar {year_ok}")),
1799            vec![PiiKind::NationalId]
1800        );
1801    }
1802
1803    /// No match starts or ends inside a longer alphanumeric token.
1804    #[test]
1805    fn matches_are_whole_tokens() {
1806        assert_eq!(redact("IL 62704-1234 stays"), "IL 62704-1234 stays");
1807        assert_eq!(
1808            redact("row 60602 0002 9800 00120 stays"),
1809            "row 60602 0002 9800 00120 stays"
1810        );
1811        assert_eq!(redact("id x555-010-0199"), "id x555-010-0199");
1812        assert_eq!(redact("id 555-010-0199x"), "id 555-010-0199x");
1813        assert_eq!(redact("ref 4111111111111111a"), "ref 4111111111111111a");
1814        assert_eq!(redact("ref AB123456CD"), "ref AB123456CD");
1815        assert_eq!(redact("call 555-010-0199."), "call [PHONE].");
1816    }
1817
1818    /// Allow patterns: a document's own number shapes pass every detector.
1819    #[test]
1820    fn allow_patterns_exempt_matching_spans() {
1821        let opts = RedactionOptions {
1822            allow_patterns: vec![r"\+1 555 010 \d{4}".into(), r"[a-z]+@example\.com".into()],
1823            ..Default::default()
1824        };
1825        let d = detector(&opts);
1826        let mut s =
1827            "+1 555 010 0199, +1 555 010 0200, +1 555 011 0200, bob@example.com, bob@example.org"
1828                .to_string();
1829        Redactor::new(&opts, &d).redact_text(&mut s);
1830        assert_eq!(
1831            s,
1832            "+1 555 010 0199, +1 555 010 0200, [PHONE], bob@example.com, [EMAIL]"
1833        );
1834        assert!(PatternDetector::new(&RedactionOptions {
1835            allow_patterns: vec!["(".into()],
1836            ..Default::default()
1837        })
1838        .is_err());
1839    }
1840
1841    /// A transcript (#694): the `[time: …]` prefix is never a phone number
1842    /// and the cue's words are counted once; a spoken phone number is
1843    /// redacted in both `cue` and the paragraph, with one pseudonym.
1844    #[test]
1845    fn transcript_timings_are_not_phones_and_cues_count_once() {
1846        fn track(start: f64, end: f64, cue: &str) -> Node {
1847            Node::Track {
1848                track: crate::tree::TreeTrack {
1849                    start_time: start,
1850                    end_time: end,
1851                    identifier: None,
1852                    voice: None,
1853                },
1854                cue: cue.to_string(),
1855                inner: Box::new(Node::Paragraph {
1856                    text: format!("[time: {start}-{end}] {cue}"),
1857                }),
1858            }
1859        }
1860        let opts = RedactionOptions {
1861            replacement: Replacement::Pseudonym,
1862            ..Default::default()
1863        };
1864        let d = detector(&opts);
1865        let mut doc = DoclingDocument::new("t");
1866        doc.nodes = vec![
1867            track(1234.5, 1240.8, "Shakespeare on scenery by Oscar Wilde."),
1868            track(
1869                1240.8,
1870                1251.25,
1871                "Call me on 555-010-0199 or 555-010-0199 again.",
1872            ),
1873        ];
1874        let report = doc.redact(&opts, &d);
1875        assert_eq!(report.counts.get("PHONE"), Some(&2));
1876        assert_eq!(report.total, 2);
1877        let texts: Vec<(String, String)> = doc
1878            .nodes
1879            .iter()
1880            .map(|n| match n {
1881                Node::Track { cue, inner, .. } => match inner.as_ref() {
1882                    Node::Paragraph { text, .. } => (cue.clone(), text.clone()),
1883                    _ => unreachable!(),
1884                },
1885                _ => unreachable!(),
1886            })
1887            .collect();
1888        assert_eq!(
1889            texts[0],
1890            (
1891                "Shakespeare on scenery by Oscar Wilde.".to_string(),
1892                "[time: 1234.5-1240.8] Shakespeare on scenery by Oscar Wilde.".to_string()
1893            )
1894        );
1895        assert_eq!(
1896            texts[1],
1897            (
1898                "Call me on [PHONE_1] or [PHONE_1] again.".to_string(),
1899                "[time: 1240.8-1251.25] Call me on [PHONE_1] or [PHONE_1] again.".to_string()
1900            )
1901        );
1902        // The timing alone, as the ASR writes long transcripts, is no phone.
1903        assert_eq!(kinds("[time: 1234.5-1240.8]"), vec![]);
1904    }
1905
1906    #[test]
1907    fn overlapping_spans_keep_the_longest() {
1908        // A Luhn-valid 16-digit number is a card, not three phone groups.
1909        assert_eq!(kinds("4111 1111 1111 1111"), vec![PiiKind::CreditCard]);
1910        // Custom pattern covering an e-mail wins by length.
1911        let opts = RedactionOptions {
1912            custom_patterns: vec![CustomPattern::parse("contact=Contact: \\S+").unwrap()],
1913            ..Default::default()
1914        };
1915        let d = detector(&opts);
1916        let mut s = "Contact: a@x.com now".to_string();
1917        Redactor::new(&opts, &d).redact_text(&mut s);
1918        assert_eq!(s, "[CONTACT] now");
1919    }
1920
1921    #[test]
1922    fn inline_runs_are_matched_on_the_joined_text() {
1923        let opts = RedactionOptions::default();
1924        let d = detector(&opts);
1925        let mut runs = vec![
1926            InlineRun {
1927                text: "Mail john".into(),
1928                bold: true,
1929                ..Default::default()
1930            },
1931            InlineRun {
1932                text: "@example".into(),
1933                ..Default::default()
1934            },
1935            InlineRun {
1936                text: ".com today, or 555-".into(),
1937                italic: true,
1938                ..Default::default()
1939            },
1940            InlineRun {
1941                text: "123-4567.".into(),
1942                ..Default::default()
1943            },
1944        ];
1945        Redactor::new(&opts, &d).redact_runs(&mut runs);
1946        let texts: Vec<&str> = runs.iter().map(|r| r.text.as_str()).collect();
1947        assert_eq!(texts, vec!["Mail [EMAIL]", "", " today, or [PHONE]", "."]);
1948        assert!(runs[0].bold && runs[2].italic);
1949    }
1950
1951    #[test]
1952    fn the_document_walk_reaches_every_string() {
1953        use crate::tree::TreeKind;
1954        let mut doc = DoclingDocument::new("file.docx");
1955        doc.push(Node::Heading {
1956            level: 1,
1957            text: "Re: a@x.com".into(),
1958        });
1959        doc.push(Node::Group {
1960            label: "section".into(),
1961            name: Some("b@x.com".into()),
1962            layer: None,
1963            children: vec![Node::Furniture {
1964                layer: crate::ContentLayer::Furniture,
1965                inner: Box::new(Node::Paragraph {
1966                    text: "c@x.com".into(),
1967                }),
1968            }],
1969        });
1970        let mut table = Table {
1971            rows: vec![vec!["d@x.com".into()]],
1972            ..Default::default()
1973        };
1974        table.cell_blocks = Some(vec![vec![vec![Node::Paragraph {
1975            text: "e@x.com".into(),
1976        }]]]);
1977        doc.push(Node::Table(table));
1978        doc.push(Node::Picture {
1979            caption: Some("f@x.com".into()),
1980            caption_href: Some("mailto:f@x.com".into()),
1981            image: Some(crate::PictureImage {
1982                mimetype: "image/png".into(),
1983                width: 1,
1984                height: 1,
1985                data: vec![1],
1986                dpi: 72,
1987                display_size: None,
1988            }),
1989            classification: None,
1990            description: Some(crate::PictureDescription {
1991                text: "ocr j@x.com".into(),
1992                provenance: "ppocr".into(),
1993            }),
1994            caption_parent: Default::default(),
1995            caption_location: None,
1996        });
1997        doc.push(Node::Formula {
1998            latex: "x@y.com".into(),
1999            orig: "g@x.com".into(),
2000            location: None,
2001        });
2002        doc.links.push(("h@x.com".into(), "mailto:h@x.com".into()));
2003        let mut tree = ItemTree::default();
2004        tree.add(
2005            None,
2006            None,
2007            TreeKind::Text {
2008                label: "text".into(),
2009                text: "i@x.com".into(),
2010                orig: Some("i@x.com".into()),
2011                formatting: None,
2012                hyperlink: Some("mailto:i@x.com".into()),
2013                level: None,
2014                list: None,
2015            },
2016        );
2017        doc.tree = Some(tree);
2018        doc.page_images.insert(
2019            1,
2020            crate::PictureImage {
2021                mimetype: "image/png".into(),
2022                width: 1,
2023                height: 1,
2024                data: vec![1],
2025                dpi: 72,
2026                display_size: None,
2027            },
2028        );
2029
2030        let opts = RedactionOptions::default();
2031        let d = detector(&opts);
2032        let report = doc.redact(&opts, &d);
2033        // The tree's three copies are redacted but not counted again.
2034        assert_eq!(report.counts["EMAIL"], 11);
2035        let json = doc.export_to_json();
2036        let md = doc.export_to_markdown();
2037        let dl = doc.export_to_doclang();
2038        for out in [&json, &md, &dl] {
2039            assert!(!out.contains("@x.com"), "{out}");
2040        }
2041        assert_eq!(doc.name, "file.docx", "the file name is not touched");
2042        assert!(doc.page_images.is_empty());
2043        assert!(matches!(&doc.nodes[3], Node::Picture { image: None, .. }));
2044        // The LaTeX stays: it is a rendering, not extracted text.
2045        assert!(matches!(&doc.nodes[4], Node::Formula { latex, .. } if latex == "x@y.com"));
2046
2047        // `Keep` leaves the images.
2048        let mut doc2 = DoclingDocument::new("t");
2049        doc2.page_images.insert(
2050            1,
2051            crate::PictureImage {
2052                mimetype: "image/png".into(),
2053                width: 1,
2054                height: 1,
2055                data: vec![1],
2056                dpi: 72,
2057                display_size: None,
2058            },
2059        );
2060        let opts = RedactionOptions {
2061            images: ImageRedaction::Keep,
2062            ..Default::default()
2063        };
2064        let d = detector(&opts);
2065        doc2.redact(&opts, &d);
2066        assert_eq!(doc2.page_images.len(), 1);
2067    }
2068
2069    #[test]
2070    fn wire_spellings_parse() {
2071        assert_eq!(PiiKind::parse("Credit-Card"), Some(PiiKind::CreditCard));
2072        assert_eq!(PiiKind::parse("ip"), Some(PiiKind::IpAddress));
2073        assert_eq!(PiiKind::parse("nope"), None);
2074        assert_eq!(
2075            Replacement::parse("fixed:XXX"),
2076            Some(Replacement::Fixed("XXX".into()))
2077        );
2078        assert_eq!(
2079            Replacement::parse("Pseudonym"),
2080            Some(Replacement::Pseudonym)
2081        );
2082        assert_eq!(
2083            ImageRedaction::parse("box-out"),
2084            Some(ImageRedaction::BoxOut)
2085        );
2086        let p = CustomPattern::parse("case_id=CASE-\\d+").unwrap();
2087        assert_eq!(
2088            (p.name.as_str(), p.regex.as_str()),
2089            ("case_id", "CASE-\\d+")
2090        );
2091        assert_eq!(CustomPattern::parse("(?i)x=y").unwrap().name, "custom");
2092    }
2093}