pii 0.1.0

PII detection and anonymization with deterministic, capability-aware NLP pipelines.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

//! Dictionary-based recognizer using Aho-Corasick.
//!
//! Dictionary recognizers are useful for names, codewords, or any curated
//! list of terms. Matching is exact (case-insensitive if configured), and
//! the emitted detections include the dictionary source label for audit.

use crate::recognizers::Recognizer;
use crate::types::{Detection, DetectionExplanation, EntityType, NlpArtifacts};
use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind};

/// Emits detections for exact dictionary terms.
#[derive(Clone)]
pub struct DictionaryRecognizer {
    name: String,
    entity: EntityType,
    matcher: AhoCorasick,
    score: f32,
    source: String,
}

impl DictionaryRecognizer {
    /// Creates a dictionary recognizer.
    pub fn new(
        name: impl Into<String>,
        entity: EntityType,
        terms: &[String],
        score: f32,
        source: impl Into<String>,
        case_insensitive: bool,
    ) -> Self {
        let matcher = AhoCorasickBuilder::new()
            .ascii_case_insensitive(case_insensitive)
            .match_kind(MatchKind::LeftmostLongest)
            .build(terms)
            .expect("failed to build dictionary matcher");
        Self {
            name: name.into(),
            entity,
            matcher,
            score,
            source: source.into(),
        }
    }
}

impl Recognizer for DictionaryRecognizer {
    fn name(&self) -> &str {
        &self.name
    }

    fn supported_entities(&self) -> &[EntityType] {
        std::slice::from_ref(&self.entity)
    }

    fn analyze(&self, text: &str, _artifacts: &NlpArtifacts) -> Vec<Detection> {
        self.matcher
            .find_iter(text)
            .map(|m| Detection {
                entity_type: self.entity.clone(),
                start: m.start(),
                end: m.end(),
                score: self.score,
                recognizer: self.name.clone(),
                explanation: DetectionExplanation::Dictionary {
                    source: self.source.clone(),
                },
            })
            .collect()
    }
}