veil_engine/
rules.rs

1//! The rules file (`rules.yaml`) and its compiled form.
2
3use std::fmt;
4use std::marker::PhantomData;
5use std::sync::Arc;
6
7#[cfg(feature = "schema")]
8use schemars::JsonSchema;
9use serde::de::{self, Deserializer, MapAccess, Visitor};
10use serde::{Deserialize, Serialize};
11
12use crate::allowlist::Allowlist;
13use crate::detect::{self, Detector};
14use crate::kind::{kinds, EntityKind, InvalidKind};
15use crate::ner::EntityRecognizer;
16use crate::placeholder::{PlaceholderFormat, DEFAULT_FORMAT};
17
18/// What gets redacted. Serialised as `rules.yaml`.
19#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
20#[cfg_attr(feature = "schema", derive(JsonSchema))]
21#[serde(default, deny_unknown_fields)]
22#[non_exhaustive]
23pub struct RulesFile {
24    /// The format version. Only `1` exists.
25    pub version: u32,
26    /// Which built-in detectors run, and their options.
27    pub detectors: Detectors,
28    /// Fixed terms to redact, such as client or project names.
29    pub dictionary: Vec<DictionaryEntry>,
30    /// Patterns to redact, each with its own kind.
31    pub regex: Vec<RegexRule>,
32    /// Values that are never redacted. An entry with `@` and no local part
33    /// (`@acme.com`) or a bare domain (`acme.com`) allows every email address
34    /// and URL at that domain and its subdomains.
35    pub allowlist: Vec<String>,
36}
37
38impl Default for RulesFile {
39    fn default() -> Self {
40        RulesFile {
41            version: 1,
42            detectors: Detectors::default(),
43            dictionary: Vec::new(),
44            regex: Vec::new(),
45            allowlist: Vec::new(),
46        }
47    }
48}
49
50/// Built-in detectors. Each accepts `true`/`false`, `on`/`off`, or a map with
51/// `enabled` and its options.
52#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
53#[cfg_attr(feature = "schema", derive(JsonSchema))]
54#[serde(default, deny_unknown_fields)]
55#[non_exhaustive]
56pub struct Detectors {
57    /// Email addresses.
58    pub email: Setting<NoOptions>,
59    /// Phone numbers. Needs the `phone` feature.
60    pub phone: Setting<PhoneOptions>,
61    /// Payment card numbers from known issuers that pass the Luhn check.
62    pub card: Setting<NoOptions>,
63    /// IBANs with a valid checksum.
64    pub iban: Setting<NoOptions>,
65    /// API keys and tokens with a recognisable prefix or shape.
66    pub secret: Setting<NoOptions>,
67    /// IPv4 and IPv6 addresses.
68    pub ip: Setting<IpOptions>,
69    /// URLs.
70    pub url: Setting<NoOptions>,
71    /// National ID numbers of the configured countries.
72    pub natid: Setting<NatIdOptions>,
73    /// Names of people, organisations and places, found by a local model
74    /// that is downloaded on first use.
75    pub ner: Setting<NerOptions>,
76}
77
78impl Default for Detectors {
79    fn default() -> Self {
80        Detectors {
81            email: Setting::on(),
82            // On when it's built in, so the default rules always compile.
83            phone: Setting { enabled: cfg!(feature = "phone"), options: PhoneOptions::default() },
84            card: Setting::on(),
85            iban: Setting::on(),
86            secret: Setting::on(),
87            ip: Setting::on(),
88            // Off by default: the model often needs links to help.
89            url: Setting::off(),
90            natid: Setting::on(),
91            // Off by default: it needs a download and makes more mistakes
92            // than the pattern detectors.
93            ner: Setting::off(),
94        }
95    }
96}
97
98/// One detector: on or off, and its options. In YAML it can be just `true`,
99/// `false`, `on` or `off`.
100#[derive(Clone, Debug, PartialEq, Serialize)]
101#[cfg_attr(feature = "schema", derive(JsonSchema))]
102#[cfg_attr(feature = "schema", schemars(rename = "Setting_{T}"))]
103pub struct Setting<T> {
104    /// Whether the detector runs.
105    pub enabled: bool,
106    /// The detector's options.
107    #[serde(flatten)]
108    pub options: T,
109}
110
111impl<T: Default> Setting<T> {
112    /// On, with default options.
113    pub fn on() -> Self {
114        Setting { enabled: true, options: T::default() }
115    }
116    /// Off, with default options.
117    pub fn off() -> Self {
118        Setting { enabled: false, options: T::default() }
119    }
120}
121
122impl<T: Default> Default for Setting<T> {
123    fn default() -> Self {
124        Self::on()
125    }
126}
127
128impl<'de, T: Deserialize<'de> + Default> Deserialize<'de> for Setting<T> {
129    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
130        struct V<T>(PhantomData<T>);
131        impl<'de, T: Deserialize<'de> + Default> Visitor<'de> for V<T> {
132            type Value = Setting<T>;
133            fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
134                f.write_str("true, false, \"on\", \"off\" or a map with `enabled` and options")
135            }
136            fn visit_bool<E: de::Error>(self, v: bool) -> Result<Self::Value, E> {
137                Ok(Setting { enabled: v, options: T::default() })
138            }
139            fn visit_str<E: de::Error>(self, v: &str) -> Result<Self::Value, E> {
140                match v {
141                    "on" | "true" | "yes" => self.visit_bool(true),
142                    "off" | "false" | "no" => self.visit_bool(false),
143                    _ => Err(E::invalid_value(de::Unexpected::Str(v), &self)),
144                }
145            }
146            fn visit_map<A: MapAccess<'de>>(self, map: A) -> Result<Self::Value, A::Error> {
147                #[derive(Deserialize)]
148                struct Full<T> {
149                    #[serde(default = "yes")]
150                    enabled: bool,
151                    #[serde(flatten)]
152                    options: T,
153                }
154                fn yes() -> bool {
155                    true
156                }
157                let full = Full::<T>::deserialize(de::value::MapAccessDeserializer::new(map))?;
158                Ok(Setting { enabled: full.enabled, options: full.options })
159            }
160        }
161        d.deserialize_any(V(PhantomData))
162    }
163}
164
165/// For detectors without options.
166#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
167#[cfg_attr(feature = "schema", derive(JsonSchema))]
168#[non_exhaustive]
169pub struct NoOptions {}
170
171/// Options for phone number detection.
172#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
173#[cfg_attr(feature = "schema", derive(JsonSchema))]
174#[serde(default)]
175#[non_exhaustive]
176pub struct PhoneOptions {
177    /// Countries whose national formats are recognised (ISO 3166 codes).
178    /// Numbers written with a leading `+` or `00` are recognised for every country.
179    pub regions: Vec<String>,
180}
181
182impl Default for PhoneOptions {
183    fn default() -> Self {
184        PhoneOptions { regions: vec!["US".into()] }
185    }
186}
187
188/// Options for IP address detection.
189#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
190#[cfg_attr(feature = "schema", derive(JsonSchema))]
191#[serde(default)]
192#[non_exhaustive]
193pub struct IpOptions {
194    /// Also redact private-range addresses such as 192.168.x.x. Loopback and
195    /// unspecified addresses are never redacted.
196    pub include_private: bool,
197}
198
199impl Default for IpOptions {
200    fn default() -> Self {
201        IpOptions { include_private: true }
202    }
203}
204
205/// Options for national ID detection.
206#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
207#[cfg_attr(feature = "schema", derive(JsonSchema))]
208#[serde(default)]
209#[non_exhaustive]
210pub struct NatIdOptions {
211    /// Supported: US (Social Security number, with dashes), GB (National
212    /// Insurance number), DE (tax ID), FR (NIR), ES (DNI and NIE), IT (codice
213    /// fiscale), NL (BSN), PL (PESEL) and RO (CNP).
214    pub locales: Vec<String>,
215}
216
217impl Default for NatIdOptions {
218    fn default() -> Self {
219        NatIdOptions { locales: ["US", "GB", "DE"].map(String::from).to_vec() }
220    }
221}
222
223/// Options for name detection.
224#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
225#[cfg_attr(feature = "schema", derive(JsonSchema))]
226#[serde(default)]
227#[non_exhaustive]
228pub struct NerOptions {
229    /// Minimum confidence, from 0 to 1. Higher means fewer false alarms and
230    /// more missed names.
231    pub threshold: f32,
232    /// Which of PERSON, ORG and LOCATION to redact.
233    pub kinds: Vec<String>,
234}
235
236impl Default for NerOptions {
237    fn default() -> Self {
238        NerOptions { threshold: 0.8, kinds: [kinds::PERSON, kinds::ORG, kinds::LOCATION].map(String::from).to_vec() }
239    }
240}
241
242/// A fixed term to redact.
243#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
244#[cfg_attr(feature = "schema", derive(JsonSchema))]
245#[serde(deny_unknown_fields)]
246pub struct DictionaryEntry {
247    /// The text to find. It matches only as a whole word.
248    pub term: String,
249    /// The placeholder kind, such as `CLIENT`. Written `type` in YAML.
250    #[serde(rename = "type")]
251    pub kind: String,
252    /// Match the term's case exactly. Off by default.
253    #[serde(default)]
254    pub match_case: bool,
255}
256
257/// A pattern to redact.
258#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
259#[cfg_attr(feature = "schema", derive(JsonSchema))]
260#[serde(deny_unknown_fields)]
261pub struct RegexRule {
262    /// A name for the rule, reported as [`Source::Regex`](crate::Source::Regex).
263    pub name: String,
264    /// The placeholder kind. Written `type` in YAML.
265    #[serde(rename = "type")]
266    pub kind: String,
267    /// The pattern, in the [`regex`](https://docs.rs/regex) crate's syntax.
268    pub pattern: String,
269}
270
271/// How rules run, besides the rules themselves: the placeholder template
272/// and restoration options. Not part of `rules.yaml`.
273#[derive(Clone, Debug, PartialEq, Eq)]
274pub struct Options {
275    /// The placeholder template; see [`PlaceholderFormat`]. Default `[{TYPE}_{N}]`.
276    pub placeholder_format: String,
277    /// Also restore placeholders the model wrote slightly differently, such
278    /// as `[email_1]` or `EMAIL_1`. Off by default.
279    pub fuzzy_restore: bool,
280}
281
282impl Default for Options {
283    fn default() -> Self {
284        Options { placeholder_format: DEFAULT_FORMAT.to_owned(), fuzzy_restore: false }
285    }
286}
287
288/// Why rules could not be compiled.
289#[derive(Debug, thiserror::Error)]
290#[non_exhaustive]
291pub enum RulesError {
292    /// A kind name is invalid.
293    #[error(transparent)]
294    Kind(#[from] InvalidKind),
295    /// A regex rule's pattern doesn't compile.
296    #[error("regex rule {name:?}: {error}")]
297    Regex {
298        /// The rule's name.
299        name: String,
300        /// What is wrong with the pattern.
301        error: String,
302    },
303    /// A dictionary entry's term is empty. Holds the entry's index.
304    #[error("dictionary entry {0}: the term is empty")]
305    EmptyTerm(usize),
306    /// A phone region code isn't known.
307    #[error("unknown phone region {0:?}")]
308    Region(String),
309    /// A national ID locale isn't supported.
310    #[error("unsupported national ID locale {0:?} (supported: US, GB, DE, FR, ES, IT, NL, PL, RO)")]
311    Locale(String),
312    /// The placeholder template is invalid.
313    #[error("placeholder format: {0}")]
314    Placeholder(String),
315    /// The rules' version isn't supported.
316    #[error("unsupported rules file version {0}")]
317    Version(u32),
318    /// The name detection options are invalid.
319    #[error("name detection: {0}")]
320    Ner(String),
321    /// The rules turn on a detector this build leaves out, such as `phone`
322    /// without the `phone` feature.
323    #[error("the {0} detector is not in this build (cargo feature `{0}`)")]
324    NotBuiltIn(&'static str),
325}
326
327/// Rules ready to run. Built once per change of the rules file or format.
328pub struct CompiledRules {
329    /// Unique per compile, so cached scans never outlive the rules they used.
330    pub(crate) id: u64,
331    pub(crate) detectors: Vec<Box<dyn Detector>>,
332    pub(crate) ner: Option<NerStage>,
333    pub(crate) allowlist: Allowlist,
334    pub(crate) placeholder: PlaceholderFormat,
335    pub(crate) fuzzy_restore: bool,
336}
337
338/// Name detection as configured. `recognizer` is `None` when it is turned on
339/// but no model is loaded: scanning then fails, so nothing is sent unredacted.
340pub(crate) struct NerStage {
341    pub recognizer: Option<Arc<dyn EntityRecognizer>>,
342    pub threshold: f32,
343    pub kinds: Vec<EntityKind>,
344}
345
346impl CompiledRules {
347    /// Compiles `rules` with `options`. `recognizer` is the name-detection
348    /// model; if the rules turn name detection on and it is `None`, scanning
349    /// fails until rules are compiled with one, so names are never sent.
350    pub fn compile(
351        rules: &RulesFile,
352        options: &Options,
353        recognizer: Option<Arc<dyn EntityRecognizer>>,
354    ) -> Result<Self, RulesError> {
355        if rules.version != 1 {
356            return Err(RulesError::Version(rules.version));
357        }
358        let placeholder = PlaceholderFormat::parse(&options.placeholder_format).map_err(RulesError::Placeholder)?;
359        let ner = &rules.detectors.ner;
360        let ner = if ner.enabled {
361            let o = &ner.options;
362            if !(0.0..=1.0).contains(&o.threshold) {
363                return Err(RulesError::Ner(format!("the threshold must be between 0 and 1, not {}", o.threshold)));
364            }
365            let allowed = [kinds::PERSON, kinds::ORG, kinds::LOCATION];
366            let kinds = o
367                .kinds
368                .iter()
369                .map(|k| match allowed.contains(&k.as_str()) {
370                    true => Ok(kind(k)?),
371                    false => Err(RulesError::Ner(format!("unknown kind {k:?} (use PERSON, ORG or LOCATION)"))),
372                })
373                .collect::<Result<_, _>>()?;
374            Some(NerStage { recognizer, threshold: o.threshold, kinds })
375        } else {
376            None
377        };
378        static NEXT_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
379        Ok(CompiledRules {
380            id: NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
381            detectors: detect::build(rules)?,
382            ner,
383            allowlist: Allowlist::new(&rules.allowlist),
384            placeholder,
385            fuzzy_restore: options.fuzzy_restore,
386        })
387    }
388
389    /// True if the rules turn name detection on but no model was supplied.
390    pub fn ner_missing(&self) -> bool {
391        self.ner.as_ref().is_some_and(|n| n.recognizer.is_none())
392    }
393
394    /// True if the rules turn name detection on.
395    pub fn ner_enabled(&self) -> bool {
396        self.ner.is_some()
397    }
398
399    /// The placeholder format these rules use.
400    pub fn placeholder(&self) -> &PlaceholderFormat {
401        &self.placeholder
402    }
403}
404
405impl Default for CompiledRules {
406    fn default() -> Self {
407        CompiledRules::compile(&RulesFile::default(), &Options::default(), None).expect("default rules compile")
408    }
409}
410
411pub(crate) fn kind(name: &str) -> Result<EntityKind, InvalidKind> {
412    EntityKind::new(name)
413}
414
415#[cfg(test)]
416mod tests {
417    use super::*;
418
419    fn format(template: &str) -> Options {
420        Options { placeholder_format: template.into(), ..Options::default() }
421    }
422
423    #[test]
424    fn parses_shorthand_and_full_settings() {
425        let yaml = r#"
426version: 1
427detectors:
428  email: on
429  url: off
430  card: false
431  phone: { enabled: true, regions: [GB] }
432  ip: { include_private: false }
433dictionary:
434  - { term: "Project Halcyon", type: CODENAME, match_case: true }
435regex:
436  - { name: tickets, type: TICKET, pattern: '\bTCK-\d{6}\b' }
437allowlist: [support@acme.com]
438"#;
439        let r: RulesFile = serde_yaml_ng::from_str(yaml).unwrap();
440        assert!(r.detectors.email.enabled);
441        assert!(!r.detectors.url.enabled && !r.detectors.card.enabled);
442        assert_eq!(r.detectors.phone.options.regions, ["GB"]);
443        assert!(r.detectors.ip.enabled && !r.detectors.ip.options.include_private);
444        assert!(r.detectors.natid.enabled);
445        assert_eq!(r.dictionary[0].kind, "CODENAME");
446        CompiledRules::compile(&r, &Options::default(), None).unwrap();
447    }
448
449    #[test]
450    fn reports_bad_rules() {
451        let mut r = RulesFile::default();
452        r.regex.push(RegexRule { name: "x".into(), kind: "X".into(), pattern: "(".into() });
453        assert!(matches!(CompiledRules::compile(&r, &Options::default(), None), Err(RulesError::Regex { .. })));
454        let mut r = RulesFile::default();
455        r.dictionary.push(DictionaryEntry { term: "a".into(), kind: "lower".into(), match_case: false });
456        assert!(matches!(CompiledRules::compile(&r, &Options::default(), None), Err(RulesError::Kind(_))));
457        let mut r = RulesFile::default();
458        r.detectors.phone.options.regions = vec!["XX".into()];
459        assert!(matches!(CompiledRules::compile(&r, &Options::default(), None), Err(RulesError::Region(_))));
460        let mut r = RulesFile::default();
461        r.detectors.natid.options.locales = vec!["XX".into()];
462        assert!(matches!(CompiledRules::compile(&r, &Options::default(), None), Err(RulesError::Locale(_))));
463        assert!(matches!(
464            CompiledRules::compile(&RulesFile::default(), &format("{TYPE}"), None),
465            Err(RulesError::Placeholder(_))
466        ));
467    }
468
469    #[test]
470    fn phone_detection_is_on_by_default_only_when_built_in() {
471        assert_eq!(RulesFile::default().detectors.phone.enabled, cfg!(feature = "phone"));
472        CompiledRules::default();
473        let mut r = RulesFile::default();
474        r.detectors.phone.enabled = true;
475        let compiled = CompiledRules::compile(&r, &Options::default(), None);
476        if cfg!(feature = "phone") {
477            assert!(compiled.is_ok());
478        } else {
479            assert!(matches!(compiled, Err(RulesError::NotBuiltIn("phone"))));
480        }
481    }
482
483    #[test]
484    fn rejects_unknown_fields() {
485        assert!(serde_yaml_ng::from_str::<RulesFile>("detectors: { emial: on }").is_err());
486    }
487}
veil_engine/rules.rs

veil_engine/
rules.rs
