better_profanity/
lib.rs

1//! A port of the Python word filter [better_profanity] 0.7.0 that censors exactly the same
2//! words, including their leetspeak spellings and words split by separators
3//! (`f.u.c.k`, `H4nd j0b`), and leaves every other character of the text as it was.
4//!
5//! ```
6//! use better_profanity::Profanity;
7//!
8//! let profanity = Profanity::new();
9//! assert_eq!(
10//!     profanity.censor("That wh0re gave m3 a very good H@nD j0b.", "*"),
11//!     "That **** gave m3 a very good ****."
12//! );
13//! assert!(!profanity.contains_profanity("Hello there"));
14//! ```
15//!
16//! [`Profanity`] follows better_profanity's `Profanity` class. The lower-level
17//! [`hide_swear_words`], [`WordSet`] and [`CharSet`] reproduce its algorithm with any word
18//! set, separator set and lowercasing; the Python package `better-profanity-rs` uses them to
19//! match the interpreter it runs on.
20//!
21//! [better_profanity]: https://github.com/snguyenthanh/better_profanity
22
23mod censor;
24mod charset;
25pub mod data;
26mod words;
27
28use std::path::Path;
29use std::sync::atomic::{AtomicBool, Ordering};
30use std::sync::OnceLock;
31
32use rustc_hash::{FxHashMap, FxHashSet};
33
34pub use censor::{hide_swear_words, Config, Lowercase, StdLowercase};
35pub use charset::CharSet;
36pub use words::{SetId, Walk, WordSet, WordSetBuilder};
37
38/// better_profanity's `Profanity`: a list of words to censor and how to match them.
39///
40/// Words are lowercased with Rust's Unicode tables, which can differ from those of a given
41/// Python version for characters added to Unicode recently.
42#[derive(Debug)]
43pub struct Profanity {
44    words: Vec<Vec<Box<[char]>>>,
45    set: WordSet,
46    chars_mapping: FxHashMap<char, Box<[char]>>,
47    custom_mapping: bool,
48    max_number_combinations: usize,
49    /// Set when `censor` found no words. better_profanity loads the default list at that point,
50    /// so getters report it and the next change starts from it.
51    defaults_loaded: AtomicBool,
52}
53
54impl Clone for Profanity {
55    fn clone(&self) -> Self {
56        Profanity {
57            words: self.words.clone(),
58            set: self.set.clone(),
59            chars_mapping: self.chars_mapping.clone(),
60            custom_mapping: self.custom_mapping,
61            max_number_combinations: self.max_number_combinations,
62            defaults_loaded: AtomicBool::new(self.defaults_pending()),
63        }
64    }
65}
66
67impl Default for Profanity {
68    fn default() -> Self {
69        Self::new()
70    }
71}
72
73impl Profanity {
74    /// The default word list, as `Profanity()`.
75    pub fn new() -> Self {
76        let mut p = Self::empty();
77        p.load_default_censor_words::<&str>(&[]);
78        p
79    }
80
81    /// `Profanity(words)`; an empty list loads the default words.
82    pub fn with_words<I, S>(words: I) -> Self
83    where
84        I: IntoIterator<Item = S>,
85        S: AsRef<str>,
86    {
87        let mut p = Self::empty();
88        p.load_censor_words::<_, _, &str>(words, &[]);
89        p
90    }
91
92    fn empty() -> Self {
93        Profanity {
94            words: Vec::new(),
95            set: WordSet::builder().build(),
96            chars_mapping: data::CHARS_MAPPING
97                .iter()
98                .map(|&(c, variants)| (c, variants.into()))
99                .collect(),
100            custom_mapping: false,
101            max_number_combinations: 1,
102            defaults_loaded: AtomicBool::new(false),
103        }
104    }
105
106    /// `load_censor_words(custom_words, whitelist_words)`: replaces the words with `words`,
107    /// or with the default list if `words` is empty, leaving out those in `whitelist`.
108    pub fn load_censor_words<I, S, W>(&mut self, words: I, whitelist: &[W])
109    where
110        I: IntoIterator<Item = S>,
111        S: AsRef<str>,
112        W: AsRef<str>,
113    {
114        self.settle();
115        let words: Vec<S> = words.into_iter().collect();
116        if words.is_empty() {
117            self.load_default_censor_words(whitelist);
118        } else {
119            self.populate(words.iter().map(AsRef::as_ref), whitelist);
120        }
121    }
122
123    /// `load_censor_words(whitelist_words=whitelist)`: the default word list.
124    pub fn load_default_censor_words<W: AsRef<str>>(&mut self, whitelist: &[W]) {
125        self.settle();
126        self.populate(data::read_wordlist(data::WORDLIST), whitelist);
127    }
128
129    /// `load_censor_words_from_file(path, whitelist_words)`: one word per line of a UTF-8
130    /// file. Unlike [`load_censor_words`](Self::load_censor_words), an empty file leaves no
131    /// words.
132    pub fn load_censor_words_from_file<W: AsRef<str>>(
133        &mut self,
134        path: impl AsRef<Path>,
135        whitelist: &[W],
136    ) -> std::io::Result<()> {
137        self.settle();
138        let text = std::fs::read_to_string(path)?;
139        self.populate(data::read_wordlist(&text), whitelist);
140        Ok(())
141    }
142
143    /// `add_censor_words(words)`: adds words as they are, without lowercasing them or
144    /// updating [`max_number_combinations`](Self::max_number_combinations).
145    pub fn add_censor_words<I, S>(&mut self, words: I)
146    where
147        I: IntoIterator<Item = S>,
148        S: AsRef<str>,
149    {
150        self.settle();
151        for word in words {
152            let entry = self.varying(word.as_ref());
153            self.words.push(entry);
154        }
155        self.rebuild();
156    }
157
158    /// Replaces `CHARS_MAPPING`, which applies to words loaded or added afterwards.
159    pub fn set_chars_mapping<I, V>(&mut self, mapping: I)
160    where
161        I: IntoIterator<Item = (char, V)>,
162        V: IntoIterator<Item = char>,
163    {
164        self.settle();
165        self.chars_mapping = mapping
166            .into_iter()
167            .map(|(c, variants)| (c, variants.into_iter().collect()))
168            .collect();
169        self.custom_mapping = true;
170    }
171
172    /// `MAX_NUMBER_COMBINATIONS`: how many following words are joined to the current one
173    /// when looking for phrases. Loading words raises it to the largest number of separator
174    /// characters in a word; it never decreases on its own.
175    pub fn max_number_combinations(&self) -> usize {
176        if self.defaults_pending() {
177            return self
178                .max_number_combinations
179                .max(default_profanity().max_number_combinations);
180        }
181        self.max_number_combinations
182    }
183
184    pub fn set_max_number_combinations(&mut self, n: usize) {
185        self.settle();
186        self.max_number_combinations = n;
187    }
188
189    /// The number of words, as `len(CENSOR_WORDSET)`.
190    pub fn len(&self) -> usize {
191        if self.defaults_pending() {
192            return default_profanity().len();
193        }
194        self.words.len()
195    }
196
197    pub fn is_empty(&self) -> bool {
198        self.len() == 0
199    }
200
201    fn defaults_pending(&self) -> bool {
202        self.defaults_loaded.load(Ordering::Relaxed) && self.words.is_empty()
203    }
204
205    /// Loads the default list that `censor` used on an empty list, as better_profanity did.
206    fn settle(&mut self) {
207        if std::mem::take(self.defaults_loaded.get_mut()) && self.words.is_empty() {
208            self.populate(data::read_wordlist(data::WORDLIST), &[] as &[&str]);
209        }
210    }
211
212    /// `censor(text, censor_char)`: replaces each swear word with `censor_char` four times.
213    /// With no words loaded, better_profanity loads the default list and keeps it; so does this
214    /// method, in effect: later calls and changes see the default list.
215    pub fn censor(&self, text: &str, censor_char: &str) -> String {
216        if self.words.is_empty() {
217            self.defaults_loaded.store(true, Ordering::Relaxed);
218            let mut defaults;
219            let source = if self.custom_mapping {
220                defaults = self.clone();
221                defaults.settle();
222                &defaults
223            } else {
224                default_profanity()
225            };
226            let max = self
227                .max_number_combinations
228                .max(source.max_number_combinations);
229            return source.censor_with(text, censor_char, max);
230        }
231        self.censor_with(text, censor_char, self.max_number_combinations)
232    }
233
234    /// `contains_profanity(text)`: whether censoring changes the text.
235    pub fn contains_profanity(&self, text: &str) -> bool {
236        self.censor(text, "*") != text
237    }
238
239    fn censor_with(&self, text: &str, censor_char: &str, max: usize) -> String {
240        let allowed = data::allowed_characters();
241        let config = Config {
242            words: &self.set,
243            allowed,
244            helper_allowed: allowed,
245            max_number_combinations: max.max(1),
246        };
247        hide_swear_words(text, censor_char, &config, &mut StdLowercase)
248            .unwrap_or_else(|| text.to_owned())
249    }
250
251    fn populate<'w, W: AsRef<str>>(
252        &mut self,
253        words: impl Iterator<Item = &'w str>,
254        whitelist: &[W],
255    ) {
256        let whitelist: FxHashSet<String> = whitelist
257            .iter()
258            .map(|w| w.as_ref().to_lowercase())
259            .collect();
260        let unique: FxHashSet<&str> = words.collect();
261        let allowed = data::allowed_characters();
262        let mut entries = Vec::with_capacity(unique.len());
263        for word in unique {
264            let word = word.to_lowercase();
265            if whitelist.contains(&word) {
266                continue;
267            }
268            let separators = word.chars().filter(|&c| !allowed.contains(c)).count();
269            self.max_number_combinations = self.max_number_combinations.max(separators);
270            entries.push(self.varying(&word));
271        }
272        self.words = entries;
273        self.rebuild();
274    }
275
276    /// `VaryingString(word, char_map=CHARS_MAPPING)`.
277    fn varying(&self, word: &str) -> Vec<Box<[char]>> {
278        word.chars()
279            .map(|c| {
280                self.chars_mapping
281                    .get(&c)
282                    .cloned()
283                    .unwrap_or_else(|| Box::new([c]))
284            })
285            .collect()
286    }
287
288    fn rebuild(&mut self) {
289        let mut builder = WordSet::builder();
290        for word in &self.words {
291            builder.insert(word.iter().map(|p| p.iter().copied()));
292        }
293        self.set = builder.build();
294    }
295}
296
297fn default_profanity() -> &'static Profanity {
298    static DEFAULT: OnceLock<Profanity> = OnceLock::new();
299    DEFAULT.get_or_init(Profanity::new)
300}
301
302#[cfg(test)]
303mod tests {
304    use super::*;
305
306    fn censor(text: &str) -> String {
307        static P: OnceLock<Profanity> = OnceLock::new();
308        P.get_or_init(Profanity::new).censor(text, "*")
309    }
310
311    #[test]
312    fn upstream_examples() {
313        assert_eq!(
314            censor("That wh0re gave m3 a very good H4nd j0b, dude. You gotta check."),
315            "That **** gave m3 a very good ****, dude. You gotta check."
316        );
317        assert_eq!(
318            censor("Those 2 girls 1 cup. You gotta check. "),
319            "Those ****. You gotta check. "
320        );
321        assert_eq!(censor("2 girls 1 cup"), "****");
322        assert_eq!(censor("fuck 2 girls 1 cup"), "**** ****");
323        assert_eq!(
324            censor("  wh0re gave m3 a very good H@nD j0b."),
325            "  **** gave m3 a very good ****."
326        );
327        assert_eq!(
328            censor("...pen1s...hello_cat_vagina,,,,qew"),
329            "...****...hello_cat_****,,,,qew"
330        );
331        assert_eq!(censor(""), "");
332        assert_eq!(censor("Hi there"), "Hi there");
333        assert!(Profanity::new().contains_profanity("he is a m0th3rf*cker"));
334    }
335
336    #[test]
337    fn custom_words_and_whitelist() {
338        let mut p = Profanity::with_words(["happy", "jolly", "merry"]);
339        assert!(!p.contains_profanity("Fuck you!"));
340        assert!(p.contains_profanity("Have a merry day! :)"));
341        p.load_default_censor_words(&["boobs"]);
342        assert_eq!(p.censor("I have boobs", "*"), "I have boobs");
343        p.add_censor_words(["supremacia ariana"]);
344        assert_eq!(p.censor("supremacia ariana", "*"), "****");
345
346        let mut p = Profanity::with_words(["gâu"]);
347        assert_eq!(
348            p.censor("Con chó sủa gâu gâu!", "-"),
349            "Con chó sủa ---- ----!"
350        );
351        p.load_censor_words(["хайль"], &["x"]);
352        assert_eq!(
353            p.censor("соседский мальчик сказал хайль и я опешил.", "*"),
354            "соседский мальчик сказал **** и я опешил."
355        );
356    }
357
358    #[test]
359    fn added_phrase_matches_across_a_space() {
360        let mut custom = Profanity::with_words(["happy", "jolly"]);
361        assert_eq!(custom.max_number_combinations(), 1);
362        custom.add_censor_words(["merry christmas"]);
363        assert!(custom.contains_profanity("Merry Christmas!"));
364        assert_eq!(custom.censor("m3rry christmas", "#"), "####");
365        // With one following word, better_profanity joins only every other word to the next.
366        assert_eq!(custom.censor("a m3rry christmas", "#"), "a m3rry christmas");
367        assert_eq!(custom.censor("a b m3rry christmas", "#"), "a b ####");
368    }
369
370    #[test]
371    fn defaults_loaded_by_censor_are_kept() {
372        let mut p = Profanity::with_words(["happy"]);
373        p.load_censor_words(["happy"], &["happy"]);
374        assert_eq!((p.len(), p.max_number_combinations()), (0, 1));
375        assert_eq!(p.censor("fuck", "*"), "****");
376        assert_eq!((p.len(), p.max_number_combinations()), (916, 5));
377        p.add_censor_words(["jolly"]);
378        assert_eq!(p.censor("fuck jolly", "*"), "**** ****");
379        assert_eq!(p.len(), 917);
380
381        let mut p = Profanity::with_words(["happy"]);
382        p.load_censor_words(["happy"], &["happy"]);
383        p.add_censor_words(["jolly"]);
384        assert_eq!(p.censor("fuck jolly", "*"), "fuck ****");
385    }
386
387    #[test]
388    fn empty_list_falls_back_to_defaults() {
389        let mut p = Profanity::with_words(["only"]);
390        p.load_censor_words(["only"], &["ONLY"]);
391        assert!(p.is_empty());
392        assert_eq!(p.censor("fuck", "*"), "****");
393    }
394}
better_profanity/lib.rs

better_profanity/
lib.rs
