better_profanity/
data.rs

1//! The data of better_profanity 0.7.0: its word list, `CHARS_MAPPING` and `ALLOWED_CHARACTERS`.
2
3use std::sync::OnceLock;
4
5use crate::CharSet;
6
7/// `profanity_wordlist.txt`, the default word list (916 words and phrases, one per line).
8pub const WORDLIST: &str = include_str!("../data/profanity_wordlist.txt");
9
10/// `alphabetic_unicode.json`: the letters and marks that, with ASCII letters, digits and
11/// `@$*"'`, make up `ALLOWED_CHARACTERS`.
12const ALPHABETIC_UNICODE: &str = include_str!("../data/alphabetic_unicode.json");
13
14/// The default `CHARS_MAPPING`: the characters each letter of a word may be written as.
15pub const CHARS_MAPPING: &[(char, &[char])] = &[
16    ('a', &['a', '@', '*', '4']),
17    ('i', &['i', '*', 'l', '1']),
18    ('o', &['o', '*', '0', '@']),
19    ('u', &['u', '*', 'v']),
20    ('v', &['v', '*', 'u']),
21    ('l', &['l', '1']),
22    ('e', &['e', '*', '3']),
23    ('s', &['s', '$', '5']),
24    ('t', &['t', '7']),
25];
26
27/// `ALLOWED_CHARACTERS`: characters that belong to words. Every other character separates them.
28pub fn allowed_characters() -> &'static CharSet {
29    static ALLOWED: OnceLock<CharSet> = OnceLock::new();
30    ALLOWED.get_or_init(|| {
31        let ascii = ('a'..='z')
32            .chain('A'..='Z')
33            .chain('0'..='9')
34            .chain("@$*\"'".chars());
35        CharSet::new(
36            ascii.chain(parse_string_array(ALPHABETIC_UNICODE)),
37            is_python_space,
38        )
39    })
40}
41
42/// Python's `str.isspace()`, which is the same on CPython 3.9 to 3.15.
43pub fn is_python_space(c: char) -> bool {
44    matches!(
45        c,
46        '\t'..='\r'
47            | '\x1c'..=' '
48            | '\u{85}'
49            | '\u{a0}'
50            | '\u{1680}'
51            | '\u{2000}'..='\u{200a}'
52            | '\u{2028}'
53            | '\u{2029}'
54            | '\u{202f}'
55            | '\u{205f}'
56            | '\u{3000}'
57    )
58}
59
60/// The non-empty lines of a word list with surrounding whitespace removed, as
61/// better_profanity's `read_wordlist` returns them from a file opened in text mode.
62pub fn read_wordlist(text: &str) -> impl Iterator<Item = &str> {
63    text.split(['\n', '\r'])
64        .map(|line| line.trim_matches(is_python_space))
65        .filter(|line| !line.is_empty())
66}
67
68/// The characters of a JSON array of one-character strings, as `alphabetic_unicode.json` is.
69fn parse_string_array(src: &str) -> Vec<char> {
70    let mut out = Vec::new();
71    let mut chars = src.chars();
72    let mut in_string = false;
73    let mut high: Option<u32> = None;
74    while let Some(c) = chars.next() {
75        if !in_string {
76            in_string = c == '"';
77            continue;
78        }
79        let code = match c {
80            '"' => {
81                in_string = false;
82                continue;
83            }
84            '\\' => match chars.next() {
85                Some('u') => {
86                    let hex: String = chars.by_ref().take(4).collect();
87                    u32::from_str_radix(&hex, 16).expect("valid \\u escape")
88                }
89                Some('n') => '\n' as u32,
90                Some('t') => '\t' as u32,
91                Some('r') => '\r' as u32,
92                Some('b') => 8,
93                Some('f') => 12,
94                Some(other) => other as u32,
95                None => break,
96            },
97            other => other as u32,
98        };
99        match (high.take(), code) {
100            (None, 0xD800..=0xDBFF) => high = Some(code),
101            (Some(h), 0xDC00..=0xDFFF) => {
102                out.extend(char::from_u32(
103                    0x10000 + ((h - 0xD800) << 10) + (code - 0xDC00),
104                ));
105            }
106            (_, code) => out.extend(char::from_u32(code)),
107        }
108    }
109    out
110}
111
112#[cfg(test)]
113mod tests {
114    use super::*;
115
116    #[test]
117    fn allowed_characters_match_python() {
118        let unicode = parse_string_array(ALPHABETIC_UNICODE);
119        assert_eq!(unicode.len(), 5849);
120        assert_eq!(unicode[26], '\u{b5}');
121        assert_eq!(unicode.iter().max(), Some(&'\u{e01ef}'));
122        let allowed = allowed_characters();
123        assert_eq!(allowed.len(), 5849 + 10 + 5);
124        assert!(allowed.contains('@') && allowed.contains('é') && allowed.contains('\u{e01ef}'));
125        assert!(!allowed.contains(' ') && !allowed.contains('-') && !allowed.contains('_'));
126    }
127
128    #[test]
129    fn wordlist_has_916_words() {
130        assert_eq!(read_wordlist(WORDLIST).count(), 916);
131        let words: Vec<&str> = read_wordlist(" a \r\n\r\nb\u{85}\rc\n\n").collect();
132        assert_eq!(words, ["a", "b", "c"]);
133    }
134}
better_profanity/data.rs

better_profanity/
data.rs
