rust_censure/lang/
common.rs

1use fancy_regex::Regex;
2use once_cell::sync::Lazy;
3use std::collections::HashMap;
4
5/// The censor 'beep' words
6pub static BEEP: &str = "***";
7pub static BEEP_HTML: &str = "<span class=\"censored\">***</span>";
8
9/// Utility: apply TRANS_TAB
10pub fn translate_similar_chars(s: &str, trans_table: &HashMap<char, char>) -> String {
11    s.chars().map(|c| trans_table.get(&c).copied().unwrap_or(c)).collect()
12}
13
14/// (order matters)
15pub static NORMALIZATION_PATTERNS: Lazy<Vec<(Regex, &'static str)>> = Lazy::new(|| {
16    vec![
17        (Regex::new(r"&[Ee][Uu][Mm][Ll];").unwrap(), "е"),
18        (Regex::new(r"&[Uu][Uu][Mm][Ll];").unwrap(), "и"),
19        (Regex::new(r"&[Aa][Uu][Mm][Ll];").unwrap(), "а"),
20        (Regex::new(r"&[Oo][Uu][Mm][Ll];").unwrap(), "о"),
21        (Regex::new(r"&[Yy][Uu][Mm][Ll];").unwrap(), "у"),
22
23        (Regex::new(r"&#203;").unwrap(), "е"),
24
25        (Regex::new(r"&[Cc][Ee][Nn][Tt];").unwrap(), "с"),
26        (Regex::new(r"&#162;").unwrap(), "с"),
27
28        (Regex::new(r"\|\\\|").unwrap(), "и"), // И
29        (Regex::new(r"/\\").unwrap(), "л"),    // Л
30
31        (Regex::new(r"><").unwrap(), "х"),
32        (Regex::new(r"\)\(").unwrap(), "х"),
33        (Regex::new(r"}{").unwrap(), "х"),
34
35        (Regex::new(r">\|<").unwrap(), "ж"),
36        (Regex::new(r"}\|{").unwrap(), "ж"),
37
38        (Regex::new(r"`/").unwrap(), "y"),
39        (Regex::new(r"\-/").unwrap(), "y"),
40        (Regex::new(r"`\-/").unwrap(), "y"),
41
42        (Regex::new(r"b\|").unwrap(), "ы"),
43        (Regex::new(r"bI").unwrap(), "ы"),
44        (Regex::new(r"bl").unwrap(), "ы"),
45
46        (Regex::new(r"&#120;").unwrap(), "х"),
47        (Regex::new(r"&#121;").unwrap(), "у"),
48        (Regex::new(r"3[\.,]14[\d]*").unwrap(), "пи"),
49        (Regex::new(r"2[\.,]72[\d]*").unwrap(), "е"),
50    ]
51});
52
53pub static PAT_SPACE: Lazy<Regex>       = Lazy::new(|| Regex::new(r"\s+").unwrap());
54
55pub static PAT_PUNCT1: Lazy<Regex>      = Lazy::new(|| Regex::new(r#"["\-+;.,*?()]+"#).unwrap()); // zero-widths
56pub static PAT_PUNCT2: Lazy<Regex>      = Lazy::new(|| Regex::new(r#"[!:_]+"#).unwrap());
57pub static PAT_PUNCT3: Lazy<Regex>      = Lazy::new(|| Regex::new(r#"["\-+;.,*?()!:_]+"#).unwrap());
58
59pub static PAT_HTML_TAG: Lazy<Regex>    = Lazy::new(|| Regex::new(r#"(<.*?>)|(&[\w]{2,6};)|(<![-]+)|([-]+>)"#).unwrap());
60// pub static PAT_HTML_TAG_OR_SPACER: Lazy<Regex>    = Lazy::new(|| Regex::new(r#"(?P<tag><.*?>)|(?P<spacer>[\s]+)"#).unwrap());
61// pub static PAT_HTML_CSS: Lazy<Regex>  = Lazy::new(|| Regex::new(r"[\w\s}{.#;:\-+]").unwrap());
62pub static PAT_HTML_SPACE: Lazy<Regex>  = Lazy::new(|| Regex::new(r"(?i)&nbsp;").unwrap());
63pub static PAT_PREP: Lazy<Regex>               = Lazy::new(|| Regex::new(r"(а[х]?)|(в)|([вмт]ы)|(д[ао])|(же)|(за)").unwrap());
rust_censure/lang/common.rs

rust_censure/lang/
common.rs
