rust-censure 0.2.1

Simple profanity filter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

use crate::lang::common::{PAT_HTML_TAG, PAT_HTML_SPACE, PAT_SPACE};

#[derive(Debug, Clone)]
pub enum TokType { Word, Space, TagOpen, TagClose, TagSelf, Spacer }

#[derive(Debug, Clone)]
pub struct Token {
    pub value: String,
    pub tag: String,     // lowercased name for tags
    pub kind: TokType,
}

impl Token {
    pub fn new_tag(raw: &str) -> Self {
        let mut head = raw[1..raw.len()-1].to_lowercase(); // strip '<', '>'
        let kind = if head.starts_with('/') {
            head.remove(0);
            TokType::TagClose
        } else if raw.ends_with("/>") {
            TokType::TagSelf
        } else {
            TokType::TagOpen
        };

        let tag_name = head.split_whitespace().next().unwrap_or("").to_string();

        // spacer check (mirrors Python PAT_HTML_SPACE test)
        let kind = if matches!(kind, TokType::TagOpen|TokType::TagClose|TokType::TagSelf)
            && PAT_HTML_SPACE.is_match(raw).unwrap_or(false)
        {
            TokType::Spacer
        } else { kind };

        Token { value: raw.to_string(), tag: tag_name, kind }
    }
}

/// Split an HTML line into alternating text and tags, then words/spaces inside text
pub fn tokenize_html(line: &str) -> Vec<Token> {
    let mut tokens = Vec::new();
    let mut start = 0usize;
    for m in PAT_HTML_TAG.find_iter(line) {
        let m = m.unwrap();
        let (s, e) = (m.start(), m.end());
        if s > start {
            let text = &line[start..s];
            split_text(text, &mut tokens);
        }
        tokens.push(Token::new_tag(&line[s..e]));
        start = e;
    }
    if start < line.len() {
        split_text(&line[start..], &mut tokens);
    }
    tokens
}

fn split_text(text: &str, out: &mut Vec<Token>) {
    // split by whitespace but keep spaces as separate tokens, like Python
    let mut last = 0usize;
    for m in PAT_SPACE.find_iter(text) {
        let m = m.unwrap();

        let (s, e) = (m.start(), m.end());
        if s > last {
            out.push(Token { value: text[last..s].to_string(), tag: "".into(), kind: TokType::Word });
        }
        out.push(Token { value: text[s..e].to_string(), tag: "".into(), kind: TokType::Space });
        last = e;
    }
    if last < text.len() {
        out.push(Token { value: text[last..].to_string(), tag: "".into(), kind: TokType::Word });
    }
}