yozora-tokenizer-html-block 0.1.0

Yozora tokenizer for HTML blocks.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

use yozora_character::{is_whitespace_character, AsciiCodePoint, NodePoint};

pub fn eat_start_condition6(
    node_points: &[NodePoint],
    start: usize,
    end: usize,
    tag: &str,
) -> Option<usize> {
    if !is_included_tag(tag) {
        return None;
    }
    if start >= end {
        return Some(end);
    }
    let c = node_points[start].code_point;
    if is_whitespace_character(c) || c == AsciiCodePoint::CLOSE_ANGLE as i32 {
        return Some(start + 1);
    }
    (c == AsciiCodePoint::SLASH as i32
        && start + 1 < end
        && node_points[start + 1].code_point == AsciiCodePoint::CLOSE_ANGLE as i32)
        .then_some(start + 2)
}

fn is_included_tag(tag: &str) -> bool {
    matches!(
        tag,
        "address"
            | "article"
            | "aside"
            | "base"
            | "basefont"
            | "blockquote"
            | "body"
            | "caption"
            | "center"
            | "col"
            | "colgroup"
            | "dd"
            | "details"
            | "dialog"
            | "dir"
            | "div"
            | "dl"
            | "dt"
            | "fieldset"
            | "figcaption"
            | "figure"
            | "footer"
            | "form"
            | "frame"
            | "frameset"
            | "h1"
            | "h2"
            | "h3"
            | "h4"
            | "h5"
            | "h6"
            | "head"
            | "header"
            | "hr"
            | "html"
            | "iframe"
            | "legend"
            | "li"
            | "link"
            | "main"
            | "menu"
            | "menuitem"
            | "nav"
            | "noframes"
            | "ol"
            | "optgroup"
            | "option"
            | "p"
            | "param"
            | "section"
            | "source"
            | "summary"
            | "table"
            | "tbody"
            | "td"
            | "tfoot"
            | "th"
            | "thead"
            | "title"
            | "tr"
            | "track"
            | "ul"
    )
}