fhp-tokenizer 0.1.1

SIMD-accelerated HTML tokenizer with structural indexing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

//! Token types emitted by the HTML tokenizer.
//!
//! All variants carry `&'a str` references into the original input,
//! achieving zero-copy tokenization.

use std::borrow::Cow;

use fhp_core::tag::Tag;

/// A single token produced by the tokenizer.
///
/// All string payloads borrow from the original input (`'a` lifetime),
/// except entity-decoded values which use `Cow::Owned`.
#[derive(Clone, Debug, PartialEq)]
pub enum Token<'a> {
    /// An opening tag, e.g. `<div class="foo">`.
    OpenTag {
        /// Interned tag name.
        tag: Tag,
        /// Raw tag name slice from the input.
        name: Cow<'a, str>,
        /// Attribute list (may be empty).
        attributes: Vec<Attribute<'a>>,
        /// Whether the tag is self-closing (`<br/>`).
        self_closing: bool,
    },

    /// A closing tag, e.g. `</div>`.
    CloseTag {
        /// Interned tag name.
        tag: Tag,
        /// Raw tag name slice from the input.
        name: Cow<'a, str>,
    },

    /// Text content between tags.
    Text {
        /// The text content, entity-decoded if needed.
        content: Cow<'a, str>,
    },

    /// An HTML comment, e.g. `<!-- comment -->`.
    Comment {
        /// The comment body (without `<!--` and `-->`).
        content: Cow<'a, str>,
    },

    /// A DOCTYPE declaration, e.g. `<!DOCTYPE html>`.
    Doctype {
        /// The content after `DOCTYPE`.
        content: Cow<'a, str>,
    },

    /// A CDATA section, e.g. `<![CDATA[...]]>`.
    CData {
        /// The CDATA content (without `<![CDATA[` and `]]>`).
        content: Cow<'a, str>,
    },
}

/// A single HTML attribute (name-value pair).
#[derive(Clone, Debug, PartialEq)]
pub struct Attribute<'a> {
    /// Attribute name, borrowed from input.
    pub name: Cow<'a, str>,
    /// Attribute value, entity-decoded if needed.
    /// `None` for boolean attributes (e.g. `disabled`).
    pub value: Option<Cow<'a, str>>,
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn token_clone_and_debug() {
        let tok = Token::Text {
            content: Cow::Borrowed("hello"),
        };
        let tok2 = tok.clone();
        assert_eq!(tok, tok2);
        assert!(format!("{tok:?}").contains("hello"));
    }

    #[test]
    fn attribute_with_value() {
        let attr = Attribute {
            name: Cow::Borrowed("class"),
            value: Some(Cow::Borrowed("foo")),
        };
        assert_eq!(attr.name.as_ref(), "class");
        assert_eq!(attr.value.as_deref(), Some("foo"));
    }

    #[test]
    fn boolean_attribute() {
        let attr = Attribute {
            name: Cow::Borrowed("disabled"),
            value: None,
        };
        assert!(attr.value.is_none());
    }
}