Skip to main content

muskitty_html5_tokenizer/
lib.rs

1//! MusKitty HTML5 Tokenizer
2//!
3//! Implements the tokenization stage of the WHATWG HTML parsing model
4//! (§13.2.5 Tokenization).
5//!
6//! The tokenizer consumes a stream of Unicode code points and emits
7//! tokens (start tags, end tags, comments, characters, DOCTYPEs, EOF).
8//! These tokens are consumed by the tree construction stage to build
9//! the DOM tree.
10//!
11//! # References
12//!
13//! - WHATWG HTML Living Standard: <https://html.spec.whatwg.org/multipage/parsing.html>
14//! - WPT test suite: <https://github.com/web-platform-tests/wpt/tree/master/html/syntax/parsing>
15
16mod entities;
17mod impls;
18mod trait_def;
19mod types;
20
21pub use impls::HtmlTokenizer;
22pub use trait_def::Tokenizer;
23pub use types::{DoctypeToken, ParseError, State, TagKind, TagToken, Token};